hannahyh commited on
Commit
95ad783
·
verified ·
1 Parent(s): 5396bd5

Add files using upload-large-folder tool

Browse files
Files changed (50) hide show
  1. model/tachin_paired30_cache_20260923/README.md +85 -0
  2. model/tachin_paired30_cache_20260923/SHA256SUMS +264 -0
  3. model/tachin_paired30_cache_20260923/audit/cache_audit.json +57 -0
  4. model/tachin_paired30_cache_20260923/audit/comparison.json +155 -0
  5. model/tachin_paired30_cache_20260923/cache/complete.json +21 -0
  6. model/tachin_paired30_cache_20260923/code/scripts/audit_tachin_paired30.py +78 -0
  7. model/tachin_paired30_cache_20260923/code/scripts/build_tachin_paired30_cache.py +174 -0
  8. model/tachin_paired30_cache_20260923/code/scripts/cache_tachin_adapted_base.py +251 -0
  9. model/tachin_paired30_cache_20260923/code/scripts/cache_tachin_twla_w3.py +299 -0
  10. model/tachin_paired30_cache_20260923/code/scripts/summarize_tachin_paired30.py +77 -0
  11. model/tachin_paired30_cache_20260923/code/scripts/train_tachin_table1_editor.py +539 -0
  12. model/tachin_paired30_cache_20260923/code/src/tactile_editor/action_layout.py +123 -0
  13. model/tachin_paired30_cache_20260923/code/src/tactile_editor/action_objectives.py +233 -0
  14. model/tachin_paired30_cache_20260923/code/src/tactile_editor/action_output.py +71 -0
  15. model/tachin_paired30_cache_20260923/code/src/tactile_editor/architecture_ablation.py +423 -0
  16. model/tachin_paired30_cache_20260923/code/src/tactile_editor/auxiliary_objectives.py +264 -0
  17. model/tachin_paired30_cache_20260923/code/src/tactile_editor/config.py +160 -0
  18. model/tachin_paired30_cache_20260923/code/src/tactile_editor/contact_filter.py +91 -0
  19. model/tachin_paired30_cache_20260923/code/src/tactile_editor/direct_residual.py +327 -0
  20. model/tachin_paired30_cache_20260923/code/src/tactile_editor/editor_backends.py +196 -0
  21. model/tachin_paired30_cache_20260923/code/src/tactile_editor/editor_interface.py +96 -0
  22. model/tachin_paired30_cache_20260923/code/src/tactile_editor/encoder.py +209 -0
  23. model/tachin_paired30_cache_20260923/code/src/tactile_editor/full_cache.py +86 -0
  24. model/tachin_paired30_cache_20260923/code/src/tactile_editor/fusion.py +211 -0
  25. model/tachin_paired30_cache_20260923/code/src/tactile_editor/informative_data.py +177 -0
  26. model/tachin_paired30_cache_20260923/code/src/tactile_editor/memoryvla_baseline.py +104 -0
  27. model/tachin_paired30_cache_20260923/code/src/tactile_editor/model_adapters.py +287 -0
  28. model/tachin_paired30_cache_20260923/code/src/tactile_editor/native_tachin.py +201 -0
  29. model/tachin_paired30_cache_20260923/code/src/tactile_editor/rgb_world_editor.py +419 -0
  30. model/tachin_paired30_cache_20260923/code/src/tactile_editor/runtime.py +376 -0
  31. model/tachin_paired30_cache_20260923/code/src/tactile_editor/touch_study.py +196 -0
  32. model/tachin_paired30_cache_20260923/code/src/tactile_editor/trainability.py +99 -0
  33. model/tachin_paired30_cache_20260923/code/src/tactile_editor/training.py +180 -0
  34. model/tachin_paired30_cache_20260923/code/src/tactile_editor/unified_cache.py +122 -0
  35. model/tachin_paired30_cache_20260923/code/src/tactile_editor/visual_dynamics_pilot.py +215 -0
  36. model/tachin_paired30_cache_20260923/code/src/tactile_editor/world_residual_probe.py +362 -0
  37. model/tachin_paired30_cache_20260923/code/tests/test_tachin_native_editor.py +63 -0
  38. model/tachin_paired30_cache_20260923/environment.txt +7 -0
  39. model/twla_tachin_common_adaptation_20260922/README.md +45 -0
  40. model/twla_tachin_common_adaptation_20260922/SHA256SUMS +13 -0
  41. model/twla_tachin_common_adaptation_20260922/code/adapt_twla_tachin_common.py +374 -0
  42. model/twla_tachin_common_adaptation_20260922/code/cache_tachin_twla_w3.py +299 -0
  43. model/twla_tachin_common_adaptation_20260922/code/rebuild_twla_adapted_checkpoint.py +96 -0
  44. model/twla_tachin_common_adaptation_20260922/code/verify_twla_tachin_adaptation.py +128 -0
  45. model/twla_tachin_common_adaptation_20260922/provenance/COMPLETE.json +12 -0
  46. model/twla_tachin_common_adaptation_20260922/provenance/config.json +33 -0
  47. model/twla_tachin_common_adaptation_20260922/provenance/initial_config_used.json +115 -0
  48. model/twla_tachin_common_adaptation_20260922/provenance/initial_ten_domain_config.json +115 -0
  49. model/twla_tachin_common_adaptation_20260922/provenance/metrics.jsonl +0 -0
  50. model/twla_tachin_common_adaptation_20260922/provenance/verification.json +15 -0
model/tachin_paired30_cache_20260923/README.md ADDED
@@ -0,0 +1,85 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Tachin paired-30 policy caches and tactile-editor artifacts
2
+
3
+ This release contains the cached outputs and editor artifacts used for the Tachin
4
+ Table 1 experiments on 2026-09-23. The policy checkpoints were first adapted on
5
+ the Tachin training split. SmolVLA, VITRA, and T-Rex use the adaptation
6
+ checkpoints already published in `model/tachin_adaptation_20260922`. T-WLA uses
7
+ the separately published `model/twla_tachin_common_adaptation_20260922`.
8
+
9
+ ## Protocol
10
+
11
+ - Dataset split: the same fixed Tachin train/validation/test episode split for
12
+ every method.
13
+ - Plan/vision rate: 10 Hz.
14
+ - Tactile/editor rate: 30 Hz, with causal decision times at 0, 33, and 67 ms
15
+ after each plan time.
16
+ - Action rate: 80 Hz; each prediction contains eight future actions (100 ms).
17
+ - The first editable action indices for the three tactile decisions are 0, 3,
18
+ and 6.
19
+ - Tactile input is not threshold-gated in this release (`gated=false`).
20
+ - Normalization statistics use only the original Tachin training split.
21
+ - The held-out test split was used only for final evaluation, not checkpoint
22
+ selection.
23
+
24
+ The cache audit reports 18,208/1,006/1,169 plans and
25
+ 49,744/2,655/2,433 paired 30-Hz rows for train/validation/test. See
26
+ `audit/cache_audit.json`; its final status is `PASS`.
27
+
28
+ ## Contents
29
+
30
+ - `cache/shared/{train,val,test}`: aligned tactile pressure, masks, action
31
+ targets, MANO beta, row identity, and causal future-pressure arrays shared by
32
+ all methods.
33
+ - `cache/bases/{smolvla,vitra,trex}`: cached action predictions from the three
34
+ Tachin-adapted baseline policies.
35
+ - `cache/bases/twla`: T-WLA W3 cached action predictions plus image tokens,
36
+ visual memory, clean RGB world-model latents, ROI targets, and validity
37
+ metadata. Each split has two shards.
38
+ - `editors/{model}/best.pt`: selected tactile-editor checkpoint.
39
+ - `editors/{model}/config.json`, `epochs.jsonl`, and `summary.json`: training
40
+ configuration, validation history, and final physical metrics.
41
+ - `editors/{model}/test_*.npy`: predictions for normal touch, shuffled touch,
42
+ and the zero-touch control.
43
+ - `audit/comparison.json`: Table 1 comparison and physical errors.
44
+ - `code`: the exact cache, training, audit, and tactile-editor implementation
45
+ snapshot used for this release.
46
+ - `SHA256SUMS`: integrity hashes for all other files in this directory.
47
+
48
+ The zero-touch reference gate was explicitly audited: the editor returns the
49
+ base action to numerical precision (maximum action difference at most
50
+ `7.5e-9`).
51
+
52
+ ## Minimal loading example
53
+
54
+ ```python
55
+ from pathlib import Path
56
+ import numpy as np
57
+
58
+ root = Path("tachin_paired30_cache_20260923")
59
+ target = np.load(root / "cache/shared/test/target_action.npy", mmap_mode="r")
60
+ mask = np.load(root / "cache/shared/test/action_mask.npy", mmap_mode="r")
61
+ smolvla = np.load(root / "cache/bases/smolvla/test/base_action.npy", mmap_mode="r")
62
+ ```
63
+
64
+ Use `code/scripts/train_tachin_table1_editor.py` to train an editor from this
65
+ cache. Paths stored inside provenance JSON files are the original absolute run
66
+ paths; use paths relative to this release when reproducing the experiment.
67
+
68
+ ## Selected editor results
69
+
70
+ Physical test errors are measured against the Tachin ground truth.
71
+
72
+ | Method | Translation (cm) | Rotation (degree) | Finger MPJPE (cm) | Selected epoch |
73
+ |---|---:|---:|---:|---:|
74
+ | SmolVLA + editor | 1.2413 | 4.6655 | 1.3737 | 10 |
75
+ | VITRA + editor | 1.3520 | 5.5228 | 0.6283 | 9 |
76
+ | T-Rex + editor | 1.2979 | 4.7934 | 1.1978 | 9 |
77
+ | T-WLA W3 + editor | 1.2665 | 4.9293 | 0.4751 | 8 |
78
+
79
+ ## Code and environment
80
+
81
+ The source snapshot corresponds to repository revision
82
+ `0a2fec56fad4e2d830403d1371e44dfbd5edc270` on branch
83
+ `tactile_editor_root`, with additional local experiment files included here.
84
+ Package versions are recorded in `environment.txt`.
85
+
model/tachin_paired30_cache_20260923/SHA256SUMS ADDED
@@ -0,0 +1,264 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 011e867c049464f7176fccd0d1a4cf550d90ac0e2bd1406548559e1eecb079ac ./README.md
2
+ 234fac67e7a7bb128cefbdb56c7c78d4f0852b329202fd8b01d56d015ff5f1c5 ./audit/cache_audit.json
3
+ 26adbd78cb19c8d5deec76fb03bc8c20659bfadf05d7addba12f7e813667cc5d ./audit/comparison.json
4
+ 2080313884c15626cefee53153b15a3ba3d316f6d31e04c9b82e0eb4d30bf744 ./cache/bases/smolvla/test/base_action.npy
5
+ 8c17503a945e74df1b6073faabcd7171bc9e62188a2f8bb8888c85e86add57d5 ./cache/bases/smolvla/test/complete.json
6
+ b7a466bff507433809b0252ec67be686f9cc5779518d603e80103fcd56dbf6ea ./cache/bases/smolvla/test/row_complete.npy
7
+ 350ffc92743684d1b17d55fb0154030260f1e66941e2ac3064d65d9320b064e7 ./cache/bases/smolvla/test/smoke_base_action.npy
8
+ 9c5a4540fcd1e0dc307098950bc9aa6baabfd7f809ae253e416d529379197939 ./cache/bases/smolvla/test/smoke_complete.json
9
+ ac537e78759740605ab6c13c1755caf6d517d19b47b4b20cf679e5fed978ed67 ./cache/bases/smolvla/test/smoke_row_complete.npy
10
+ 582829bcd03feb04814e06a19e371e6b5be4ed19ab811cb9bb2131c5b371d4d6 ./cache/bases/smolvla/train/base_action.npy
11
+ 8810e5cd488786b89f22e9e4c46f6dac670cd0f0c1a485e7a8a425f193731a32 ./cache/bases/smolvla/train/complete.json
12
+ 986043163cdc5d322abd94b3f38aa38b27bbe0cccdaf3388cd912d63d2aa17ff ./cache/bases/smolvla/train/row_complete.npy
13
+ 03a701b74f9e23c0a272768ac4d04d33257c53623a5c305226fcca65f17f2f66 ./cache/bases/smolvla/train/smoke_base_action.npy
14
+ 5dbf6b8c835c9a38deb11bfcf8a3b9782ee83c49abad4791139c2ca28950cb66 ./cache/bases/smolvla/train/smoke_complete.json
15
+ ac537e78759740605ab6c13c1755caf6d517d19b47b4b20cf679e5fed978ed67 ./cache/bases/smolvla/train/smoke_row_complete.npy
16
+ 5007d2be755f0ae19eb02955b59d4f2c779933c4e2e990d1c57cfbbaced2e449 ./cache/bases/smolvla/val/base_action.npy
17
+ c5ecbada9581dfe6ae627bea5ade115b9dd6bda528d060bb7631f23885a3a1a1 ./cache/bases/smolvla/val/complete.json
18
+ ec8bbb1bd9074ae02818c9b23996b2bfd206ff8d6088db2b1f529c1c15ef845f ./cache/bases/smolvla/val/row_complete.npy
19
+ b30bbf9880451281e79f165c517f9dee25a100426be3f47e4c4361cf211b34fb ./cache/bases/smolvla/val/smoke_base_action.npy
20
+ 22309cb83f233a9b83f557ef0a727b218a61183a341359090bbcff0a30001eb1 ./cache/bases/smolvla/val/smoke_complete.json
21
+ ac537e78759740605ab6c13c1755caf6d517d19b47b4b20cf679e5fed978ed67 ./cache/bases/smolvla/val/smoke_row_complete.npy
22
+ 1ddfe3b9ea6bbab2b8feda241005e7135fc054c6c0c13d3e42819cc5fb18e0e8 ./cache/bases/trex/test/base_action.npy
23
+ 3390c158bb4cbd8fe1bddb6c95e13699be7795906bf687e2fe75ff6ed7a0f07c ./cache/bases/trex/test/complete.json
24
+ 300e7137567e46dc1ed50689269205253a37e76cdbc265c871dc91d832b2ab37 ./cache/bases/trex/test/complete_shard_00_of_04.json
25
+ 217e4d0e35d39fbdd533eee84e30f28fcb44967891e9a4f490ebe0da4855066c ./cache/bases/trex/test/complete_shard_01_of_04.json
26
+ 6d8c718507bc7e819b55a8e92ac6925271df334411905f73dbdf081b78e48a16 ./cache/bases/trex/test/complete_shard_02_of_04.json
27
+ dc1998a3c1057f24a833ae2cd8a76f9f7ad62f05e7bf605c698f269054cdba80 ./cache/bases/trex/test/complete_shard_03_of_04.json
28
+ b7a466bff507433809b0252ec67be686f9cc5779518d603e80103fcd56dbf6ea ./cache/bases/trex/test/row_complete.npy
29
+ 10ecb5687069cc23a158fee0a15967511bfca2b6090a075113f2ad4b1ef4b11a ./cache/bases/trex/test/smoke_base_action.npy
30
+ 582afc5b9818a720e974ab45340c7ebd5259b01aa2bd14309c73d801b4426bd0 ./cache/bases/trex/test/smoke_complete.json
31
+ d13bab448a58cfff3fb7336525998915b7cde741811292a048f4f925e89edb8f ./cache/bases/trex/test/smoke_row_complete.npy
32
+ a213b36f4483861a589ccb946d39d73c2189f182c83050372971c3cc762d64d2 ./cache/bases/trex/train/base_action.npy
33
+ a7a8408ee35d00c551e56fe1aa14a45f106345844bf76fa2d95d31e384b5dd5a ./cache/bases/trex/train/complete.json
34
+ a7495d9e8a981f6ebdc3e5654e832fe7fe758afa165df078cd18b95edab57683 ./cache/bases/trex/train/complete_shard_00_of_04.json
35
+ 4a3703a7cbf8c5e721e8522a68be126e5d3249181783ef044d0cba775f3edb5b ./cache/bases/trex/train/complete_shard_01_of_04.json
36
+ d7d44f9b1fc631a66437afdaabb1dc27fea63169b231191706f23ca5f3d7b341 ./cache/bases/trex/train/complete_shard_02_of_04.json
37
+ 72a62a216f146b188280bad3a4749a3bed7b34687d13903c8d96a53279f40543 ./cache/bases/trex/train/complete_shard_03_of_04.json
38
+ 986043163cdc5d322abd94b3f38aa38b27bbe0cccdaf3388cd912d63d2aa17ff ./cache/bases/trex/train/row_complete.npy
39
+ ebecfd58f31a466878a9331e457f5f7e2abb17b08d4f1d9e171899d769352073 ./cache/bases/trex/train/smoke_base_action.npy
40
+ 4a38c1b9bfc50285cea78c59ab851d948b5348b90375cac67f8dde63ff5089cf ./cache/bases/trex/train/smoke_complete.json
41
+ d13bab448a58cfff3fb7336525998915b7cde741811292a048f4f925e89edb8f ./cache/bases/trex/train/smoke_row_complete.npy
42
+ 7b87c9ca444e112e020ce21606e18ef20214e39f50cd09df33bdfb1e97d15489 ./cache/bases/trex/val/base_action.npy
43
+ 90431ef3b9199120e450613eefa761a5534e23973fb201070b0cb32cfa184fa3 ./cache/bases/trex/val/complete.json
44
+ b9ef7a09fb539886dd9675419cecbff93d9be271aa11d34b8665f80381879348 ./cache/bases/trex/val/complete_shard_00_of_04.json
45
+ e9c585ab970ba79b1eb6d6af6efac178c017305f8457d7380fb5733ae83840d8 ./cache/bases/trex/val/complete_shard_01_of_04.json
46
+ 30232b5ad4050127d1f3c27a157ddd94d477d8be83def05eccd7ca89e2eddec1 ./cache/bases/trex/val/complete_shard_02_of_04.json
47
+ 6fa4df77f8ccb4f0e52cdd684a6f08fadf13db1975b6ee08d3aa52e3b4ae4cce ./cache/bases/trex/val/complete_shard_03_of_04.json
48
+ ec8bbb1bd9074ae02818c9b23996b2bfd206ff8d6088db2b1f529c1c15ef845f ./cache/bases/trex/val/row_complete.npy
49
+ f71427f7b16dca5d9009dd7cbf51be3a6a994e7834f5c4dafd65704391ff6f30 ./cache/bases/trex/val/smoke_base_action.npy
50
+ 38b66d9746aae67a7d68370f7ea0836120d654da010ff057db4553d3b5c579c0 ./cache/bases/trex/val/smoke_complete.json
51
+ d13bab448a58cfff3fb7336525998915b7cde741811292a048f4f925e89edb8f ./cache/bases/trex/val/smoke_row_complete.npy
52
+ 3fa7d31355427bba1eb38c2e3fe57a16952f00e6b25f4722178b042df920e9ba ./cache/bases/twla/test/shard_00/base_action.npy
53
+ 88d4a91bad54d4753300c74aabd6d563abc8b4a35619dc82483c6d9245a06fa4 ./cache/bases/twla/test/shard_00/clean_latent.npy
54
+ 52ca86fa559c3a4149eb65adbd8160b6b4fda5a1ad8312de0d7b32708268b67c ./cache/bases/twla/test/shard_00/complete.json
55
+ db7c81a10ee497c8575d94ba9f232a755defbf9e41ca6b734096d5d7ac29d103 ./cache/bases/twla/test/shard_00/final_memory.npy
56
+ fc6e0a84d62bbb9d054a428a019f94a89bb30aff8ba006bad254e98a4c40af14 ./cache/bases/twla/test/shard_00/future_roi.npy
57
+ dbbde1b30e9708a9d4218269585b2c0ec191753945a553ea9488c524b93a1cf3 ./cache/bases/twla/test/shard_00/image_tokens.npy
58
+ a8a06cb8e9651cbdd27ce6752e8cc5ab4327db6c0673c176cfa3054e2bb29f6a ./cache/bases/twla/test/shard_00/query_index.npy
59
+ 9ad5cfa7086814e276e26484795902a41c83b13bf7c60d2dfd2cb97629ef8610 ./cache/bases/twla/test/shard_00/rgb_valid.npy
60
+ 6a87724b86d8d5ed460a5a8be16d5334a351effdef32f8281d19455f17ae0a41 ./cache/bases/twla/test/shard_00/row_complete.npy
61
+ 581d0dc8abcac0a070ccc451d60470d3c54227bf9196c8ab8b6b38118e6ca11a ./cache/bases/twla/test/shard_01/base_action.npy
62
+ 47a16c5a2dab718bef42c29038b3dcd27c694e6e4443365f5f9ad4783dc02177 ./cache/bases/twla/test/shard_01/clean_latent.npy
63
+ a06e37eaa3f26f9c8499a37c49fd669573f2c769e8cc1f749109c79524863214 ./cache/bases/twla/test/shard_01/complete.json
64
+ 290d4fc65379fd8927c183ce67d7604f611067b68452154b98c5722b7acde94f ./cache/bases/twla/test/shard_01/final_memory.npy
65
+ c051b0e49b771a8c7571018e625321c194f0e5e7bd012efe89f5d5cb0f537c1b ./cache/bases/twla/test/shard_01/future_roi.npy
66
+ 81582ec12020a0d0927c0c007d970e35aeb8c397a4d214c794b7ae459e90bb69 ./cache/bases/twla/test/shard_01/image_tokens.npy
67
+ 658259bd8faeaed759725fcc1bc44cb87192b0727a0e4cfb59456fe78d8c08c0 ./cache/bases/twla/test/shard_01/query_index.npy
68
+ b7cd2b4647e6edfa390e89a0dad83c9472386b161c0fbe725bf70a511c35b378 ./cache/bases/twla/test/shard_01/rgb_valid.npy
69
+ f0a9c6d971f34c731260e6c33f6cfa1443f559950087b1a2b3ac786d5f70ee38 ./cache/bases/twla/test/shard_01/row_complete.npy
70
+ 3a6f6993f02aa0451068c65600d8457014406e0bcc3d8dd33c36d53bb6f240c3 ./cache/bases/twla/train/shard_00/base_action.npy
71
+ 2615630dc6fccf733397fc85f3b1b38e69ddb93b565a9c03447f1f2081575106 ./cache/bases/twla/train/shard_00/clean_latent.npy
72
+ 66915d828f4ba10e14ddcb61223b84da4a5629b397cfc012e1f4e7aed567e478 ./cache/bases/twla/train/shard_00/complete.json
73
+ 4ee9df3923fcf9c9b35dac3fd237a7723e9677d35eeef3ff2a6f2f35eea09844 ./cache/bases/twla/train/shard_00/final_memory.npy
74
+ 6457be467bc9a04a96f1fe4f6a373f09e0d5b02b5a64dd31863ba89c52735e78 ./cache/bases/twla/train/shard_00/future_roi.npy
75
+ accf30aad2c0dd61bcf85bf40d9c6881ec11b01413c1ee678583d6a19afed418 ./cache/bases/twla/train/shard_00/image_tokens.npy
76
+ f1fe407378b7cbc941e1ff7199ad403ad0919ac333c4746804b158425304bc31 ./cache/bases/twla/train/shard_00/query_index.npy
77
+ 8f35c351e5a67a000d3f55071ef6f753d7285567703f2c2ac92520122406ab57 ./cache/bases/twla/train/shard_00/rgb_valid.npy
78
+ 7f60693c7b55eab0c4c1d1449c3f4ef7f695023c20b975a00cef6e5288a8ade7 ./cache/bases/twla/train/shard_00/row_complete.npy
79
+ cd39bb8d1fa62ebc15b669c7d2cec56b8447259df6d2f4d632eddc321f0389bf ./cache/bases/twla/train/shard_01/base_action.npy
80
+ 7b4fe8cc0b0c74d4b9ae2529f4dab5afb418b9e7dd016947017bce5794932cf1 ./cache/bases/twla/train/shard_01/clean_latent.npy
81
+ 4b1b279c01fa4deb577f1a54c877ed588faf325192e05b03669b116f0a94e9fa ./cache/bases/twla/train/shard_01/complete.json
82
+ 49e9aa11e02ba8023431ffb763e8bb567f253f991dddd7774fef587a1a078818 ./cache/bases/twla/train/shard_01/final_memory.npy
83
+ fd76cfeabaa2d773a9e2d10ce18ad1520e5d384dd3cdfae9417af4b32d567d1c ./cache/bases/twla/train/shard_01/future_roi.npy
84
+ 271ae4ab3219cbb0944db0df693c7a1b8694b7e171f33c6daaa43a641d9756bc ./cache/bases/twla/train/shard_01/image_tokens.npy
85
+ 62a3a083607f759477828c3f3d958b347578ca48fc95fda7d446f2e97dcfe49a ./cache/bases/twla/train/shard_01/query_index.npy
86
+ 25953dbc165e8f60826eb297f145a41ad23e46be5de98e5554a8c84bdb3b1f67 ./cache/bases/twla/train/shard_01/rgb_valid.npy
87
+ 7f60693c7b55eab0c4c1d1449c3f4ef7f695023c20b975a00cef6e5288a8ade7 ./cache/bases/twla/train/shard_01/row_complete.npy
88
+ 471795a441696ce123880aa10c01536a9088656daa64df25c924dbbcd04cf43b ./cache/bases/twla/val/shard_00/base_action.npy
89
+ 7a58584c88c9dcc87481b7d386f4a618ca7cf17c476afda014aba57a66060fcb ./cache/bases/twla/val/shard_00/clean_latent.npy
90
+ 275cad954848f4edeeb2b1df8f5c9122351dea58b62388761c5e9cc592afae89 ./cache/bases/twla/val/shard_00/complete.json
91
+ c3ea8878477559f7608596a544736c512a7189bcbc11ddc8bf7b9ed1a09ca37e ./cache/bases/twla/val/shard_00/final_memory.npy
92
+ de2aa3a0dc1d87729199491eb25b433b56424398492720f28e7eb37bc198303f ./cache/bases/twla/val/shard_00/future_roi.npy
93
+ a5ef36109052b4f13fd12c3c766b9b0979b76a897d84b7afeb1f8fd19a9fabbb ./cache/bases/twla/val/shard_00/image_tokens.npy
94
+ f24f2f19907d8c0f41ab9b48dd2878a9621ac86c1726a6994260039629b11467 ./cache/bases/twla/val/shard_00/query_index.npy
95
+ 9f4c0a48e103b67f33de2c4861ac6b54aca17029049c3f343b4c443263c12933 ./cache/bases/twla/val/shard_00/rgb_valid.npy
96
+ bfd4644c8144920f31eff79f61596e2be33fa5fa79059f1b8812ab88b2acf984 ./cache/bases/twla/val/shard_00/row_complete.npy
97
+ acc81026b51aa6a34210357c083639c930b3e1d1f631124353c6f35daf588227 ./cache/bases/twla/val/shard_01/base_action.npy
98
+ 453f7aa7fc809e569dd52c53e21a9bb862246d967f36ce94efaa5159405a029f ./cache/bases/twla/val/shard_01/clean_latent.npy
99
+ 9ede76b0c8f75d5fd7ffcfbeb0b6432ac2e7076c5e6d6a4a498b2bbfc7e232c7 ./cache/bases/twla/val/shard_01/complete.json
100
+ 3b3042e33ad920bc9a410e5b88517b3c90fc6d8fe768236fddd8511b4a2c84dc ./cache/bases/twla/val/shard_01/final_memory.npy
101
+ 46ef458b9394ea1d79ee5c02e2f2f9ce401d0a45765395d232ff47f38c89c0c4 ./cache/bases/twla/val/shard_01/future_roi.npy
102
+ 490e8402141835a26f4b1c2d4ab415ec59d56644d02c3b0ba8eafca40c8617fa ./cache/bases/twla/val/shard_01/image_tokens.npy
103
+ 51514a016dd1890ae289f77da43997114fe68631a706907309388fd9e090a5b3 ./cache/bases/twla/val/shard_01/query_index.npy
104
+ 9f4c0a48e103b67f33de2c4861ac6b54aca17029049c3f343b4c443263c12933 ./cache/bases/twla/val/shard_01/rgb_valid.npy
105
+ bfd4644c8144920f31eff79f61596e2be33fa5fa79059f1b8812ab88b2acf984 ./cache/bases/twla/val/shard_01/row_complete.npy
106
+ a95f1ab323df8f5635521757f71833cc2110411e6a91f14297e57fb0c0808ef4 ./cache/bases/vitra/test/base_action.npy
107
+ ba9727c21ade6329c2bc6e7b741146aebdb3fbf370e018cee876f1d97739297c ./cache/bases/vitra/test/complete.json
108
+ b7a466bff507433809b0252ec67be686f9cc5779518d603e80103fcd56dbf6ea ./cache/bases/vitra/test/row_complete.npy
109
+ 9338306750dc7822ced14d17e0c66377259d4d58b93f2fff0e6d754be869588f ./cache/bases/vitra/test/smoke_base_action.npy
110
+ 99b9ee6097d40303565c4297e76385e52b8b2ff8b91a92557be3a9ecf9cfec47 ./cache/bases/vitra/test/smoke_complete.json
111
+ ac537e78759740605ab6c13c1755caf6d517d19b47b4b20cf679e5fed978ed67 ./cache/bases/vitra/test/smoke_row_complete.npy
112
+ b7d31839721395353da069e46163e92b6ed2f4bfabc582df84c69db3971f0cc9 ./cache/bases/vitra/train/base_action.npy
113
+ 36416b564cd2115e718bb1bac066897a32089104056c0307f331ececf1f0253f ./cache/bases/vitra/train/complete.json
114
+ 986043163cdc5d322abd94b3f38aa38b27bbe0cccdaf3388cd912d63d2aa17ff ./cache/bases/vitra/train/row_complete.npy
115
+ 137cdbe6533a87127f56cf32cd748ac31f79ebdd338743b53a810c757c00dd58 ./cache/bases/vitra/train/smoke_base_action.npy
116
+ a871055da5d32e7f863f303dea8b2de5765f8ede37ae34a01d6298a444bfe69e ./cache/bases/vitra/train/smoke_complete.json
117
+ ac537e78759740605ab6c13c1755caf6d517d19b47b4b20cf679e5fed978ed67 ./cache/bases/vitra/train/smoke_row_complete.npy
118
+ 1ded4e5266fbb534454c91d43ca4ca716841a84d8f3296e783e5ccf16301a455 ./cache/bases/vitra/val/base_action.npy
119
+ 087760382ecfc13b152e7bb42ca2b3321fee4a45aa5dc71ce0bf936595e24e76 ./cache/bases/vitra/val/complete.json
120
+ ec8bbb1bd9074ae02818c9b23996b2bfd206ff8d6088db2b1f529c1c15ef845f ./cache/bases/vitra/val/row_complete.npy
121
+ 5a0269ef27573f876a1ee35ceff21068249c1076d106e12f598cb40d248623e6 ./cache/bases/vitra/val/smoke_base_action.npy
122
+ 0a3c55d318db2ca6a24738c83560e97290b9de204eaf7e299dfa82a0a16a8e7f ./cache/bases/vitra/val/smoke_complete.json
123
+ ac537e78759740605ab6c13c1755caf6d517d19b47b4b20cf679e5fed978ed67 ./cache/bases/vitra/val/smoke_row_complete.npy
124
+ cd5818dc9360a1d5aead430c597c06202e5cb607d66918122b9107bed9d1d8e4 ./cache/complete.json
125
+ 072e87f8ca325e3f1868a93aea53dc69478d588293fdd092efd0e5f06c38635f ./cache/shared/test/action_mask.npy
126
+ 998b06226830defdd6769987a929fce9a3c0e682f6762d8738f9c8a30733d1af ./cache/shared/test/beta.npy
127
+ 8c7adad5564c3cfa43472f65fbca41b1497f943d0dea93a3fe2d4a4783fe46fa ./cache/shared/test/complete.json
128
+ a1082dff98c0c90fdad594d35eb07be034d561f4bcddf9b6a889d6bd12d3f6e1 ./cache/shared/test/future_extra_pressure.npy
129
+ cd418be7c9941c85e8d5606f0a15c4d245b0d454c9a98adfcd207e9ef75ccf49 ./cache/shared/test/future_extra_pressure_mask.npy
130
+ e276a2a27c0685a7c633c320e3fce944004d3f9e33440c031d6b1ab5389cbc02 ./cache/shared/test/future_extra_valid.npy
131
+ 9abe6b8fd405f60d967dc532603f266228c8730e6e8b520a2e6a0033801df486 ./cache/shared/test/future_pressure.npy
132
+ 926e708b285cc8b6e94d73a126a8b714abcbf641a48aefe6a3cdd2ff31ed46ee ./cache/shared/test/future_pressure_mask.npy
133
+ da672a934da52c9e76fe58614712fff4d695dbb1de27958bac53950de40055f6 ./cache/shared/test/future_valid.npy
134
+ 57e56a41ca45b82f53d0e2333c80d98d5413edf27d9f3088e57017f4cf25191d ./cache/shared/test/identity.npz
135
+ 455055bc47f3b7acf70a023d0bbaa60446b06dedb8a184005b12809edf1dcde4 ./cache/shared/test/pressure.npy
136
+ 13937516d2348601e25174ca47fd9545a0e0cd7739c9be60a24ddb766684d7ed ./cache/shared/test/pressure_mask.npy
137
+ 328e71ba351a243c5bcd6009b19d7c0aa7c2f32753738558e276d1c2c46c272f ./cache/shared/test/sample_valid.npy
138
+ b7a8a959c5933cef6bd2cbadc5bf0d2feca69bb46c9a6e2cd1915ca73054ceea ./cache/shared/test/target_action.npy
139
+ 7e684b5c919fa1a1c1879e054917f61140ce9133b9f4c45b661556af6ee40ac6 ./cache/shared/train/action_mask.npy
140
+ b0ad9e9a421c1890710298662d54ce21293f5f697eaec875ac4f2a7c77b7b31a ./cache/shared/train/beta.npy
141
+ 34c4c29d1624479855ddf6bda36a617f66bfde88bdb31df2378d9c56c0cd2053 ./cache/shared/train/complete.json
142
+ 7daa9135b33fe209efc890a822574bb46d289ba73d30750ec67f8097c82244a2 ./cache/shared/train/future_extra_pressure.npy
143
+ ebc23669e79bbe51d672d30e510f7b59ee7e9b0922d565a47cc4c2a0bdbf35dd ./cache/shared/train/future_extra_pressure_mask.npy
144
+ 9efea0fb8bba7bca3c7921f07ff55f43d4edee264791885800494c1bbe8b1492 ./cache/shared/train/future_extra_valid.npy
145
+ 90a63fe771ce15b192171ec773507205615c8962a94cc39977182b793ab591be ./cache/shared/train/future_pressure.npy
146
+ 7eed9677354f55d0afd92107ec8f05aca3fb059db3db3b56bbe143d17f452923 ./cache/shared/train/future_pressure_mask.npy
147
+ 0cbf19384a4ffcafed6dd87868e167512ba59ab64652d08fabe20062f110e0ab ./cache/shared/train/future_valid.npy
148
+ 6062ae9d1969eb09cbbf378963f02376d7d5525db04be06970ebf3815ef8cf56 ./cache/shared/train/identity.npz
149
+ 1534ad0f7e0aaabc8a871317618aa0ab7c62be1fb88bae480280a9ac046adbe0 ./cache/shared/train/pressure.npy
150
+ 06bfaedd2e75f9ad1f8322ebd2402b2652ef0a8e0e6c42a434d8d740b0a3bd3f ./cache/shared/train/pressure_mask.npy
151
+ ce00e301736362e1d1efa0c182685d1e77d431c763200c6f7e5e34bee66dd97a ./cache/shared/train/sample_valid.npy
152
+ 42a48f1b21639ccd00f9bc49553440ee220f64beb2285f8db4456be8d0c32693 ./cache/shared/train/target_action.npy
153
+ 8c263711f368b6475605f49198a2b6a5ba9a94f615d5cd27bb5a600348a2669b ./cache/shared/val/action_mask.npy
154
+ 7e15197f5c45129ce8f30d0c901eb339126c5237b63db24f441347f751f0dc08 ./cache/shared/val/beta.npy
155
+ 339b74ea7926aec4ef516246fdf671fbb33bda4f9a8bc2d0a91f23fbbd988006 ./cache/shared/val/complete.json
156
+ 80c9029f6be000061b59527d2198c98549bdef1a56e4e1115363860c59a278f4 ./cache/shared/val/future_extra_pressure.npy
157
+ fd608aa9533c5ea16596d0d80ebe5efdedeba2edf98b8cdd98c0b0df9cf820e3 ./cache/shared/val/future_extra_pressure_mask.npy
158
+ c6a671eead7588e91177cf5ac9e90e61c799c6dd21cb7e231d5251a3c224dd50 ./cache/shared/val/future_extra_valid.npy
159
+ 050d008fafbbc7a20f064eac31b727e12af4ddd3ea8631b8cc82398bcaece11d ./cache/shared/val/future_pressure.npy
160
+ 6056c81a8603fb3edef9ca066cbc9dd7896d132b8658c599759586c28d782f5a ./cache/shared/val/future_pressure_mask.npy
161
+ 44a1aeb5bb4c8f72868cb6663b8e0d4c0ffc16d90a45f2106744b3b2d9023067 ./cache/shared/val/future_valid.npy
162
+ 3dab5808ba13c98cd344ac7b94d3938feef570570b4b4788501caea9083792b5 ./cache/shared/val/identity.npz
163
+ 32bf4eed1d0d85cfdc0fae69ea5557813407a155ca2dead87f42d084812478ee ./cache/shared/val/pressure.npy
164
+ 9dc4c55be149a56a8e97c9491d0d96212bfc8af83d43ab2278761b9a31804800 ./cache/shared/val/pressure_mask.npy
165
+ 549f1b76cddf05bdbc7ec1799585526e74cb3f9d48da243b095b2fc8b2e6bd49 ./cache/shared/val/sample_valid.npy
166
+ eb575e92d2ed75b4e9068d21315cac627d6e404af74291f8a570bd95fef54b16 ./cache/shared/val/target_action.npy
167
+ 7441a432cad24f900a8d47c7cbbfe559de716c53eb82330ef82edfcd41165e5b ./cache/statistics.npz
168
+ 8117bc566c3728c63b5eeb08bbb06772ae83e8629e7e9c8be246a2516836d694 ./code/scripts/audit_tachin_paired30.py
169
+ 726e38f8c64cd119653edb4863659742277cefcc8c574a73aae386184ab74334 ./code/scripts/build_tachin_paired30_cache.py
170
+ 9c10f52ac1adec1dd488dff2f3fd992200379431b7f70606cfdf6399765245ed ./code/scripts/cache_tachin_adapted_base.py
171
+ f89146ac7f47157deea98e6dd713093f5c80d25af811a8581f553e593de9e7ea ./code/scripts/cache_tachin_twla_w3.py
172
+ 6cd5ea561f79c36ec34407c77fce721fe3358c8d01bc596fed85b057a4af9ba3 ./code/scripts/summarize_tachin_paired30.py
173
+ 75e6cb8d5ee3117bbf000e28595f69fa9409efdd7e307afda51488e9c2127f0a ./code/scripts/train_tachin_table1_editor.py
174
+ 63d688862a2f82dcf06c182947cafc00917698c3c5d551372f956860a534c46b ./code/src/tactile_editor/__init__.py
175
+ 9259591d593df5d6d661c7afccc54886a5b7ef2f9493c2ce79ae3d19e304bc30 ./code/src/tactile_editor/_baselines/__init__.py
176
+ e5a0703829318d9704109d11ad068e8fa6fd83a0899ba2eaa86c2d0b2ecf8fe6 ./code/src/tactile_editor/_baselines/provenance.json
177
+ e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 ./code/src/tactile_editor/_baselines/smolvla_official/__init__.py
178
+ ca4af48d152030f4741068308f9b0faac63af18178eefd8c9cbf2efe2b776f34 ./code/src/tactile_editor/_baselines/smolvla_official/config.py
179
+ 8ec960f681de59fc47bcc9dc0a048b5f6f095ee83259e4ced6b85d2196c414fd ./code/src/tactile_editor/_baselines/smolvla_official/flow.py
180
+ f8f26dc7af38af10db0d012d527a53c77493a034d96c0fda4e7462d2f6af0164 ./code/src/tactile_editor/_baselines/smolvla_official/group_loss.py
181
+ c7f6dac2430db912f277fd8442011269207cd3e163a88705eceb55436443b733 ./code/src/tactile_editor/_baselines/smolvla_official/model.py
182
+ 8b17d09c5ab9b68e324e1b4f5011ab21e8d66165d8c955f3d36c19469229619e ./code/src/tactile_editor/_baselines/smolvla_official/smolvlm_with_expert.py
183
+ 689fea725369aba4f7bb5eb72112eb7f96911061bbde225113793626d7b687b2 ./code/src/tactile_editor/_baselines/smolvla_official/vla_utils.py
184
+ e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 ./code/src/tactile_editor/_baselines/vitra_original/__init__.py
185
+ 306ce5d342acbe665c9160771b4c3711463a70598fddbc39a41df5362d7bdbc7 ./code/src/tactile_editor/_baselines/vitra_original/config.py
186
+ 43f8e9932aac6f98258bdf7902815b06c37a9e6f619f2733ce242d0bdda396ee ./code/src/tactile_editor/_baselines/vitra_original/diffusion.py
187
+ 7448464d79b9bbda800688798ba825e1864f9240502d2a5f6848e5d650074b73 ./code/src/tactile_editor/_baselines/vitra_original/dit.py
188
+ bb021ee4c9d8f38db633f1f60fc8f9ffc6c4b963f442850233f794c42a79524a ./code/src/tactile_editor/_baselines/vitra_original/group_loss.py
189
+ edaaf0ce60a97459594f034a53a8edf9b1b07d2354cecf829ac74e957848bdc7 ./code/src/tactile_editor/_baselines/vitra_original/model.py
190
+ b0816ec6cd3f66e61bbd5ebd77b393a794a847fa0e801ca6a174b82ba6876fc7 ./code/src/tactile_editor/_baselines/vitra_original/prompt.py
191
+ f8e4f6c753f660201398a544d2b8f32bb94b573573acb21037221a341cb68807 ./code/src/tactile_editor/_baselines/vitra_original/vlm.py
192
+ 5cfaab144e97a49fc928cc04ab5fd77dbaa380d22bed38ec98f007feb7babe28 ./code/src/tactile_editor/action_layout.py
193
+ 2179bb8ecba4dc3a74086853c0c61ca1bbd2e2b1021c0054438f6541d4a6a144 ./code/src/tactile_editor/action_objectives.py
194
+ 5e09d6a397a5e048cfe201c572fa9f3f7660743c7cba3ce6b7ccf0e654ca5430 ./code/src/tactile_editor/action_output.py
195
+ 7ef6141988b5e7bb33e44537b4a8a226b2934b9a1e685545ed7f71e90f72a77f ./code/src/tactile_editor/architecture_ablation.py
196
+ a3f3b5ac96322efbbedb496f562e5ba34e39cdc767da3105dfb9bb10f28fd072 ./code/src/tactile_editor/auxiliary_objectives.py
197
+ 0ffc136277938b915e770da6c3b27bef08abffd4238b13a3c269208010009a2b ./code/src/tactile_editor/checkpoint_loaders.py
198
+ d441fd06da718276de55811a262db2d5beb684a70ec088d9a2bded5cd0581b7e ./code/src/tactile_editor/config.py
199
+ b15af5c93cd5db8a4888d32e15476854815fb8e1cc729601456c1928d048cfce ./code/src/tactile_editor/contact_filter.py
200
+ 9ec5b7e82350fb1167b0ea7894c50e7969af6e072b2a59494bc0943eaae8e4c0 ./code/src/tactile_editor/direct_residual.py
201
+ 4725aee12e9f98d942b494325c01dffa40a37b59195f8e929f36949f812ddfb4 ./code/src/tactile_editor/editor_backends.py
202
+ 8d0503690a247468fee690fe7bb16b3b6b6272ba709d8cf93f739832fbe89a46 ./code/src/tactile_editor/editor_interface.py
203
+ 5b889e99f587e66b4cae09fd13cefe517ba357883cac1e8386d4a0296203a793 ./code/src/tactile_editor/encoder.py
204
+ 82d51681d4d3631ce876ee9ee68d12263ec0960f1de8636f02f5013f71c98dcc ./code/src/tactile_editor/frozen_stage1.py
205
+ a5810e83a7b5c70f0630110b0dece8703750902b10973e850a79c2572b13f457 ./code/src/tactile_editor/full_cache.py
206
+ 385f2df034fc45f91464507dbf88827518a240d8c2ce5ce1aecdde58ec8d84ec ./code/src/tactile_editor/fusion.py
207
+ 988ea09d24e771dd3f6db4b08a78467421cd41ccec18a72916a2186c28c3e517 ./code/src/tactile_editor/human_baselines.py
208
+ b09c0afff1f5fc99c4f104dd8ee998922c639f199ffdb0e4b18cf88bc591f1c5 ./code/src/tactile_editor/informative_data.py
209
+ 054f426876e418a9bc243e70512a27bdbfb281aa092f91f4dcd422ef0936d88c ./code/src/tactile_editor/mano_action.py
210
+ b0fedec1b8f6d104ea8cd2551e19d4bc499f3162c256a73589eb4f06a4e433a2 ./code/src/tactile_editor/memoryvla_baseline.py
211
+ dd6f77f9967f66928a02da1ebdf6ee4c36d163164eeda3e95f32431189611e64 ./code/src/tactile_editor/mock.py
212
+ 0c8dbb0d1da02d95d4e69b45abe091fbe7556ac8651640e26932ec42628e3ead ./code/src/tactile_editor/model.py
213
+ 9e60ec589dc4fc63fe7082a0d9422bda942963e9b033810ec3580f94607ba7bc ./code/src/tactile_editor/model_adapters.py
214
+ 6627dd50c1fdcf18b1df11c7387875dc62a91df6412573bb5926d0c2e9d29b30 ./code/src/tactile_editor/native_tachin.py
215
+ 14ddc4222f42ef3c03d333e490bfae4bf8049dedd256d08d5fa3172f096b6b05 ./code/src/tactile_editor/offline_data.py
216
+ fb5a907bbe4e24912244e4eec58f3e369201222eb8dff6774a970940416ba777 ./code/src/tactile_editor/paired30_cache.py
217
+ c2c8cd891d297509bbb63fc921d5be1b68af0f3ccaab65d03369ef96c3e03835 ./code/src/tactile_editor/real_data.py
218
+ b2def8c80c9515f6686ff689f5ce5c926d74ca8309d05161cd6b1c315072518d ./code/src/tactile_editor/rgb_world_data.py
219
+ 8bd6478f81aaf058b73a2a7cd8ccd646e22c841d53a7479a8d48328be4888040 ./code/src/tactile_editor/rgb_world_editor.py
220
+ e0d979702559b5361d53a1f3df0cb2a3ed999b9496aaea9531280eb17def02e4 ./code/src/tactile_editor/runtime.py
221
+ 585edb9909006d293b01f0a0b22499e53b9db9d145b9d754402c5c615192035d ./code/src/tactile_editor/single_stage_world.py
222
+ 6e97c4a65f68e701c9dafdd73964843e6b17ed7ec1265e206347c48d97e2da9b ./code/src/tactile_editor/smoke.py
223
+ d262a8c2075d200eb51cbd246bc6f3cec7a1d52b4eb19c1566b93b173b77bce9 ./code/src/tactile_editor/table1_editor_cache.py
224
+ bcf4d1e566a29e6d17fd608d51502bc832b91ee9c23d0a554980723c35829846 ./code/src/tactile_editor/table1_experiment.py
225
+ d74683999995a121e8a31d4da277d554e83caa8cfdb5f39d5a157eed18832590 ./code/src/tactile_editor/table1_protocol.py
226
+ e0bd6df948b0004ad15756f58079aee128d75058e964971b121061019a26adec ./code/src/tactile_editor/touch_study.py
227
+ 8d74e8aed5a765d6d5f7b0ab4b77ed02beaa8e460f83bc7429eb1cdc92ba3ebc ./code/src/tactile_editor/trainability.py
228
+ cb7913357ee11ee8c84a739aea3c15fef3bae809c2d57c680a018559b205d95e ./code/src/tactile_editor/training.py
229
+ 40e5eb86b8ea6dea91d2cbc0755005ea857751946ed0c28518d960540244dac5 ./code/src/tactile_editor/unified_cache.py
230
+ c10fb673b0af2c6ac3673cb8220facfd9a7fb20b68419fe82270f569866cbe1f ./code/src/tactile_editor/unified_world.py
231
+ a5608efcaee31e8f6a74bea32dbfa088fc2f3dce209a4b1c5d77e3e61a5256cf ./code/src/tactile_editor/visual_dynamics_pilot.py
232
+ e0d895c3c8661b7cde1be24f7a343a9d0da5bd62921d96928c8eb2dcdb317bf3 ./code/src/tactile_editor/world_flow.py
233
+ a9e5d2b77c83b53863f5cde861b0d7641032f5efec5dd31c916daf6af3c98aea ./code/src/tactile_editor/world_regularized_data.py
234
+ 058c61c0e36c95a684138394b7275d71b9db66aa3faaac9a5f444014c9a0eb26 ./code/src/tactile_editor/world_residual_probe.py
235
+ 30541b63e49b9a68f20dfb65f775d5076a5ebcd1b7cdb30957d43d5b2aace6f6 ./code/tests/test_tachin_native_editor.py
236
+ da3a1e31ad14e9844532228121545af4e1bb3dcae443dae7c4a20f372c893884 ./editors/smolvla/best.pt
237
+ 6c6bf15aee4036efcc2009ade5d66e35803e912add518a552e6808c0c9343cfe ./editors/smolvla/config.json
238
+ d7fcd2c60a7cce29603d5afb065c2e127ffa91eb90a7848aed9149d1efb8f913 ./editors/smolvla/epochs.jsonl
239
+ 16cd7b468c98d1bb58c813b2688f60852e888b6bd88fd88d5aefb0d220a22964 ./editors/smolvla/summary.json
240
+ 933932400d7925a5a54d2f71ed6ab8efd191463c4a8d52750e93d0bddf327bc5 ./editors/smolvla/test_editor_action.npy
241
+ 27fd2986caba17c53b0061aaf0cf4b89653e9a88f266bf1973855f2092927fa9 ./editors/smolvla/test_shuffled_touch_action.npy
242
+ daf66f08fa5a2b477c7a54d37df30d2dc14e2938f047ab7eb81bacc8c37d3da1 ./editors/smolvla/test_zero_touch_action.npy
243
+ c39c64ea31d1080c3ff8423bd6621639420dd39a5ff3726e60d9ddcf981caf73 ./editors/trex/best.pt
244
+ 9b0e34d73a062c8a2636cec3fa046cfdd788b2bb81b0d85f1642c04b9fc7ea8b ./editors/trex/config.json
245
+ 9463a52418c7f3544a7cb92456ba6e6b8f66a1cdb205c46d600e9e3b42772acb ./editors/trex/epochs.jsonl
246
+ d3e07ee3a1d429b1d6d8a1f8115521c67e851b0e16aa6c3cb4ed13891e41b6a0 ./editors/trex/summary.json
247
+ 6be42371c333061b410c9c3678a4b0e4802ec473de37557daff79b38acd30868 ./editors/trex/test_editor_action.npy
248
+ fede955cf5dbecec4dbe92dcec1901b052f24f14e409c669c10f07d2b99bde84 ./editors/trex/test_shuffled_touch_action.npy
249
+ 80f2ee3d3f08260e783ef560cb3f9211d4f650dd9d0963bc7f18bd257331e38e ./editors/trex/test_zero_touch_action.npy
250
+ 3955de4381cfe33223081eea7589dab1a32af99e65e4ba4a392a82102fc5c1b0 ./editors/twla/best.pt
251
+ 1300487a3dd85b48efeddef92e4887ad1683c22e619197579ebec3922ad7154d ./editors/twla/config.json
252
+ 52d2bad73d91de86898440079a134c91a9907cbb40d9e1c67899c19885fe615d ./editors/twla/epochs.jsonl
253
+ 0190786e58f244c0e2d021b1870ec7b6a8992af1e503875da8ad2842ca4fe613 ./editors/twla/summary.json
254
+ 409497ac11a9fecb1792c30ccdc8e876ebc9ef73ac776e2526de3f608edcc3cd ./editors/twla/test_editor_action.npy
255
+ 5c582cb663f2f6483f9f5ea99a50c815c5c5d05ef5315efe35a768a19bae9ce7 ./editors/twla/test_shuffled_touch_action.npy
256
+ 0fb186f3b70c85d3c1d0cdfd93b4ac7940a0d94207498f3b54ac2e6e1a07ef04 ./editors/twla/test_zero_touch_action.npy
257
+ 5f9d4bd6c0ea2170e22bed59f4448d2f975e1e6a66b8c4a91616a6f02954bd47 ./editors/vitra/best.pt
258
+ 98132855d066b96c4661ca84392aba547eb6e4b233396971ae74a56f5bc14280 ./editors/vitra/config.json
259
+ 532ea7340f79863c20306a0cbc374fd9115af545bee0d616e4506c7991c1b5c5 ./editors/vitra/epochs.jsonl
260
+ 8689e18fa3bff2bb29903e43e55a17b7fac274f129de4e6efb37784982e074bd ./editors/vitra/summary.json
261
+ 9e2724e241b3698b801eff61e6f682de08f22659f394a79b2adeedc5edefaecc ./editors/vitra/test_editor_action.npy
262
+ 7cce8d5d3e57304d9728f70d97982160d79df36dd486cb01beffdde958967160 ./editors/vitra/test_shuffled_touch_action.npy
263
+ e8b1afc84e34207f431227985de120463fc801019d8c178f218b93dbec451b29 ./editors/vitra/test_zero_touch_action.npy
264
+ b7a5464413a57d315ad3064bc6ca2c9d8bc24e791962065282fca16de5ef35c7 ./environment.txt
model/tachin_paired30_cache_20260923/audit/cache_audit.json ADDED
@@ -0,0 +1,57 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "cache": "/scratch/hannahyao24/VLA-tactile_paper/artifacts/tachin_table1_paired30_20260923",
3
+ "splits": {
4
+ "train": {
5
+ "plans": 18208,
6
+ "paired_rows": 49744,
7
+ "rows_by_decision_tick": [
8
+ 16576,
9
+ 16596,
10
+ 16572
11
+ ],
12
+ "models_share_exact_rows": true,
13
+ "causal_windows_exact": true,
14
+ "decision_relative_w3_targets_exact": true,
15
+ "first_editable_action": [
16
+ 0,
17
+ 3,
18
+ 6
19
+ ]
20
+ },
21
+ "val": {
22
+ "plans": 1006,
23
+ "paired_rows": 2655,
24
+ "rows_by_decision_tick": [
25
+ 883,
26
+ 886,
27
+ 886
28
+ ],
29
+ "models_share_exact_rows": true,
30
+ "causal_windows_exact": true,
31
+ "decision_relative_w3_targets_exact": true,
32
+ "first_editable_action": [
33
+ 0,
34
+ 3,
35
+ 6
36
+ ]
37
+ },
38
+ "test": {
39
+ "plans": 1169,
40
+ "paired_rows": 2433,
41
+ "rows_by_decision_tick": [
42
+ 809,
43
+ 812,
44
+ 812
45
+ ],
46
+ "models_share_exact_rows": true,
47
+ "causal_windows_exact": true,
48
+ "decision_relative_w3_targets_exact": true,
49
+ "first_editable_action": [
50
+ 0,
51
+ 3,
52
+ 6
53
+ ]
54
+ }
55
+ },
56
+ "status": "PASS"
57
+ }
model/tachin_paired30_cache_20260923/audit/comparison.json ADDED
@@ -0,0 +1,155 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "protocol": "Tachin paired-30",
3
+ "models": {
4
+ "smolvla": {
5
+ "training_budget_epochs": 10,
6
+ "selected_epoch": 10,
7
+ "validation_action_loss": 0.2469381999610283,
8
+ "base_metrics": {
9
+ "translation_cm": 1.3014105013364496,
10
+ "rotation_deg": 4.997658153014678,
11
+ "joint_mpjpe_cm": 1.6248546858030588
12
+ },
13
+ "editor_metrics": {
14
+ "translation_cm": 1.2412666298214436,
15
+ "rotation_deg": 4.665520451312771,
16
+ "joint_mpjpe_cm": 1.3737346600468263
17
+ },
18
+ "shuffled_touch_metrics": {
19
+ "translation_cm": 1.2407394562784146,
20
+ "rotation_deg": 4.663409782922765,
21
+ "joint_mpjpe_cm": 1.3714706658511227
22
+ },
23
+ "zero_touch_metrics": {
24
+ "translation_cm": 1.3014105013364496,
25
+ "rotation_deg": 4.99765815311085,
26
+ "joint_mpjpe_cm": 1.6248546858030588
27
+ },
28
+ "table1_cells": {
29
+ "translation_cm": "0.06 / (4.62%)",
30
+ "rotation_deg": "0.33 / (6.65%)",
31
+ "joint_mpjpe_cm": "0.25 / (15.45%)"
32
+ },
33
+ "zero_touch_max_action_difference": 3.725290298461914e-09,
34
+ "editor_vs_shuffled_action_mae": 0.0003912480897270143
35
+ },
36
+ "vitra": {
37
+ "training_budget_epochs": 10,
38
+ "selected_epoch": 9,
39
+ "validation_action_loss": 0.19477003950295924,
40
+ "base_metrics": {
41
+ "translation_cm": 1.7086415295051836,
42
+ "rotation_deg": 7.852093026854487,
43
+ "joint_mpjpe_cm": 0.6802416275515677
44
+ },
45
+ "editor_metrics": {
46
+ "translation_cm": 1.3520344510315767,
47
+ "rotation_deg": 5.52281293424393,
48
+ "joint_mpjpe_cm": 0.6283003161906104
49
+ },
50
+ "shuffled_touch_metrics": {
51
+ "translation_cm": 1.3519262729830466,
52
+ "rotation_deg": 5.5235926742795565,
53
+ "joint_mpjpe_cm": 0.6271929750119718
54
+ },
55
+ "zero_touch_metrics": {
56
+ "translation_cm": 1.7086415295354782,
57
+ "rotation_deg": 7.852093026936965,
58
+ "joint_mpjpe_cm": 0.6802416272885641
59
+ },
60
+ "table1_cells": {
61
+ "translation_cm": "0.36 / (20.87%)",
62
+ "rotation_deg": "2.33 / (29.66%)",
63
+ "joint_mpjpe_cm": "0.05 / (7.64%)"
64
+ },
65
+ "zero_touch_max_action_difference": 7.450580596923828e-09,
66
+ "editor_vs_shuffled_action_mae": 7.935501344036311e-05
67
+ },
68
+ "trex": {
69
+ "training_budget_epochs": 10,
70
+ "selected_epoch": 9,
71
+ "validation_action_loss": 0.2388624380761845,
72
+ "base_metrics": {
73
+ "translation_cm": 1.5101441290157958,
74
+ "rotation_deg": 5.626478859715439,
75
+ "joint_mpjpe_cm": 1.2290510475647314
76
+ },
77
+ "editor_metrics": {
78
+ "translation_cm": 1.2979139497825751,
79
+ "rotation_deg": 4.793363297812164,
80
+ "joint_mpjpe_cm": 1.1977501825700754
81
+ },
82
+ "shuffled_touch_metrics": {
83
+ "translation_cm": 1.2985424613853025,
84
+ "rotation_deg": 4.801285892773723,
85
+ "joint_mpjpe_cm": 1.197848930770959
86
+ },
87
+ "zero_touch_metrics": {
88
+ "translation_cm": 1.5101441290157958,
89
+ "rotation_deg": 5.626478859702203,
90
+ "joint_mpjpe_cm": 1.2290510475647314
91
+ },
92
+ "table1_cells": {
93
+ "translation_cm": "0.21 / (14.05%)",
94
+ "rotation_deg": "0.83 / (14.81%)",
95
+ "joint_mpjpe_cm": "0.03 / (2.55%)"
96
+ },
97
+ "zero_touch_max_action_difference": 3.725290298461914e-09,
98
+ "editor_vs_shuffled_action_mae": 0.0001029103368637152
99
+ },
100
+ "twla": {
101
+ "training_budget_epochs": 30,
102
+ "selected_epoch": 8,
103
+ "validation_action_loss": 0.1752967903169535,
104
+ "base_metrics": {
105
+ "translation_cm": 1.3520767220769714,
106
+ "rotation_deg": 5.292899332876988,
107
+ "joint_mpjpe_cm": 0.5639765192191974
108
+ },
109
+ "editor_metrics": {
110
+ "translation_cm": 1.2665439394654596,
111
+ "rotation_deg": 4.929279282635632,
112
+ "joint_mpjpe_cm": 0.47511430038172686
113
+ },
114
+ "shuffled_touch_metrics": {
115
+ "translation_cm": 1.2630061238186077,
116
+ "rotation_deg": 4.932078891277113,
117
+ "joint_mpjpe_cm": 0.47602607038290146
118
+ },
119
+ "zero_touch_metrics": {
120
+ "translation_cm": 1.352076721918785,
121
+ "rotation_deg": 5.292899332792065,
122
+ "joint_mpjpe_cm": 0.5639765192191974
123
+ },
124
+ "table1_cells": {
125
+ "translation_cm": "0.09 / (6.33%)",
126
+ "rotation_deg": "0.36 / (6.87%)",
127
+ "joint_mpjpe_cm": "0.09 / (15.76%)"
128
+ },
129
+ "zero_touch_max_action_difference": 3.725290298461914e-09,
130
+ "editor_vs_shuffled_action_mae": 0.0001719161809887737
131
+ }
132
+ },
133
+ "twla_vs_best_competing_editor": {
134
+ "translation_cm": {
135
+ "best_competing_model": "smolvla",
136
+ "best_competing_error": 1.2412666298214436,
137
+ "twla_error": 1.2665439394654596,
138
+ "twla_reduction_percent_vs_best_competitor": -2.0364125673508293
139
+ },
140
+ "rotation_deg": {
141
+ "best_competing_model": "smolvla",
142
+ "best_competing_error": 4.665520451312771,
143
+ "twla_error": 4.929279282635632,
144
+ "twla_reduction_percent_vs_best_competitor": -5.653363522362134
145
+ },
146
+ "joint_mpjpe_cm": {
147
+ "best_competing_model": "vitra",
148
+ "best_competing_error": 0.6283003161906104,
149
+ "twla_error": 0.47511430038172686,
150
+ "twla_reduction_percent_vs_best_competitor": 24.381018417697376
151
+ }
152
+ },
153
+ "cache_audit": "/scratch/hannahyao24/VLA-tactile_paper/runs/tachin_paired30_20260923/cache_audit.json",
154
+ "status": "PASS"
155
+ }
model/tachin_paired30_cache_20260923/cache/complete.json ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "schema": "tachin.table1.editor_cache.paired30.v1",
3
+ "counts": {
4
+ "train": 18208,
5
+ "val": 1006,
6
+ "test": 1169
7
+ },
8
+ "source_cache": "/scratch/hannahyao24/VLA-tactile_paper/artifacts/tachin_table1_twla_adapted_20260922",
9
+ "benchmark": "/scratch/hannahyao24/VLA-tactile_paper/artifacts/datasets/tachin_benchmark_v2_release",
10
+ "adaptation_data": "/scratch/hannahyao24/VLA-tactile_paper/artifacts/datasets/tachin_adaptation_20260922",
11
+ "vision_plan_hz": 10.0,
12
+ "tactile_input_hz": 30.0,
13
+ "editor_decision_hz": 30.0,
14
+ "decision_ticks": [
15
+ 0,
16
+ 1,
17
+ 2
18
+ ],
19
+ "action_hz": 80.0,
20
+ "gated": false
21
+ }
model/tachin_paired30_cache_20260923/code/scripts/audit_tachin_paired30.py ADDED
@@ -0,0 +1,78 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Fail-closed audit of the native Tachin paired-30 cache and row contract."""
2
+ from __future__ import annotations
3
+
4
+ import argparse
5
+ import json
6
+ import sys
7
+ from pathlib import Path
8
+
9
+ import numpy as np
10
+ import torch
11
+
12
+ ROOT = Path(__file__).resolve().parents[1]
13
+ sys.path[:0] = [str(ROOT / "src"), str(ROOT / "scripts")]
14
+ from train_tachin_table1_editor import TachinEditorCache, edit_mask
15
+
16
+
17
+ def main() -> None:
18
+ parser = argparse.ArgumentParser()
19
+ parser.add_argument("--cache", type=Path, required=True)
20
+ parser.add_argument("--output", type=Path, required=True)
21
+ args = parser.parse_args()
22
+ report = {"cache": str(args.cache.resolve()), "splits": {}}
23
+ for split in ("train", "val", "test"):
24
+ datasets = {
25
+ model: TachinEditorCache(args.cache, split, model,
26
+ train_or_val=split != "test", include_w3=model == "twla")
27
+ for model in ("smolvla", "vitra", "trex", "twla")
28
+ }
29
+ reference = datasets["smolvla"]
30
+ for model, dataset in datasets.items():
31
+ if not dataset.paired30 or not np.array_equal(dataset.rows, reference.rows):
32
+ raise AssertionError(f"{split}/{model} does not use the common paired rows")
33
+ counts = np.bincount(reference.rows[:, 1], minlength=3)
34
+ old_pressure = reference.common["pressure"]
35
+ old_future = reference.common["future_pressure"]
36
+ old_mask = reference.common["pressure_mask"]
37
+ old_future_mask = reference.common["future_pressure_mask"]
38
+ extra = reference.common["future_extra_pressure"]
39
+ extra_mask = reference.common["future_extra_pressure_mask"]
40
+ # Audit all rows. Exact equality is expected because paired windows are
41
+ # views of the original immutable values, not a second resampling pass.
42
+ for item, (query, tick) in enumerate(reference.rows):
43
+ sample = reference[item]
44
+ sequence = np.concatenate((old_pressure[query], old_future[query, :2]), axis=0)
45
+ sequence_mask = np.concatenate((old_mask[query], old_future_mask[query, :2]), axis=0)
46
+ targets = np.concatenate((old_future[query], extra[query]), axis=0)
47
+ target_masks = np.concatenate((old_future_mask[query], extra_mask[query]), axis=0)
48
+ if not np.array_equal(sample["pressure"].numpy(), sequence[tick:tick + 8]):
49
+ raise AssertionError("Tactile input window mismatch")
50
+ if not np.array_equal(sample["pressure_mask"].numpy(), sequence_mask[tick:tick + 8]):
51
+ raise AssertionError("Tactile input mask mismatch")
52
+ if not np.array_equal(sample["future_pressure"].numpy(), targets[tick + 2]):
53
+ raise AssertionError("Decision-relative W3 target mismatch")
54
+ if not np.array_equal(sample["future_pressure_mask"].numpy(), target_masks[tick + 2]):
55
+ raise AssertionError("Decision-relative W3 target mask mismatch")
56
+ batch = {name: value.unsqueeze(0) for name, value in sample.items()
57
+ if torch.is_tensor(value)}
58
+ mask = edit_mask(batch)[0]
59
+ editable = torch.where(mask.any(1))[0]
60
+ if len(editable) == 0 or int(editable[0]) != (0, 3, 6)[tick]:
61
+ raise AssertionError("Committed action prefix is editable")
62
+ report["splits"][split] = {
63
+ "plans": len(reference.common["target_action"]),
64
+ "paired_rows": len(reference),
65
+ "rows_by_decision_tick": counts.tolist(),
66
+ "models_share_exact_rows": True,
67
+ "causal_windows_exact": True,
68
+ "decision_relative_w3_targets_exact": True,
69
+ "first_editable_action": [0, 3, 6],
70
+ }
71
+ report["status"] = "PASS"
72
+ args.output.parent.mkdir(parents=True, exist_ok=True)
73
+ args.output.write_text(json.dumps(report, indent=2) + "\n")
74
+ print(json.dumps(report, indent=2))
75
+
76
+
77
+ if __name__ == "__main__":
78
+ main()
model/tachin_paired30_cache_20260923/code/scripts/build_tachin_paired30_cache.py ADDED
@@ -0,0 +1,174 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Upgrade the native Tachin Table-1 cache to the paired-30 protocol.
2
+
3
+ The original cache already contains the eight frames available at plan time
4
+ and the next three 30-Hz tactile samples. Paired-30 needs the first two of
5
+ those samples as inputs at decision ticks 1 and 2, and needs two additional
6
+ samples (+4 and +5) as W3's decision-relative +100-ms targets. Base-policy,
7
+ memory, RGB-world, action-target, and split artifacts are reused by symlink.
8
+ """
9
+ from __future__ import annotations
10
+
11
+ import argparse
12
+ import hashlib
13
+ import json
14
+ import os
15
+ from pathlib import Path
16
+
17
+ import numpy as np
18
+
19
+
20
+ def load_npz(path: Path) -> dict[str, np.ndarray]:
21
+ with np.load(path, allow_pickle=False) as payload:
22
+ return {key: payload[key] for key in payload.files}
23
+
24
+
25
+ def atomic_json(path: Path, value) -> None:
26
+ temporary = path.with_suffix(path.suffix + ".tmp")
27
+ temporary.write_text(json.dumps(value, indent=2) + "\n")
28
+ os.replace(temporary, path)
29
+
30
+
31
+ def sha256(path: Path) -> str:
32
+ digest = hashlib.sha256()
33
+ with path.open("rb") as stream:
34
+ for block in iter(lambda: stream.read(8 << 20), b""):
35
+ digest.update(block)
36
+ return digest.hexdigest()
37
+
38
+
39
+ def link(source: Path, destination: Path) -> None:
40
+ if destination.is_symlink():
41
+ if destination.resolve() != source.resolve():
42
+ raise ValueError(f"Wrong existing symlink: {destination}")
43
+ return
44
+ if destination.exists():
45
+ raise FileExistsError(destination)
46
+ destination.parent.mkdir(parents=True, exist_ok=True)
47
+ destination.symlink_to(source.resolve(), target_is_directory=source.is_dir())
48
+
49
+
50
+ def main() -> None:
51
+ parser = argparse.ArgumentParser()
52
+ parser.add_argument("--source-cache", type=Path, required=True)
53
+ parser.add_argument("--adaptation-data", type=Path, required=True)
54
+ parser.add_argument("--benchmark", type=Path, required=True)
55
+ parser.add_argument("--output", type=Path, required=True)
56
+ args = parser.parse_args()
57
+
58
+ root = Path(__file__).resolve().parents[1]
59
+ code = root / "artifacts/huggingface/vla-hand-baselines/model/tachin_adaptation_20260922/code"
60
+ import sys
61
+ sys.path.insert(0, str(code))
62
+ from data_preparation.tachin_benchmark.core import tactile_history
63
+
64
+ schema = json.loads((args.benchmark / "schema.json").read_text())
65
+ metadata = {
66
+ row["episode_id"]: row
67
+ for line in (args.benchmark / "episodes.jsonl").read_text().splitlines()
68
+ if (row := json.loads(line))
69
+ }
70
+
71
+ args.output.mkdir(parents=True, exist_ok=True)
72
+ link(args.source_cache / "statistics.npz", args.output / "statistics.npz")
73
+ for model in ("smolvla", "vitra", "trex", "twla"):
74
+ link(args.source_cache / "bases" / model, args.output / "bases" / model)
75
+
76
+ counts: dict[str, int] = {}
77
+ for split in ("train", "val", "test"):
78
+ source = args.source_cache / "shared" / split
79
+ destination = args.output / "shared" / split
80
+ destination.mkdir(parents=True, exist_ok=True)
81
+ rows_path = args.adaptation_data / split / "rows.json"
82
+ rows = json.loads(rows_path.read_text())
83
+ count = len(rows)
84
+ counts[split] = count
85
+
86
+ # Preserve the exact original sample identity and every immutable field.
87
+ for name in (
88
+ "target_action.npy", "action_mask.npy", "pressure.npy",
89
+ "pressure_mask.npy", "sample_valid.npy", "future_pressure.npy",
90
+ "future_pressure_mask.npy", "future_valid.npy", "beta.npy",
91
+ "identity.npz",
92
+ ):
93
+ link(source / name, destination / name)
94
+
95
+ extra_path = destination / "future_extra_pressure.npy"
96
+ extra_mask_path = destination / "future_extra_pressure_mask.npy"
97
+ extra_valid_path = destination / "future_extra_valid.npy"
98
+ if (destination / "complete.json").is_file():
99
+ print(json.dumps({"split": split, "status": "already complete"}), flush=True)
100
+ continue
101
+ for path in (extra_path, extra_mask_path, extra_valid_path):
102
+ if path.exists():
103
+ raise FileExistsError(f"Incomplete paired-30 output exists: {path}")
104
+ pressure = np.lib.format.open_memmap(
105
+ extra_path, mode="w+", shape=(count, 2, 2, 880), dtype="float32")
106
+ pressure_mask = np.lib.format.open_memmap(
107
+ extra_mask_path, mode="w+", shape=(count, 2, 2, 880), dtype="bool")
108
+ valid = np.lib.format.open_memmap(
109
+ extra_valid_path, mode="w+", shape=(count, 2, 2), dtype="bool")
110
+
111
+ cached_recording = None
112
+ streams = None
113
+ for index, row in enumerate(rows):
114
+ meta = metadata[row["episode_id"]]
115
+ recording = meta["recording_id"]
116
+ if recording != cached_recording:
117
+ streams = [
118
+ load_npz(args.benchmark / "streams" / recording / f"{hand}.npz")
119
+ for hand in ("left", "right")
120
+ ]
121
+ cached_recording = recording
122
+ timestamp = float(row["timestamp"])
123
+ for extra_index, frame_offset in enumerate((4, 5)):
124
+ future = tactile_history(
125
+ streams,
126
+ timestamp + frame_offset / 30.0,
127
+ frames=1,
128
+ history_hz=30.0,
129
+ max_age_s=float(schema["max_tactile_age_s"]),
130
+ )
131
+ pressure[index, extra_index] = future["pressure"][0]
132
+ pressure_mask[index, extra_index] = future["pressure_mask"][0]
133
+ valid[index, extra_index] = future["sample_mask"][0]
134
+ if index % 1000 == 0:
135
+ print(json.dumps({"split": split, "done": index, "total": count}), flush=True)
136
+ pressure.flush(); pressure_mask.flush(); valid.flush()
137
+
138
+ identity = dict(np.load(source / "identity.npz", allow_pickle=False))
139
+ if not np.array_equal(identity["sample_id"].astype(str),
140
+ np.asarray([row["sample_id"] for row in rows]).astype(str)):
141
+ raise ValueError(f"Sample identity mismatch in {split}")
142
+ atomic_json(destination / "complete.json", {
143
+ "schema": "tachin.table1.paired30.shared.v1",
144
+ "split": split,
145
+ "plans": count,
146
+ "rows_sha256": sha256(rows_path),
147
+ "decision_ticks": [0, 1, 2],
148
+ "decision_hz": 30.0,
149
+ "action_hz": 80.0,
150
+ "first_editable_action": [0, 3, 6],
151
+ "touch_input": "8-frame causal windows ending at plan + decision_tick/30 s",
152
+ "w3_tactile_target": "decision-relative +100 ms (+3 frames)",
153
+ "base_plan_reused_across_decisions": True,
154
+ "gated": False,
155
+ })
156
+ print(json.dumps({"split": split, "status": "complete", "plans": count}), flush=True)
157
+
158
+ atomic_json(args.output / "complete.json", {
159
+ "schema": "tachin.table1.editor_cache.paired30.v1",
160
+ "counts": counts,
161
+ "source_cache": str(args.source_cache.resolve()),
162
+ "benchmark": str(args.benchmark.resolve()),
163
+ "adaptation_data": str(args.adaptation_data.resolve()),
164
+ "vision_plan_hz": 10.0,
165
+ "tactile_input_hz": 30.0,
166
+ "editor_decision_hz": 30.0,
167
+ "decision_ticks": [0, 1, 2],
168
+ "action_hz": 80.0,
169
+ "gated": False,
170
+ })
171
+
172
+
173
+ if __name__ == "__main__":
174
+ main()
model/tachin_paired30_cache_20260923/code/scripts/cache_tachin_adapted_base.py ADDED
@@ -0,0 +1,251 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Cache frozen Tachin-adapted SmolVLA, VITRA, or HumanTReX predictions."""
2
+ from __future__ import annotations
3
+
4
+ import argparse
5
+ import hashlib
6
+ import importlib.util
7
+ import json
8
+ import os
9
+ import sys
10
+ from pathlib import Path
11
+
12
+ import numpy as np
13
+ import torch
14
+ from torch.utils.data import DataLoader, Subset
15
+
16
+ ROOT = Path(__file__).resolve().parents[1]
17
+
18
+
19
+ def install_py310_file_digest() -> None:
20
+ if hasattr(hashlib, "file_digest"):
21
+ return
22
+ def file_digest(stream, algorithm):
23
+ result = hashlib.new(algorithm)
24
+ for block in iter(lambda: stream.read(8 << 20), b""):
25
+ result.update(block)
26
+ return result
27
+ hashlib.file_digest = file_digest
28
+
29
+
30
+ def sha256(path: Path) -> str:
31
+ with path.open("rb") as stream:
32
+ return hashlib.file_digest(stream, "sha256").hexdigest()
33
+
34
+
35
+ def atomic_json(path: Path, payload) -> None:
36
+ temporary = path.with_suffix(path.suffix + ".tmp")
37
+ temporary.write_text(json.dumps(payload, indent=2, default=str) + "\n")
38
+ os.replace(temporary, path)
39
+
40
+
41
+ def per_row_noise(indices, device, seed):
42
+ return torch.stack([
43
+ torch.randn((16, 192), device=device,
44
+ generator=torch.Generator(device=device).manual_seed(seed + int(index)))
45
+ for index in indices
46
+ ])
47
+
48
+
49
+ def load_adapted_local(adaptation, checkpoint_path, model_name, data_root,
50
+ release, model_assets, device):
51
+ """Restore against project-local frozen initialization/source assets."""
52
+ # transformers 5.17 moved this context manager out of modeling_utils, while
53
+ # the frozen Tachin release imports it from the former public location.
54
+ # Keep the archived release immutable and provide the compatibility alias
55
+ # in this runner instead.
56
+ if model_name != "trex":
57
+ import transformers.modeling_utils as modeling_utils
58
+ if not hasattr(modeling_utils, "no_init_weights"):
59
+ from transformers.initialization import no_init_weights
60
+ modeling_utils.no_init_weights = no_init_weights
61
+ checkpoint = torch.load(checkpoint_path, map_location="cpu", weights_only=False)
62
+ initialization = {
63
+ "smolvla": model_assets / "smolvla_official/best.pt",
64
+ "vitra": model_assets / "vitra_original/best.pt",
65
+ "trex": release / "initialization/trex_g3/best.pt",
66
+ }[model_name]
67
+ if sha256(initialization) != checkpoint["initialization"]["initial_sha256"]:
68
+ raise ValueError(f"Project-local {model_name} initialization digest mismatch")
69
+ configuration = checkpoint["config"]
70
+ if model_name != "trex":
71
+ adaptation.ASSETS = model_assets
72
+ model, package, config, tokenizer, audit = adaptation.build(
73
+ model_name, initialization, checkpoint["statistics"], device,
74
+ not configuration["unfreeze_vlm"], data_root / "statistics.json")
75
+ else:
76
+ initial = torch.load(initialization, map_location="cpu", mmap=True, weights_only=False)
77
+ adaptation.check_initialization(initial)
78
+ spec = importlib.util.spec_from_file_location(
79
+ "tachin_human_trex_local", release / "code/training/trex_lora/model.py")
80
+ module = importlib.util.module_from_spec(spec)
81
+ spec.loader.exec_module(module)
82
+ config_values = initial["config"]
83
+ source = model_assets / "trex/source"
84
+ midtrain = model_assets / "trex/midtrain"
85
+ model = module.HumanTReX(str(source), str(midtrain), config_values["rank"],
86
+ config_values["alpha"], False)
87
+ model.load_adapter(initial["adapter"])
88
+ model.set_stats(checkpoint["statistics"])
89
+ from transformers import AutoProcessor
90
+ tokenizer = AutoProcessor.from_pretrained(str(midtrain / "processor"),
91
+ local_files_only=True)
92
+ tokenizer.tokenizer.padding_side = "left"
93
+ tokenizer.image_processor.size = {"shortest_edge": 4096, "longest_edge": 262144}
94
+ model.to(device)
95
+ package = config = None
96
+ audit = {"model": "trex", "initial_checkpoint": str(initialization.resolve()),
97
+ "source": str(source.resolve()), "midtrain": str(midtrain.resolve()),
98
+ "tactile_input": False, "editor": False, "world_model": False}
99
+ expected = {name for name, parameter in model.named_parameters() if parameter.requires_grad}
100
+ expected.update(name for name in model.state_dict()
101
+ if name in ("action_mean", "action_std", "state_mean", "state_std", "stats_loaded"))
102
+ if expected != set(checkpoint["trainable"]):
103
+ raise ValueError("Adapted trainable parameter set changed")
104
+ model.load_state_dict(checkpoint["trainable"], strict=False)
105
+ model.eval()
106
+ return model, package, config, tokenizer, audit
107
+
108
+
109
+ @torch.inference_mode()
110
+ def predict(model, package, batch, indices, device, seed):
111
+ noise = per_row_noise(indices, device, seed)
112
+ if package is None:
113
+ mask = batch["observed_mask"][:, None, :].expand(-1, 16, -1)
114
+ x = noise * mask
115
+ encoded = model.encode(batch)
116
+ for step in range(10):
117
+ time = torch.full((len(x),), 1 - step / 10, device=device)
118
+ x = (x - model.velocity(batch, x, time, encoded) / 10) * mask
119
+ return (x * model.action_std + model.action_mean) * mask
120
+ import train as adaptation
121
+ observations, _, _, _ = adaptation.arguments(batch, package)
122
+ normalized = model.predict_from_noise(observations, noise.to(batch["actions"].dtype))
123
+ return model.unnormalise_action(normalized)
124
+
125
+
126
+ def main() -> None:
127
+ parser = argparse.ArgumentParser()
128
+ parser.add_argument("--model", choices=("smolvla", "vitra", "trex"), required=True)
129
+ parser.add_argument("--release", type=Path, required=True)
130
+ parser.add_argument("--data", type=Path, required=True)
131
+ parser.add_argument("--common", type=Path, required=True)
132
+ parser.add_argument("--output", type=Path, required=True)
133
+ parser.add_argument("--model-assets", type=Path,
134
+ default=ROOT / "artifacts/model_assets")
135
+ parser.add_argument("--mano", type=Path)
136
+ parser.add_argument("--batch-size", type=int, default=8)
137
+ parser.add_argument("--workers", type=int, default=2)
138
+ parser.add_argument("--seed", type=int, default=20260922)
139
+ parser.add_argument("--limit", type=int, default=0)
140
+ parser.add_argument("--shard", type=int, default=-1)
141
+ parser.add_argument("--shards", type=int, default=1)
142
+ args = parser.parse_args()
143
+ if args.model == "vitra" and args.mano is None:
144
+ parser.error("--mano is required for VITRA native MANO FK")
145
+ if args.shard >= 0 and (args.limit or not 0 <= args.shard < args.shards):
146
+ parser.error("sharded cache requires --limit 0 and 0 <= shard < shards")
147
+
148
+ install_py310_file_digest()
149
+ code = args.release / "code"
150
+ trainer = code / "training/tachin_adaptation"
151
+ sys.path[:0] = [str(trainer), str(code), str(Path(__file__).resolve().parents[1] / "src")]
152
+ import train as adaptation
153
+ from tactile_editor.mano_action import ManoActionConverter
154
+
155
+ torch.set_num_threads(4)
156
+ torch.set_float32_matmul_precision("high")
157
+ device = torch.device("cuda:0")
158
+ torch.cuda.set_device(device)
159
+ checkpoint = args.release / args.model / "best.pt"
160
+ model, package, config, tokenizer, audit = load_adapted_local(
161
+ adaptation, checkpoint, args.model, args.data, args.release,
162
+ args.model_assets, device)
163
+ collator = adaptation.Collator(args.model, tokenizer, config)
164
+ converter = ManoActionConverter(args.mano, device=device) if args.model == "vitra" else None
165
+
166
+ for split in ("train", "val", "test"):
167
+ layout = "angle" if args.model == "vitra" else "keypoints"
168
+ dataset = adaptation.TachinDataset(args.data, split, layout)
169
+ common_ids = dict(np.load(args.common / "shared" / split / "identity.npz", allow_pickle=False))
170
+ source_ids = np.asarray([row["sample_id"] for row in dataset.rows])
171
+ if not np.array_equal(source_ids.astype(str), common_ids["sample_id"].astype(str)):
172
+ raise ValueError(f"{args.model}/{split} does not match common row identity")
173
+ count = len(dataset) if not args.limit else min(len(dataset), args.limit)
174
+ output = args.output / "bases" / args.model / split
175
+ output.mkdir(parents=True, exist_ok=True)
176
+ complete_path = output / ("smoke_complete.json" if args.limit else "complete.json")
177
+ if complete_path.exists():
178
+ print(json.dumps({"model": args.model, "split": split, "status": "already complete"}), flush=True)
179
+ continue
180
+ shard_complete = (output / f"complete_shard_{args.shard:02d}_of_{args.shards:02d}.json"
181
+ if args.shard >= 0 else None)
182
+ if shard_complete is not None and shard_complete.exists():
183
+ print(json.dumps({"model": args.model, "split": split, "shard": args.shard,
184
+ "status": "already complete"}), flush=True)
185
+ continue
186
+ prediction_path = output / ("smoke_base_action.npy" if args.limit else "base_action.npy")
187
+ marker_path = output / ("smoke_row_complete.npy" if args.limit else "row_complete.npy")
188
+ if prediction_path.exists():
189
+ predictions = np.load(prediction_path, mmap_mode="r+")
190
+ markers = np.load(marker_path, mmap_mode="r+")
191
+ if predictions.shape != (count, 16, 192) or markers.shape != (count,):
192
+ raise ValueError(f"Bad resumable cache schema: {output}")
193
+ else:
194
+ predictions = np.lib.format.open_memmap(prediction_path, mode="w+", dtype="float32", shape=(count, 16, 192))
195
+ markers = np.lib.format.open_memmap(marker_path, mode="w+", dtype="bool", shape=(count,))
196
+ markers[:] = False
197
+ markers.flush()
198
+ assigned = (np.arange(args.shard, count, args.shards, dtype=np.int64)
199
+ if args.shard >= 0 else np.arange(count, dtype=np.int64))
200
+ remaining = assigned[~markers[assigned]]
201
+ loader = DataLoader(Subset(dataset, remaining.tolist()), batch_size=args.batch_size,
202
+ shuffle=False, num_workers=args.workers, collate_fn=collator,
203
+ pin_memory=True, persistent_workers=args.workers > 0)
204
+ cursor = 0
205
+ for batch_index, batch in enumerate(loader):
206
+ size = len(batch["actions"])
207
+ positions = remaining[cursor:cursor + size]
208
+ batch = adaptation.to_device(batch, device)
209
+ with adaptation.precision(package):
210
+ values = predict(model, package, batch, positions, device, args.seed)
211
+ if converter is not None:
212
+ betas = np.asarray([dataset.rows[int(index)]["beta"] for index in positions], np.float32)
213
+ conventions = np.tile(np.asarray([["mano_left", "mano_right"]]), (size, 1))
214
+ values = converter(values.float(), betas=betas, conventions=conventions)
215
+ values = values.float().cpu().numpy()
216
+ if values.shape != (size, 16, 192) or not np.isfinite(values).all():
217
+ raise ValueError(f"Invalid {args.model} prediction batch")
218
+ predictions[positions] = values
219
+ predictions.flush()
220
+ markers[positions] = True
221
+ markers.flush()
222
+ cursor += size
223
+ if batch_index % 25 == 0:
224
+ print(json.dumps({"model": args.model, "split": split,
225
+ "done": int(markers.sum()), "total": count}), flush=True)
226
+ if not markers[assigned].all():
227
+ raise RuntimeError(f"Incomplete assigned cache rows: {output}")
228
+ payload = {
229
+ "schema": "tachin.table1.frozen_base.v1",
230
+ "model": args.model,
231
+ "split": split,
232
+ "rows": count,
233
+ "checkpoint": str(checkpoint.resolve()),
234
+ "checkpoint_sha256": sha256(checkpoint),
235
+ "adaptation_step": 1000,
236
+ "noise": f"per-row torch seed {args.seed} + zero-based common row index",
237
+ "output": "physical canonical [16,192] keypoint action",
238
+ "vitra_fk": "native MANO_LEFT/MANO_RIGHT" if converter is not None else None,
239
+ "audit": audit,
240
+ }
241
+ if shard_complete is not None:
242
+ atomic_json(shard_complete, {**payload, "assigned_rows": len(assigned),
243
+ "shard": args.shard, "shards": args.shards})
244
+ if markers.all():
245
+ atomic_json(complete_path, payload)
246
+ print(json.dumps({"model": args.model, "split": split, "shard": args.shard,
247
+ "status": "complete", "global_complete": bool(markers.all())}), flush=True)
248
+
249
+
250
+ if __name__ == "__main__":
251
+ main()
model/tachin_paired30_cache_20260923/code/scripts/cache_tachin_twla_w3.py ADDED
@@ -0,0 +1,299 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Cache T-WLA-110k actions, recurrent memory, and W3 RGB-world inputs."""
2
+ from __future__ import annotations
3
+
4
+ import argparse
5
+ import copy
6
+ import hashlib
7
+ import json
8
+ import os
9
+ import sys
10
+ import time
11
+ from pathlib import Path
12
+
13
+ import numpy as np
14
+ import torch
15
+
16
+
17
+ ROOT = Path(__file__).resolve().parents[1]
18
+ V4 = ROOT / "ref/yuzhench_distill_editor/V4"
19
+ sys.path[:0] = [str(ROOT / "src"), str(ROOT / "ref/Tactile"), str(V4), str(V4 / "vla_hand_pose"), str(ROOT / "scripts")]
20
+
21
+
22
+ def sha256(path: Path) -> str:
23
+ h = hashlib.sha256()
24
+ with path.open("rb") as stream:
25
+ for block in iter(lambda: stream.read(8 << 20), b""):
26
+ h.update(block)
27
+ return h.hexdigest()
28
+
29
+
30
+ def array(root: Path, name: str, shape, dtype):
31
+ root.mkdir(parents=True, exist_ok=True)
32
+ path = root / f"{name}.npy"
33
+ if path.exists():
34
+ value = np.load(path, mmap_mode="r+")
35
+ if value.shape != tuple(shape) or value.dtype != np.dtype(dtype):
36
+ raise ValueError(f"Existing cache schema mismatch: {path}")
37
+ return value
38
+ return np.lib.format.open_memmap(path, mode="w+", shape=shape, dtype=dtype)
39
+
40
+
41
+ def atomic_json(path: Path, payload) -> None:
42
+ temporary = path.with_suffix(path.suffix + ".tmp")
43
+ temporary.write_text(json.dumps(payload, indent=2, default=str) + "\n")
44
+ os.replace(temporary, path)
45
+
46
+
47
+ def checkpoint_provenance(path: Path) -> dict:
48
+ """Read cache-facing provenance without assuming the 110k initialization."""
49
+ payload = torch.load(path, map_location="cpu", mmap=True, weights_only=False)
50
+ step = payload.get("optimizer_step", payload.get("step"))
51
+ if step is None:
52
+ raise ValueError(f"Checkpoint does not declare a training/adaptation step: {path}")
53
+ adaptation = payload.get("adaptation")
54
+ result = {
55
+ "checkpoint_step": int(step),
56
+ "checkpoint_is_tachin_adapted": isinstance(adaptation, dict),
57
+ }
58
+ if isinstance(adaptation, dict):
59
+ result.update(
60
+ adaptation_protocol=adaptation.get("protocol"),
61
+ adaptation_initial_checkpoint=adaptation.get("checkpoint"),
62
+ adaptation_initial_sha256=adaptation.get("initial_sha256"),
63
+ adaptation_common_complete_sha256=adaptation.get("common_complete_sha256"),
64
+ )
65
+ del payload
66
+ return result
67
+
68
+
69
+ class PlannedDataset:
70
+ def __init__(self, base, common_indices, anchors):
71
+ self.base = base
72
+ self.common_indices = np.asarray(common_indices)
73
+ self.anchors = np.asarray(anchors)
74
+
75
+ def __len__(self):
76
+ return len(self.common_indices)
77
+
78
+ def __getitem__(self, index):
79
+ from tactile_editor.rgb_world_data import project_future_hand_roi
80
+ item = self.base[index]
81
+ unit = self.base.units[self.base._eval_plan[index][0]]
82
+ annotation, _, _ = self.base.cores[unit.source]._load_or_cache_episode(unit.episode)
83
+ anchor = int(self.anchors[index])
84
+ roi = np.zeros((2, 16, 16), bool)
85
+ roi_valid = np.zeros(2, bool)
86
+ hand = annotation.get(str(annotation["anno_type"]))
87
+ if isinstance(hand, dict):
88
+ for future_index, offset in enumerate((3, 6)):
89
+ if anchor + offset < len(annotation["video_decode_frame"]):
90
+ projected = project_future_hand_roi(
91
+ np.asarray(hand["joints_camspace"])[anchor + offset],
92
+ annotation["intrinsics"], source_hw=(1080, 1920),
93
+ )
94
+ roi[future_index] = projected.roi
95
+ roi_valid[future_index] = projected.valid
96
+ item.update(query_index=int(self.common_indices[index]), future_roi=roi,
97
+ future_roi_valid=roi_valid)
98
+ return item
99
+
100
+
101
+ def main() -> None:
102
+ parser = argparse.ArgumentParser()
103
+ parser.add_argument("--checkpoint", type=Path, required=True)
104
+ parser.add_argument("--checkpoint-config", type=Path, required=True)
105
+ parser.add_argument("--world-checkpoint", type=Path, required=True)
106
+ parser.add_argument("--world-config", type=Path, required=True)
107
+ parser.add_argument("--annotations", type=Path, required=True)
108
+ parser.add_argument("--jpeg-root", type=Path, required=True)
109
+ parser.add_argument("--adaptation-data", type=Path, required=True)
110
+ parser.add_argument("--common", type=Path, required=True)
111
+ parser.add_argument("--output", type=Path, required=True)
112
+ parser.add_argument("--shard", type=int, required=True)
113
+ parser.add_argument("--shards", type=int, default=4)
114
+ parser.add_argument("--batch-size", type=int, default=24)
115
+ parser.add_argument("--workers", type=int, default=4)
116
+ parser.add_argument("--seed", type=int, default=20260922)
117
+ parser.add_argument("--limit", type=int, default=0)
118
+ args = parser.parse_args()
119
+ if not 0 <= args.shard < args.shards:
120
+ raise ValueError("shard must be in [0, shards)")
121
+
122
+ checkpoint_metadata = checkpoint_provenance(args.checkpoint)
123
+
124
+ os.environ.update(METHOD_D_CHECKPOINT=str(args.checkpoint.resolve()),
125
+ METHOD_D_RUN_CONFIG=str(args.checkpoint_config.resolve()),
126
+ METHOD_D_SOURCE=str(V4 / "vla_hand_pose"))
127
+ from transformers import AutoTokenizer
128
+ from modal_build_full_editor_cache import load_backbone, apply_saved_config
129
+ from vla_hand_pose.v4_d_fm_0 import V4DConfig, MemoryBank, Observation, flow
130
+ from vla_hand_pose.dataloader import SequenceDataset, Source, make_collate, to_observations
131
+ from vla_hand_pose.world_model import dc_ae
132
+ from tactile_editor import EditorConfig, FusionConfig, TactileMethodD, Variant
133
+
134
+ torch.set_num_threads(3)
135
+ torch.set_float32_matmul_precision("high")
136
+ torch.manual_seed(args.seed)
137
+ device = torch.device("cuda:0")
138
+ backbone, config = load_backbone(device)
139
+
140
+ world_payload = torch.load(args.world_checkpoint, map_location="cpu", mmap=True, weights_only=False)
141
+ world_metadata = json.loads(args.world_config.read_text())
142
+ saved = world_payload.get("effective_config", world_metadata.get("effective_config", world_metadata.get("config")))
143
+ world_config = V4DConfig()
144
+ apply_saved_config(world_config, saved)
145
+ world_config.validate()
146
+ del world_payload
147
+ dataset_config = copy.deepcopy(config)
148
+ dataset_config.world = world_config.world
149
+ autoencoder = dc_ae.build(world_config.world.dcae, dtype=torch.float32).to(device)
150
+ fusion = FusionConfig(hidden_dim=config.expert.hidden_size, num_heads=config.expert.num_heads)
151
+ sampler = TactileMethodD(backbone, EditorConfig(variant=Variant.V0_BASE, fusion=fusion)).eval()
152
+ tokenizer = AutoTokenizer.from_pretrained(config.vlm.model_id)
153
+ source = Source(name="tachin", video_root="/unused",
154
+ annotation_file=str(args.annotations / "episode_frame_index.npz"),
155
+ label_folder=str(args.annotations / "episodic_annotations"),
156
+ jpeg_root=str(args.jpeg_root))
157
+
158
+ for split in ("train", "val", "test"):
159
+ rows = json.loads((args.adaptation_data / split / "rows.json").read_text())
160
+ identity = dict(np.load(args.common / "shared" / split / "identity.npz", allow_pickle=False))
161
+ if not np.array_equal(np.asarray([row["sample_id"] for row in rows]).astype(str),
162
+ identity["sample_id"].astype(str)):
163
+ raise ValueError(f"T-WLA {split} row identity mismatch")
164
+ all_indices = np.arange(args.shard, len(rows), args.shards, dtype=np.int64)
165
+ if args.limit:
166
+ all_indices = all_indices[:args.limit]
167
+ output = args.output / "bases/twla" / split / f"shard_{args.shard:02d}"
168
+ completion = output / ("smoke_complete.json" if args.limit else "complete.json")
169
+ if completion.exists():
170
+ print(json.dumps({"split": split, "shard": args.shard, "status": "already complete"}), flush=True)
171
+ continue
172
+
173
+ episodes = {str(rows[index]["episode_id"]) for index in all_indices}
174
+ dataset = SequenceDataset([source], cfg=dataset_config,
175
+ video_whitelist={"tachin": episodes}, tokenizer=tokenizer,
176
+ seed=0, load_images=True, training=False, deterministic=True)
177
+ units = {(unit.episode, unit.phase): index for index, unit in enumerate(dataset.units)}
178
+ plans = []
179
+ anchors = []
180
+ for common_index in all_indices:
181
+ row = rows[int(common_index)]
182
+ anchor = int(row["anchor_source_row"])
183
+ phase, relative = anchor % 3, anchor // 3
184
+ length = min(relative + 1, 9)
185
+ plans.append((units[(str(row["episode_id"]), phase)], relative - length + 1,
186
+ length, int(common_index)))
187
+ anchors.append(anchor)
188
+ dataset.set_eval_plan(plans)
189
+ wrapped = PlannedDataset(dataset, all_indices, anchors)
190
+ base_collate = make_collate(dataset_config, tokenizer)
191
+ def collate(items):
192
+ batch = base_collate(items)
193
+ for name in ("query_index", "future_roi", "future_roi_valid"):
194
+ batch[name] = torch.from_numpy(np.asarray([item[name] for item in items]))
195
+ return batch
196
+
197
+ count = len(all_indices)
198
+ fields = {
199
+ "base_action": ((count, 16, 192), "float32"),
200
+ "final_memory": ((count, 8, 960), "float16"),
201
+ "image_tokens": ((count, 64, 960), "float16"),
202
+ "clean_latent": ((count, 2, 32, 16, 16), "float16"),
203
+ "rgb_valid": ((count, 2), "bool"),
204
+ "future_roi": ((count, 2, 16, 16), "bool"),
205
+ "query_index": ((count,), "int64"),
206
+ }
207
+ values = {name: array(output, ("smoke_" if args.limit else "") + name, shape, dtype)
208
+ for name, (shape, dtype) in fields.items()}
209
+ marker = array(output, ("smoke_" if args.limit else "") + "row_complete", (count,), "bool")
210
+ remaining = np.flatnonzero(~marker)
211
+ loader = torch.utils.data.DataLoader(torch.utils.data.Subset(wrapped, remaining.tolist()),
212
+ batch_size=args.batch_size, shuffle=False, num_workers=args.workers,
213
+ collate_fn=collate, pin_memory=True, persistent_workers=args.workers > 0)
214
+ started = time.time()
215
+ cursor = 0
216
+ max_target_difference = 0.0
217
+ mask_mismatches = 0
218
+ target = np.load(args.common / "shared" / split / "target_action.npy", mmap_mode="r")
219
+ target_mask = np.load(args.common / "shared" / split / "action_mask.npy", mmap_mode="r")
220
+ with torch.inference_mode():
221
+ for batch_index, batch in enumerate(loader):
222
+ steps, step_valid, actions, action_mask = to_observations(batch, device, Observation, dtype=torch.bfloat16)
223
+ size = len(actions)
224
+ bank = MemoryBank(size, config.memory.bank_capacity, config.memory.num_slots,
225
+ config.memory.dim, device=device, dtype=backbone.vlm.dtype)
226
+ for time_index, observation in enumerate(steps[:-1]):
227
+ active = torch.nonzero(step_valid[:, time_index], as_tuple=True)[0]
228
+ if not active.numel():
229
+ continue
230
+ subset = observation if active.numel() == size else observation.index_select(active)
231
+ bank.append_rows(active, backbone.encode(subset, bank, rows=active))
232
+ final_memory, image_tokens = backbone.encode(steps[-1], bank, return_image_tokens=True)
233
+ generation_mask = flow.component_action_mask(size, 16, 192, config.flow, device)
234
+ query_indices = batch["query_index"].numpy()
235
+ noise = torch.stack([
236
+ torch.randn((16, 192), device=device, dtype=final_memory.dtype,
237
+ generator=torch.Generator(device=device).manual_seed(args.seed + int(index)))
238
+ for index in query_indices
239
+ ])
240
+ normalized = sampler.sample_from_memory(
241
+ final_memory, generation_mask,
242
+ state_with_mask=(backbone.state_with_mask(steps[-1].state, steps[-1].state_mask)
243
+ if config.action.direct_state_to_expert else None),
244
+ noise=noise, num_steps=config.flow.num_inference_steps,
245
+ return_normalized=True,
246
+ ).float()
247
+ physical = normalized * backbone.action_std.float() + backbone.action_mean.float()
248
+ future_images = batch["future_images"].to(device).permute(0, 1, 4, 2, 3).float().div_(127.5).sub_(1)
249
+ clean_latent = dc_ae.encode_targets(autoencoder, future_images, world_config.world.dcae)
250
+ positions = remaining[cursor:cursor + size]
251
+ difference = np.abs(actions.float().cpu().numpy() - target[query_indices])
252
+ common_mask = target_mask[query_indices]
253
+ max_target_difference = max(max_target_difference,
254
+ float(difference[common_mask].max(initial=0)))
255
+ mask_mismatches += int(np.count_nonzero(action_mask.cpu().numpy() != common_mask))
256
+ payload = {
257
+ "base_action": physical,
258
+ "final_memory": final_memory,
259
+ "image_tokens": image_tokens,
260
+ "clean_latent": clean_latent,
261
+ "rgb_valid": batch["future_valid"].bool() & batch["future_roi_valid"].bool(),
262
+ "future_roi": batch["future_roi"],
263
+ "query_index": batch["query_index"],
264
+ }
265
+ for name, value in payload.items():
266
+ if value.dtype == torch.bfloat16:
267
+ value = value.float()
268
+ result = value.detach().cpu().numpy()
269
+ if not np.isfinite(result).all():
270
+ raise ValueError(f"Nonfinite T-WLA cache field {name}")
271
+ values[name][positions] = result
272
+ values[name].flush()
273
+ marker[positions] = True
274
+ marker.flush()
275
+ cursor += size
276
+ if batch_index % 25 == 0:
277
+ print(json.dumps({"split": split, "shard": args.shard,
278
+ "done": int(marker.sum()), "total": count,
279
+ "rows_per_second": cursor / max(time.time() - started, 1e-6)}), flush=True)
280
+ if not marker.all():
281
+ raise RuntimeError(f"Incomplete T-WLA shard: {output}")
282
+ atomic_json(completion, {
283
+ "schema": "tachin.table1.twla_w3_cache.v1",
284
+ "split": split, "shard": args.shard, "shards": args.shards,
285
+ "rows": count, "checkpoint": str(args.checkpoint.resolve()),
286
+ "checkpoint_sha256": sha256(args.checkpoint), **checkpoint_metadata,
287
+ "memory_protocol": "up to eight available 10-Hz historical observations plus current",
288
+ "noise": f"per-row torch seed {args.seed} + zero-based common row index",
289
+ "world_checkpoint": str(args.world_checkpoint.resolve()),
290
+ "world_checkpoint_sha256": sha256(args.world_checkpoint),
291
+ "max_common_target_difference": max_target_difference,
292
+ "common_mask_mismatches": mask_mismatches,
293
+ "seconds": time.time() - started,
294
+ })
295
+ print(json.dumps({"split": split, "shard": args.shard, "status": "complete"}), flush=True)
296
+
297
+
298
+ if __name__ == "__main__":
299
+ main()
model/tachin_paired30_cache_20260923/code/scripts/summarize_tachin_paired30.py ADDED
@@ -0,0 +1,77 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Validate and summarize the four completed Tachin paired-30 Table-1 runs."""
2
+ from __future__ import annotations
3
+
4
+ import argparse
5
+ import json
6
+ from pathlib import Path
7
+
8
+ import numpy as np
9
+
10
+
11
+ def main() -> None:
12
+ parser = argparse.ArgumentParser()
13
+ parser.add_argument("--run-root", type=Path, required=True)
14
+ parser.add_argument("--output", type=Path, required=True)
15
+ args = parser.parse_args()
16
+ metrics = ("translation_cm", "rotation_deg", "joint_mpjpe_cm")
17
+ payload = {"protocol": "Tachin paired-30", "models": {}}
18
+ for model in ("smolvla", "vitra", "trex", "twla"):
19
+ root = args.run_root / model
20
+ report = json.loads((root / "summary.json").read_text())
21
+ config = report["config"]
22
+ if config["editor_protocol"] != "paired-30":
23
+ raise AssertionError(f"{model}: wrong editor protocol")
24
+ if config["decision_ticks"] != [0, 1, 2] or config["editor_decision_hz"] != 30.0:
25
+ raise AssertionError(f"{model}: wrong decision cadence")
26
+ base = np.load(root / "test_base_action.npy") if (root / "test_base_action.npy").exists() else None
27
+ # Base is not separately written by the trainer; recover it from the
28
+ # zero-touch invariant check against the source policy below.
29
+ zero = np.load(root / "test_zero_touch_action.npy")
30
+ editor = np.load(root / "test_editor_action.npy")
31
+ shuffled = np.load(root / "test_shuffled_touch_action.npy")
32
+ source = Path(config["cache"]) / "bases" / model / "test"
33
+ if model == "twla":
34
+ base = np.empty_like(zero)
35
+ for shard in sorted(source.glob("shard_*")):
36
+ ids = np.load(shard / "query_index.npy")
37
+ base[ids] = np.load(shard / "base_action.npy")
38
+ else:
39
+ base = np.load(source / "base_action.npy")
40
+ zero_max = float(np.max(np.abs(zero - base)))
41
+ if zero_max > 2e-6:
42
+ raise AssertionError(f"{model}: reference input changes the base plan ({zero_max})")
43
+ test = report["test"]
44
+ payload["models"][model] = {
45
+ "training_budget_epochs": config["epochs"],
46
+ "selected_epoch": report["selected_epoch"],
47
+ "validation_action_loss": report["validation"]["action_loss"],
48
+ "base_metrics": test["metrics"]["base"],
49
+ "editor_metrics": test["metrics"]["editor"],
50
+ "shuffled_touch_metrics": test["metrics"]["shuffled_touch"],
51
+ "zero_touch_metrics": test["metrics"]["zero_touch"],
52
+ "table1_cells": test["table1_cells"],
53
+ "zero_touch_max_action_difference": zero_max,
54
+ "editor_vs_shuffled_action_mae": float(np.mean(np.abs(editor - shuffled))),
55
+ }
56
+ competitors = ("smolvla", "vitra", "trex")
57
+ comparison = {}
58
+ for metric in metrics:
59
+ best_model = min(competitors,
60
+ key=lambda name: payload["models"][name]["editor_metrics"][metric])
61
+ best = payload["models"][best_model]["editor_metrics"][metric]
62
+ twla = payload["models"]["twla"]["editor_metrics"][metric]
63
+ comparison[metric] = {
64
+ "best_competing_model": best_model,
65
+ "best_competing_error": best,
66
+ "twla_error": twla,
67
+ "twla_reduction_percent_vs_best_competitor": 100.0 * (best - twla) / best,
68
+ }
69
+ payload["twla_vs_best_competing_editor"] = comparison
70
+ payload["cache_audit"] = str((args.run_root.parent / "cache_audit.json").resolve())
71
+ payload["status"] = "PASS"
72
+ args.output.write_text(json.dumps(payload, indent=2) + "\n")
73
+ print(json.dumps(payload, indent=2))
74
+
75
+
76
+ if __name__ == "__main__":
77
+ main()
model/tachin_paired30_cache_20260923/code/scripts/train_tachin_table1_editor.py ADDED
@@ -0,0 +1,539 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Train and evaluate one native Tachin tactile editor for paper Table 1."""
2
+ from __future__ import annotations
3
+
4
+ import argparse
5
+ import fcntl
6
+ import json
7
+ import math
8
+ import os
9
+ import sys
10
+ import time
11
+ from pathlib import Path
12
+
13
+ import numpy as np
14
+ import torch
15
+ from torch.utils.data import DataLoader, Dataset
16
+
17
+ ROOT = Path(__file__).resolve().parents[1]
18
+ V4 = ROOT / "ref/yuzhench_distill_editor/V4"
19
+ sys.path[:0] = [str(ROOT / "src"), str(V4), str(V4 / "vla_hand_pose"), str(ROOT / "scripts")]
20
+
21
+ from tactile_editor.action_layout import FULL_ACTION_GROUPS
22
+ from tactile_editor.action_objectives import per_sample_action_loss
23
+ from tactile_editor.native_tachin import (NativeResidualEditor, NativeTachinEncoder,
24
+ NativeTachinW3Editor)
25
+ from tactile_editor.rgb_world_editor import roi_weighted_world_loss
26
+ from tactile_editor.single_stage_world import masked_tactile_flow_loss
27
+ from tactile_editor.table1_experiment import summarize_physical
28
+ from tactile_editor.training import build_future_action_mask
29
+
30
+
31
+ def atomic_json(path: Path, payload) -> None:
32
+ temporary = path.with_suffix(path.suffix + ".tmp")
33
+ temporary.write_text(json.dumps(payload, indent=2, default=str) + "\n")
34
+ os.replace(temporary, path)
35
+
36
+
37
+ class TachinEditorCache(Dataset):
38
+ def __init__(self, root: Path, split: str, model: str, *, train_or_val: bool,
39
+ include_w3: bool):
40
+ self.root, self.split, self.model = root, split, model
41
+ shared = root / "shared" / split
42
+ if not (shared / "complete.json").is_file():
43
+ raise RuntimeError(f"Incomplete common cache: {shared}")
44
+ names = ("target_action", "action_mask", "pressure", "pressure_mask",
45
+ "sample_valid", "future_pressure", "future_pressure_mask", "future_valid")
46
+ self.common = {name: np.load(shared / f"{name}.npy", mmap_mode="r") for name in names}
47
+ self.paired30 = (shared / "future_extra_pressure.npy").is_file()
48
+ if self.paired30:
49
+ for name in ("future_extra_pressure", "future_extra_pressure_mask", "future_extra_valid"):
50
+ self.common[name] = np.load(shared / f"{name}.npy", mmap_mode="r")
51
+ self.identity = dict(np.load(shared / "identity.npz", allow_pickle=False))
52
+ count = len(self.common["target_action"])
53
+ self.locations = None
54
+ if model == "twla":
55
+ self.shards = []
56
+ locations = np.full((count, 2), -1, np.int64)
57
+ for shard_index, path in enumerate(sorted((root / "bases/twla" / split).glob("shard_*"))):
58
+ if not (path / "complete.json").is_file():
59
+ continue
60
+ fields = ["base_action", "final_memory", "query_index"]
61
+ if include_w3:
62
+ fields += ["image_tokens", "clean_latent", "rgb_valid", "future_roi"]
63
+ payload = {name: np.load(path / f"{name}.npy", mmap_mode="r") for name in fields}
64
+ ids = payload["query_index"]
65
+ if np.any(locations[ids, 0] >= 0):
66
+ raise ValueError("Duplicate T-WLA query indices")
67
+ locations[ids, 0] = len(self.shards)
68
+ locations[ids, 1] = np.arange(len(ids))
69
+ self.shards.append(payload)
70
+ if not self.shards or np.any(locations < 0):
71
+ raise RuntimeError(f"Incomplete T-WLA cache for {split}")
72
+ self.locations = locations
73
+ else:
74
+ path = root / "bases" / model / split
75
+ if not (path / "complete.json").is_file():
76
+ raise RuntimeError(f"Incomplete {model} cache for {split}")
77
+ self.base_action = np.load(path / "base_action.npy", mmap_mode="r")
78
+ if self.base_action.shape != (count, 16, 192):
79
+ raise ValueError("Frozen-base cache shape mismatch")
80
+
81
+ latest_hand = self.common["sample_valid"][:, -1] & self.common["pressure_mask"][:, -1].any(-1)
82
+ if self.paired30:
83
+ future_latest = (self.common["future_valid"][:, :2]
84
+ & self.common["future_pressure_mask"][:, :2].any(-1))
85
+ latest_hand = np.concatenate((latest_hand[:, None], future_latest), axis=1)
86
+ else:
87
+ latest_hand = latest_hand[:, None]
88
+ action_hand = np.stack((self.common["action_mask"][:, :, :69].any((1, 2)),
89
+ self.common["action_mask"][:, :, 69:138].any((1, 2))), 1)
90
+ self.latest_hand = np.asarray(latest_hand)
91
+ eligible = (latest_hand & action_hand[:, None]).any(2)
92
+ if self.paired30:
93
+ for tick, committed in enumerate((0, 3, 6)):
94
+ eligible[:, tick] &= self.common["action_mask"][:, committed:, :138].any((1, 2))
95
+ self.rows = np.argwhere(eligible)
96
+ else:
97
+ indices = np.flatnonzero(eligible[:, 0]) if train_or_val else np.arange(count)
98
+ self.rows = np.stack((indices, np.zeros_like(indices)), axis=1)
99
+
100
+ def __len__(self):
101
+ return len(self.rows)
102
+
103
+ def __getitem__(self, item):
104
+ index, tick = map(int, self.rows[item])
105
+ if self.model == "twla":
106
+ shard, row = self.locations[index]
107
+ base = self.shards[int(shard)]
108
+ output = {name: torch.from_numpy(np.array(value[int(row)], copy=True))
109
+ for name, value in base.items() if name != "query_index"}
110
+ else:
111
+ output = {"base_action": torch.from_numpy(np.array(self.base_action[index], copy=True))}
112
+ for name in ("target_action", "action_mask"):
113
+ output[name] = torch.from_numpy(np.array(self.common[name][index], copy=True))
114
+ if self.paired30:
115
+ pressure = np.concatenate((self.common["pressure"][index],
116
+ self.common["future_pressure"][index, :2]), axis=0)
117
+ pressure_mask = np.concatenate((self.common["pressure_mask"][index],
118
+ self.common["future_pressure_mask"][index, :2]), axis=0)
119
+ sample_valid = np.concatenate((self.common["sample_valid"][index],
120
+ self.common["future_valid"][index, :2]), axis=0)
121
+ future_pressure = np.concatenate((self.common["future_pressure"][index],
122
+ self.common["future_extra_pressure"][index]), axis=0)
123
+ future_pressure_mask = np.concatenate((self.common["future_pressure_mask"][index],
124
+ self.common["future_extra_pressure_mask"][index]), axis=0)
125
+ future_valid = np.concatenate((self.common["future_valid"][index],
126
+ self.common["future_extra_valid"][index]), axis=0)
127
+ output["pressure"] = torch.from_numpy(np.array(pressure[tick:tick + 8], copy=True))
128
+ output["pressure_mask"] = torch.from_numpy(np.array(pressure_mask[tick:tick + 8], copy=True))
129
+ output["sample_valid"] = torch.from_numpy(np.array(sample_valid[tick:tick + 8], copy=True))
130
+ output["future_pressure"] = torch.from_numpy(np.array(future_pressure[tick + 2], copy=True))
131
+ output["future_pressure_mask"] = torch.from_numpy(
132
+ np.array(future_pressure_mask[tick + 2], copy=True))
133
+ output["future_valid"] = torch.from_numpy(np.array(future_valid[tick + 2], copy=True))
134
+ else:
135
+ for name in ("pressure", "pressure_mask", "sample_valid"):
136
+ output[name] = torch.from_numpy(np.array(self.common[name][index], copy=True))
137
+ output["future_pressure"] = torch.from_numpy(
138
+ np.array(self.common["future_pressure"][index, 2], copy=True))
139
+ output["future_pressure_mask"] = torch.from_numpy(
140
+ np.array(self.common["future_pressure_mask"][index, 2], copy=True))
141
+ output["future_valid"] = torch.from_numpy(
142
+ np.array(self.common["future_valid"][index, 2], copy=True))
143
+ output["query_index"] = torch.tensor(index)
144
+ output["decision_tick"] = torch.tensor(tick)
145
+ output["decision_seconds"] = torch.tensor(tick / 30.0, dtype=torch.float32)
146
+ return output
147
+
148
+
149
+ def edit_mask(batch):
150
+ mask = build_future_action_mask(batch["action_mask"].bool(), batch["decision_seconds"])
151
+ mask[..., 138:] = False
152
+ hand_valid = batch["sample_valid"][:, -1].bool() & batch["pressure_mask"][:, -1].any(-1)
153
+ mask[..., :69] &= hand_valid[:, 0, None, None]
154
+ mask[..., 69:138] &= hand_valid[:, 1, None, None]
155
+ return mask
156
+
157
+
158
+ def move(batch, device):
159
+ return {name: value.to(device, non_blocking=True) for name, value in batch.items()}
160
+
161
+
162
+ def prepare_batch(raw, device, statistics):
163
+ batch = move(raw, device)
164
+ mean = statistics["action_mean"].to(device)
165
+ scale = statistics["action_std"].to(device)
166
+ batch["base_action"] = (batch["base_action"].float() - mean) / scale
167
+ batch["target_action"] = (batch["target_action"].float() - mean) / scale
168
+ return batch
169
+
170
+
171
+ def make_model(model_name, statistics, device):
172
+ encoder = NativeTachinEncoder(statistics["pressure_mean"], statistics["pressure_scale"])
173
+ if model_name == "twla":
174
+ model = NativeTachinW3Editor(encoder, statistics["pressure_mean"],
175
+ statistics["pressure_scale"])
176
+ else:
177
+ model = NativeResidualEditor(encoder)
178
+ return model.to(device)
179
+
180
+
181
+ def forward(model, batch, *, auxiliary=False, rgb_head=None, generator=None):
182
+ mask = edit_mask(batch)
183
+ if isinstance(model, NativeResidualEditor):
184
+ result = model(batch["base_action"], mask, batch["pressure"].float(),
185
+ batch["pressure_mask"], batch["sample_valid"])
186
+ prediction, delta, allowed, tactile, hand_valid = result
187
+ return prediction, delta, allowed, tactile, hand_valid, None
188
+ kwargs = {}
189
+ if auxiliary:
190
+ future_pressure = batch["future_pressure"].float()
191
+ future_valid = batch["future_pressure_mask"].bool() & batch["future_valid"][:, :, None].bool()
192
+ latent = batch["clean_latent"].float()
193
+ kwargs = dict(
194
+ future_pressure=future_pressure,
195
+ future_cell_valid=future_valid,
196
+ tactile_tau=torch.rand(len(mask), device=mask.device, generator=generator),
197
+ tactile_noise=torch.randn((len(mask), 2, 880), device=mask.device, generator=generator),
198
+ rgb_world_head=rgb_head,
199
+ image_tokens=batch["image_tokens"].float(),
200
+ clean_latent=latent,
201
+ rgb_valid=batch["rgb_valid"].bool(),
202
+ rgb_tau=torch.rand(latent.shape[:2], device=mask.device, generator=generator),
203
+ rgb_noise=torch.randn(latent.shape, device=mask.device, generator=generator),
204
+ )
205
+ output = model(base_action=batch["base_action"], edit_mask=mask,
206
+ pressure=batch["pressure"].float(), cell_valid=batch["pressure_mask"],
207
+ sample_valid=batch["sample_valid"], final_memory=batch["final_memory"].float(),
208
+ compute_auxiliary=auxiliary, **kwargs)
209
+ return (output.final_action, output.delta_action, output.allowed_mask,
210
+ output.tactile_features, output.hand_valid, output)
211
+
212
+
213
+ def objective(model, batch, *, rgb_head, generator, lambda_rgb, lambda_tactile):
214
+ prediction, delta, allowed, _, _, output = forward(
215
+ model, batch, auxiliary=isinstance(model, NativeTachinW3Editor),
216
+ rgb_head=rgb_head, generator=generator)
217
+ active = allowed.any((1, 2))
218
+ action = per_sample_action_loss(prediction[active], batch["target_action"][active],
219
+ allowed[active], groups=FULL_ACTION_GROUPS).mean()
220
+ delta_l2 = delta.float().square().masked_fill(~allowed, 0).sum() / allowed.sum().clamp_min(1)
221
+ tactile = action.new_zeros(())
222
+ rgb = action.new_zeros(())
223
+ if output is not None:
224
+ tactile = masked_tactile_flow_loss(output.tactile_velocity, output.tactile_target,
225
+ output.tactile_valid)
226
+ if output.rgb.valid.any():
227
+ rgb = roi_weighted_world_loss(output.rgb, batch["future_roi"].bool(),
228
+ background_weight=0.1)
229
+ loss = action + 1e-4 * delta_l2 + lambda_rgb * rgb + lambda_tactile * tactile
230
+ return loss, {"action": float(action.detach()), "rgb": float(rgb.detach()),
231
+ "tactile": float(tactile.detach()), "delta_l2": float(delta_l2.detach())}
232
+
233
+
234
+ @torch.no_grad()
235
+ def validate(model, dataset, statistics, batch_size, workers, device):
236
+ model.eval()
237
+ total = base_total = 0.0
238
+ count = 0
239
+ for raw in DataLoader(dataset, batch_size=batch_size, num_workers=workers,
240
+ pin_memory=True, persistent_workers=workers > 0):
241
+ batch = prepare_batch(raw, device, statistics)
242
+ prediction, _, allowed, _, _, _ = forward(model, batch)
243
+ active = allowed.any((1, 2))
244
+ losses = per_sample_action_loss(prediction[active], batch["target_action"][active],
245
+ allowed[active], groups=FULL_ACTION_GROUPS)
246
+ bases = per_sample_action_loss(batch["base_action"][active], batch["target_action"][active],
247
+ allowed[active], groups=FULL_ACTION_GROUPS)
248
+ total += float(losses.sum())
249
+ base_total += float(bases.sum())
250
+ count += len(losses)
251
+ return {"action_loss": total / max(count, 1),
252
+ "base_action_loss": base_total / max(count, 1), "rows": count}
253
+
254
+
255
+ def base_actions(dataset):
256
+ count = len(dataset.common["target_action"])
257
+ if dataset.model != "twla":
258
+ return np.asarray(dataset.base_action)
259
+ result = np.empty((count, 16, 192), np.float32)
260
+ for shard in dataset.shards:
261
+ result[shard["query_index"]] = shard["base_action"]
262
+ return result
263
+
264
+
265
+ @torch.no_grad()
266
+ def infer_condition(model, dataset, statistics, batch_size, workers, device, condition,
267
+ permutation=None):
268
+ model.eval()
269
+ base = base_actions(dataset)
270
+ prediction = base.copy()
271
+ mean = statistics["action_mean"]
272
+ scale = statistics["action_std"]
273
+ loader = DataLoader(dataset, batch_size=batch_size, num_workers=workers,
274
+ pin_memory=True, persistent_workers=workers > 0)
275
+ row_offset = 0
276
+ for raw in loader:
277
+ indices = raw["query_index"].numpy()
278
+ if condition == "shuffled_touch":
279
+ donors = permutation[row_offset:row_offset + len(indices)]
280
+ for name in ("pressure", "pressure_mask", "sample_valid"):
281
+ raw[name] = torch.stack([dataset[int(donor)][name] for donor in donors])
282
+ elif condition == "zero_touch":
283
+ reference = statistics["pressure_mean"].view(1, 1, 2, 880)
284
+ raw["pressure"] = reference.expand_as(raw["pressure"]).clone()
285
+ batch = prepare_batch(raw, device, statistics)
286
+ values, _, allowed, _, _, _ = forward(model, batch)
287
+ physical = values.float().cpu() * scale + mean
288
+ physical = physical.numpy()
289
+ allowed = allowed.cpu().numpy()
290
+ for local, index in enumerate(indices):
291
+ prediction[index][allowed[local]] = physical[local][allowed[local]]
292
+ row_offset += len(indices)
293
+ return prediction
294
+
295
+
296
+ def final_test(model, dataset, statistics, args, device, output):
297
+ count = len(dataset.common["target_action"])
298
+ row_hands = dataset.latest_hand[dataset.rows[:, 0], dataset.rows[:, 1]]
299
+ codes = row_hands[:, 0].astype(np.int8) + 2 * row_hands[:, 1].astype(np.int8)
300
+ permutation = np.arange(len(dataset))
301
+ rng = np.random.default_rng(args.seed + 700)
302
+ for tick in np.unique(dataset.rows[:, 1]):
303
+ for code in (0, 1, 2, 3):
304
+ rows = np.flatnonzero((dataset.rows[:, 1] == tick) & (codes == code))
305
+ donors = rows.copy()
306
+ rng.shuffle(donors)
307
+ permutation[rows] = donors
308
+ base = base_actions(dataset)
309
+ predictions = {"base": base}
310
+ for condition in ("editor", "shuffled_touch", "zero_touch"):
311
+ predictions[condition] = infer_condition(
312
+ model, dataset, statistics, args.micro_batch, args.workers, device,
313
+ condition, permutation)
314
+ np.save(output / f"test_{condition}_action.npy", predictions[condition])
315
+ target = np.asarray(dataset.common["target_action"])
316
+ mask = np.asarray(dataset.common["action_mask"])
317
+ episodes = dataset.identity["episode_id"].astype(str)
318
+ environment = np.full(count, "Tachin")
319
+ metrics = {
320
+ name: summarize_physical(value, target, mask, episodes, environment,
321
+ horizon=8, dataset="tachin")["overall"]
322
+ for name, value in predictions.items()
323
+ }
324
+ metric_names = ("translation_cm", "rotation_deg", "joint_mpjpe_cm")
325
+ reduction = {name: metrics["base"][name] - metrics["editor"][name] for name in metric_names}
326
+ percentage = {name: 100 * reduction[name] / metrics["base"][name] for name in metric_names}
327
+ cells = {name: f"{reduction[name]:.2f} / ({percentage[name]:.2f}%)" for name in metric_names}
328
+ return {"horizon": "first 8 actions / 100 ms", "aggregation": "episode-equal",
329
+ "editor_protocol": "paired-30" if dataset.paired30 else "single-decision",
330
+ "decision_ticks": [0, 1, 2] if dataset.paired30 else [0],
331
+ "metrics": metrics, "physical_error_reduction": reduction,
332
+ "physical_error_reduction_percent": percentage, "table1_cells": cells}
333
+
334
+
335
+ def main() -> None:
336
+ parser = argparse.ArgumentParser()
337
+ parser.add_argument("--cache", type=Path, required=True)
338
+ parser.add_argument("--model", choices=("smolvla", "vitra", "trex", "twla"), required=True)
339
+ parser.add_argument("--output", type=Path, required=True)
340
+ parser.add_argument("--world-checkpoint", type=Path)
341
+ parser.add_argument("--world-config", type=Path)
342
+ parser.add_argument("--lambda-rgb", type=float, default=0.1,
343
+ help="T-WLA W3 RGB-world objective weight")
344
+ parser.add_argument("--lambda-tactile", type=float, default=1e-4,
345
+ help="T-WLA W3 tactile-world objective weight")
346
+ parser.add_argument("--epochs", type=int, default=30,
347
+ help="Maximum epochs; validation early stopping can finish sooner")
348
+ parser.add_argument("--min-epochs", type=int, default=5)
349
+ parser.add_argument("--patience", type=int, default=5,
350
+ help="Stop after this many validation epochs without improvement")
351
+ parser.add_argument("--min-delta", type=float, default=1e-4,
352
+ help="Minimum absolute validation action-loss improvement")
353
+ parser.add_argument("--micro-batch", type=int, default=32)
354
+ parser.add_argument("--effective-batch", type=int, default=256)
355
+ parser.add_argument("--workers", type=int, default=4)
356
+ parser.add_argument("--seed", type=int, default=20260922)
357
+ parser.add_argument("--smoke-steps", type=int, default=0)
358
+ parser.add_argument("--resume", action="store_true")
359
+ args = parser.parse_args()
360
+ if not 1 <= args.min_epochs <= args.epochs:
361
+ raise ValueError("min-epochs must be in [1, epochs]")
362
+ if args.patience < 1 or args.min_delta < 0:
363
+ raise ValueError("patience must be >= 1 and min-delta must be non-negative")
364
+ if args.lambda_rgb < 0 or args.lambda_tactile < 0:
365
+ raise ValueError("world objective weights must be non-negative")
366
+ if args.effective_batch % args.micro_batch:
367
+ raise ValueError("effective batch must be divisible by micro batch")
368
+ if args.model == "twla" and (args.world_checkpoint is None or args.world_config is None):
369
+ parser.error("T-WLA W3 requires the RGB world checkpoint and config")
370
+ args.output.mkdir(parents=True, exist_ok=True)
371
+ lock = (args.output / ".train.lock").open("w")
372
+ fcntl.flock(lock, fcntl.LOCK_EX)
373
+ if (args.output / "summary.json").exists():
374
+ print(json.dumps({"model": args.model, "status": "already complete"}), flush=True)
375
+ return
376
+ torch.set_num_threads(3)
377
+ torch.set_float32_matmul_precision("high")
378
+ torch.manual_seed(args.seed)
379
+ np.random.seed(args.seed)
380
+ device = torch.device("cuda:0")
381
+ stats_np = dict(np.load(args.cache / "statistics.npz", allow_pickle=False))
382
+ statistics = {name: torch.from_numpy(value).float() for name, value in stats_np.items()
383
+ if name in ("pressure_mean", "pressure_scale", "action_mean", "action_std")}
384
+ train = TachinEditorCache(args.cache, "train", args.model, train_or_val=True,
385
+ include_w3=args.model == "twla")
386
+ validation = TachinEditorCache(args.cache, "val", args.model, train_or_val=True,
387
+ include_w3=args.model == "twla")
388
+ model = make_model(args.model, statistics, device)
389
+ rgb_head = None
390
+ if args.model == "twla":
391
+ from train_unified_w0_w3 import load_rgb_world_head
392
+ rgb_head, _ = load_rgb_world_head(args.world_checkpoint, args.world_config, device)
393
+ groups = model.optimizer_groups()
394
+ lambda_rgb, lambda_tactile = args.lambda_rgb, args.lambda_tactile
395
+ else:
396
+ if args.lambda_rgb != 0.1 or args.lambda_tactile != 1e-4:
397
+ parser.error("world objective weights apply only to --model twla")
398
+ groups = [{"params": model.action.parameters(), "lr": 1e-4, "name": "action"},
399
+ {"params": model.encoder.parameters(), "lr": 1e-5, "name": "encoder"},
400
+ {"params": model.coefficients.parameters(), "lr": 1e-4, "name": "coefficients"}]
401
+ lambda_rgb = lambda_tactile = 0.0
402
+ optimizer = torch.optim.AdamW(groups, weight_decay=0.01)
403
+ steps_per_epoch = math.ceil(len(train) / args.effective_batch)
404
+ total_steps = args.epochs * steps_per_epoch
405
+ warmup = min(100, max(total_steps - 1, 0))
406
+ scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lambda step:
407
+ (step + 1) / max(warmup, 1) if step < warmup else
408
+ 0.1 + 0.45 * (1 + math.cos(math.pi * min((step - warmup) / max(total_steps - warmup, 1), 1))))
409
+ config = {**vars(args), "train_rows": len(train), "validation_rows": len(validation),
410
+ "split": "original Tachin episode split", "gated": False,
411
+ "normalization": "shared original-train-only", "horizon": "8 actions / 100 ms",
412
+ "editor_protocol": "paired-30" if train.paired30 else "single-decision",
413
+ "vision_plan_hz": 10.0,
414
+ "tactile_input_hz": 30.0 if train.paired30 else 10.0,
415
+ "editor_decision_hz": 30.0 if train.paired30 else 10.0,
416
+ "decision_ticks": [0, 1, 2] if train.paired30 else [0],
417
+ "editor": "native 15-region 880-cell", "twla_variant": "W3" if args.model == "twla" else None,
418
+ "lambda_rgb": lambda_rgb, "lambda_tactile": lambda_tactile}
419
+ atomic_json(args.output / "config.json", config)
420
+ best = float("inf")
421
+ step = 0
422
+ first_epoch = 0
423
+ epochs_without_improvement = 0
424
+ started = time.time()
425
+ accumulation = args.effective_batch // args.micro_batch
426
+ generator = torch.Generator(device=device).manual_seed(args.seed + 1)
427
+ if args.resume:
428
+ resume_path = args.output / "last.pt"
429
+ if not resume_path.is_file():
430
+ raise FileNotFoundError(f"--resume requested but {resume_path} is missing")
431
+ checkpoint = torch.load(resume_path, map_location=device, weights_only=False)
432
+ required = {"model", "optimizer", "scheduler", "epoch", "step", "best",
433
+ "epochs_without_improvement"}
434
+ missing = required - set(checkpoint)
435
+ if missing:
436
+ raise ValueError(
437
+ f"{resume_path} predates resumable checkpoints and is missing {sorted(missing)}; "
438
+ "start a new output directory for a controlled early-stopping run")
439
+ model.load_state_dict(checkpoint["model"])
440
+ optimizer.load_state_dict(checkpoint["optimizer"])
441
+ scheduler.load_state_dict(checkpoint["scheduler"])
442
+ first_epoch = int(checkpoint["epoch"])
443
+ step = int(checkpoint["step"])
444
+ best = float(checkpoint["best"])
445
+ epochs_without_improvement = int(checkpoint["epochs_without_improvement"])
446
+ if checkpoint.get("aux_generator_state") is not None:
447
+ # torch.load(map_location=device) also moves the RNG byte state,
448
+ # while Generator.set_state requires a CPU ByteTensor.
449
+ generator.set_state(checkpoint["aux_generator_state"].cpu())
450
+ print(json.dumps({"model": args.model, "status": "resumed",
451
+ "next_epoch": first_epoch + 1, "step": step,
452
+ "best_validation_action_loss": best}), flush=True)
453
+ stopped_early = False
454
+ for epoch in range(first_epoch, args.epochs):
455
+ model.train()
456
+ loader = DataLoader(train, batch_size=args.micro_batch, shuffle=True,
457
+ generator=torch.Generator().manual_seed(args.seed + epoch),
458
+ num_workers=args.workers, pin_memory=True,
459
+ persistent_workers=args.workers > 0)
460
+ optimizer.zero_grad(set_to_none=True)
461
+ window = 0
462
+ sums = np.zeros(4)
463
+ seen = 0
464
+ for batch_index, raw in enumerate(loader):
465
+ batch = prepare_batch(raw, device, statistics)
466
+ with torch.autocast("cuda", dtype=torch.bfloat16):
467
+ loss, parts = objective(model, batch, rgb_head=rgb_head, generator=generator,
468
+ lambda_rgb=lambda_rgb, lambda_tactile=lambda_tactile)
469
+ count = len(batch["base_action"])
470
+ (loss * count / args.effective_batch).backward()
471
+ window += count
472
+ seen += count
473
+ sums += np.asarray(list(parts.values())) * count
474
+ if (batch_index + 1) % accumulation == 0 or batch_index + 1 == len(loader):
475
+ if window != args.effective_batch:
476
+ for parameter in model.parameters():
477
+ if parameter.grad is not None:
478
+ parameter.grad.mul_(args.effective_batch / window)
479
+ gradient = torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0,
480
+ error_if_nonfinite=True)
481
+ optimizer.step(); scheduler.step(); optimizer.zero_grad(set_to_none=True)
482
+ window = 0
483
+ step += 1
484
+ if step % 25 == 0 or args.smoke_steps:
485
+ print(json.dumps({"model": args.model, "step": step,
486
+ "total_steps": total_steps, "loss": parts,
487
+ "gradient": float(gradient)}), flush=True)
488
+ if args.smoke_steps and step >= args.smoke_steps:
489
+ atomic_json(args.output / "smoke.json", {"steps": step, "loss": parts,
490
+ "gradient": float(gradient)})
491
+ return
492
+ validation_metrics = validate(model, validation, statistics, args.micro_batch,
493
+ args.workers, device)
494
+ improved = validation_metrics["action_loss"] < best - args.min_delta
495
+ if improved:
496
+ best = validation_metrics["action_loss"]
497
+ epochs_without_improvement = 0
498
+ else:
499
+ epochs_without_improvement += 1
500
+ checkpoint = {"model": model.state_dict(), "optimizer": optimizer.state_dict(),
501
+ "scheduler": scheduler.state_dict(), "epoch": epoch + 1, "step": step,
502
+ "validation": validation_metrics, "config": config,
503
+ "statistics": statistics, "best": best,
504
+ "epochs_without_improvement": epochs_without_improvement,
505
+ "aux_generator_state": generator.get_state()}
506
+ torch.save(checkpoint, args.output / "last.pt")
507
+ if improved:
508
+ torch.save(checkpoint, args.output / "best.pt")
509
+ record = {"epoch": epoch + 1, "step": step,
510
+ "train": dict(zip(("action", "rgb", "tactile", "delta_l2"),
511
+ (sums / max(seen, 1)).tolist())),
512
+ "validation": validation_metrics, "improved": improved,
513
+ "epochs_without_improvement": epochs_without_improvement,
514
+ "seconds": time.time() - started}
515
+ with (args.output / "epochs.jsonl").open("a") as stream:
516
+ stream.write(json.dumps(record) + "\n")
517
+ print(json.dumps({"model": args.model, **record}), flush=True)
518
+ if epoch + 1 >= args.min_epochs and epochs_without_improvement >= args.patience:
519
+ stopped_early = True
520
+ print(json.dumps({"model": args.model, "status": "early_stop",
521
+ "epoch": epoch + 1, "patience": args.patience,
522
+ "best_validation_action_loss": best}), flush=True)
523
+ break
524
+ selected = torch.load(args.output / "best.pt", map_location=device, weights_only=False)
525
+ model.load_state_dict(selected["model"])
526
+ test = TachinEditorCache(args.cache, "test", args.model, train_or_val=False,
527
+ include_w3=args.model == "twla")
528
+ result = final_test(model, test, statistics, args, device, args.output)
529
+ atomic_json(args.output / "summary.json", {
530
+ "model": args.model, "selected_epoch": selected["epoch"],
531
+ "selected_step": selected["step"], "validation": selected["validation"],
532
+ "test": result, "seconds": time.time() - started, "config": config,
533
+ "stopped_early": stopped_early})
534
+ print(json.dumps({"model": args.model, "status": "complete",
535
+ "table1": result["table1_cells"]}), flush=True)
536
+
537
+
538
+ if __name__ == "__main__":
539
+ main()
model/tachin_paired30_cache_20260923/code/src/tactile_editor/action_layout.py ADDED
@@ -0,0 +1,123 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Canonical Method-D action layout shared by training and evaluation.
2
+
3
+ The VLA emits 192 values per action step. The first 138 values are the two
4
+ 69-dimensional hand actions; the final 54 values are padding and are never
5
+ editable or scored. Keeping this contract in one module prevents model rows
6
+ from silently using different masks for Table 1.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from dataclasses import dataclass
12
+
13
+ import torch
14
+
15
+
16
+ ACTION_DIM = 192
17
+ CHUNK_SIZE = 16
18
+ VALID_ACTION_DIM = 138
19
+ HAND_ACTION_DIM = 69
20
+
21
+ LEFT_TRANSLATION = slice(0, 3)
22
+ LEFT_ROTATION = slice(3, 6)
23
+ LEFT_ARTICULATION = slice(6, 69)
24
+ RIGHT_TRANSLATION = slice(69, 72)
25
+ RIGHT_ROTATION = slice(72, 75)
26
+ RIGHT_ARTICULATION = slice(75, 138)
27
+
28
+ TRANSLATION_GROUPS = ((0, 3), (69, 72))
29
+ ROTATION_GROUPS = ((3, 6), (72, 75))
30
+ ARTICULATION_GROUPS = ((6, 69), (75, 138))
31
+ FULL_ACTION_GROUPS = (
32
+ (0, 3),
33
+ (3, 6),
34
+ (6, 69),
35
+ (69, 72),
36
+ (72, 75),
37
+ (75, 138),
38
+ )
39
+ FULL_HAND_GROUPS = ((0, 69), (69, 138))
40
+
41
+
42
+ @dataclass(frozen=True)
43
+ class ActionLayout:
44
+ """Shape and physical component metadata for a normalized action chunk."""
45
+
46
+ chunk_size: int = CHUNK_SIZE
47
+ action_dim: int = ACTION_DIM
48
+ valid_action_dim: int = VALID_ACTION_DIM
49
+
50
+ def validate(self, action: torch.Tensor, *, name: str = "action") -> None:
51
+ if action.ndim < 2 or tuple(action.shape[-2:]) != (
52
+ self.chunk_size,
53
+ self.action_dim,
54
+ ):
55
+ raise ValueError(
56
+ f"{name} must end in [{self.chunk_size},{self.action_dim}], "
57
+ f"got {tuple(action.shape)}"
58
+ )
59
+
60
+ def physical_mask(
61
+ self,
62
+ shape: tuple[int, ...],
63
+ *,
64
+ device: torch.device | None = None,
65
+ groups: tuple[tuple[int, int], ...] = FULL_ACTION_GROUPS,
66
+ ) -> torch.Tensor:
67
+ if len(shape) < 2 or shape[-2:] != (self.chunk_size, self.action_dim):
68
+ raise ValueError("mask shape must end in [16,192]")
69
+ result = torch.zeros(shape, dtype=torch.bool, device=device)
70
+ for start, stop in groups:
71
+ if not 0 <= start < stop <= self.valid_action_dim:
72
+ raise ValueError("action group is outside the valid action range")
73
+ result[..., start:stop] = True
74
+ return result
75
+
76
+ def apply_edit_scope(
77
+ self,
78
+ action_mask: torch.Tensor,
79
+ *,
80
+ groups: tuple[tuple[int, int], ...] = FULL_ACTION_GROUPS,
81
+ ) -> torch.Tensor:
82
+ """Intersect a runtime mask with physical components and padding mask."""
83
+
84
+ self.validate(action_mask, name="action_mask")
85
+ return action_mask.bool() & self.physical_mask(
86
+ tuple(action_mask.shape), device=action_mask.device, groups=groups
87
+ )
88
+
89
+
90
+ DEFAULT_ACTION_LAYOUT = ActionLayout()
91
+
92
+
93
+ @dataclass(frozen=True)
94
+ class ActionNormalizer:
95
+ """Frozen affine statistics used by every model row."""
96
+
97
+ mean: torch.Tensor
98
+ std: torch.Tensor
99
+ identity: str = "unlabelled"
100
+
101
+ def __post_init__(self) -> None:
102
+ if tuple(self.mean.shape) != (ACTION_DIM,) or tuple(self.std.shape) != (
103
+ ACTION_DIM,
104
+ ):
105
+ raise ValueError("action statistics must have shape [192]")
106
+ if not torch.isfinite(self.mean).all() or not torch.isfinite(self.std).all():
107
+ raise ValueError("action statistics must be finite")
108
+ if (self.std <= 0).any():
109
+ raise ValueError("action standard deviations must be positive")
110
+ if not self.identity:
111
+ raise ValueError("normalizer identity must be non-empty")
112
+
113
+ def normalize(self, action: torch.Tensor) -> torch.Tensor:
114
+ DEFAULT_ACTION_LAYOUT.validate(action)
115
+ mean = self.mean.to(device=action.device, dtype=action.dtype)
116
+ std = self.std.to(device=action.device, dtype=action.dtype)
117
+ return (action - mean) / std
118
+
119
+ def denormalize(self, action: torch.Tensor) -> torch.Tensor:
120
+ DEFAULT_ACTION_LAYOUT.validate(action)
121
+ mean = self.mean.to(device=action.device, dtype=action.dtype)
122
+ std = self.std.to(device=action.device, dtype=action.dtype)
123
+ return action * std + mean
model/tachin_paired30_cache_20260923/code/src/tactile_editor/action_objectives.py ADDED
@@ -0,0 +1,233 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Group-balanced action objectives for tactile-editor experiments.
2
+
3
+ All public loss functions return one value per input sample. Action objectives
4
+ require every sample to contain at least one supervised value. Motion
5
+ objectives also return a validity mask because an editable suffix can contain no
6
+ adjacent valid pair.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import math
12
+ from collections.abc import Sequence
13
+
14
+ import torch
15
+ from torch.nn import functional as F
16
+
17
+ from .training import ACTION_GROUPS
18
+
19
+
20
+ ActionGroups = Sequence[tuple[int, int]]
21
+
22
+
23
+ def _validate_action_tensors(
24
+ prediction: torch.Tensor,
25
+ target: torch.Tensor,
26
+ mask: torch.Tensor,
27
+ ) -> torch.Tensor:
28
+ if prediction.ndim != 3:
29
+ raise ValueError("prediction must be [B,T,A]")
30
+ if target.shape != prediction.shape or mask.shape != prediction.shape:
31
+ raise ValueError("prediction, target, and mask must have matching [B,T,A] shapes")
32
+ if not prediction.is_floating_point() or not target.is_floating_point():
33
+ raise TypeError("prediction and target must be floating-point tensors")
34
+ if prediction.device != target.device or prediction.device != mask.device:
35
+ raise ValueError("prediction, target, and mask must be on the same device")
36
+ return mask.bool()
37
+
38
+
39
+ def _validate_groups(groups: ActionGroups, action_dim: int) -> tuple[tuple[int, int], ...]:
40
+ groups = tuple(groups)
41
+ if not groups:
42
+ raise ValueError("at least one action group is required")
43
+ occupied: set[int] = set()
44
+ for start, stop in groups:
45
+ if not 0 <= start < stop <= action_dim:
46
+ raise ValueError("action group outside the action dimension")
47
+ indices = set(range(start, stop))
48
+ if occupied & indices:
49
+ raise ValueError("action groups must not overlap")
50
+ occupied |= indices
51
+ return groups
52
+
53
+
54
+ def _validate_timestep_weights(
55
+ timestep_weights: torch.Tensor,
56
+ prediction: torch.Tensor,
57
+ ) -> torch.Tensor:
58
+ expected = (*prediction.shape[:2], 1)
59
+ if tuple(timestep_weights.shape) != expected:
60
+ raise ValueError(f"timestep_weights must have shape {expected}")
61
+ weights = timestep_weights.to(device=prediction.device, dtype=prediction.dtype)
62
+ if not torch.isfinite(weights).all() or (weights < 0).any():
63
+ raise ValueError("timestep_weights must be finite and non-negative")
64
+ return weights
65
+
66
+
67
+ def _per_sample_group_balanced_smooth_l1(
68
+ prediction: torch.Tensor,
69
+ target: torch.Tensor,
70
+ mask: torch.Tensor,
71
+ *,
72
+ groups: ActionGroups,
73
+ timestep_weights: torch.Tensor | None = None,
74
+ ) -> tuple[torch.Tensor, torch.Tensor]:
75
+ """Return group-balanced losses and a mask of nonempty samples."""
76
+
77
+ mask = _validate_action_tensors(prediction, target, mask)
78
+ groups = _validate_groups(groups, prediction.shape[-1])
79
+ weights = (
80
+ None
81
+ if timestep_weights is None
82
+ else _validate_timestep_weights(timestep_weights, prediction)
83
+ )
84
+
85
+ # Cached padded targets can contain NaN. Select a finite, detached value
86
+ # before Smooth L1 rather than masking a NaN loss after arithmetic.
87
+ safe_target = torch.where(mask, target, prediction.detach())
88
+ error = F.smooth_l1_loss(prediction, safe_target, reduction="none")
89
+ group_values, group_present = [], []
90
+ for start, stop in groups:
91
+ selected = mask[..., start:stop]
92
+ if weights is None:
93
+ denominator = selected.sum((1, 2))
94
+ numerator = error[..., start:stop].masked_fill(~selected, 0).sum((1, 2))
95
+ else:
96
+ # weights is [B,T,1] and broadcasts over the selected action group.
97
+ weighted_mask = weights * selected.to(dtype=error.dtype)
98
+ denominator = weighted_mask.sum((1, 2))
99
+ numerator = (error[..., start:stop] * weighted_mask).sum((1, 2))
100
+ # A normalized exponential weight sum can be below one (for example,
101
+ # one far-future scalar). Clamp only for numerical safety; clamping to
102
+ # one would silently shrink such a group's loss.
103
+ group_values.append(
104
+ numerator / denominator.clamp_min(torch.finfo(error.dtype).tiny)
105
+ )
106
+ group_present.append(denominator > 0)
107
+
108
+ present = torch.stack(group_present, dim=1)
109
+ per_sample = torch.stack(group_values, dim=1).sum(1) / present.sum(1).clamp_min(1)
110
+ valid = present.any(1)
111
+ return per_sample.masked_fill(~valid, 0), valid
112
+
113
+
114
+ def per_sample_action_loss(
115
+ prediction: torch.Tensor,
116
+ target: torch.Tensor,
117
+ mask: torch.Tensor,
118
+ *,
119
+ groups: ActionGroups = ACTION_GROUPS,
120
+ ) -> torch.Tensor:
121
+ """Return the existing group-balanced Smooth L1 action loss as ``[B]``."""
122
+
123
+ values, valid = _per_sample_group_balanced_smooth_l1(
124
+ prediction, target, mask, groups=groups
125
+ )
126
+ if not valid.all():
127
+ raise ValueError("every action-loss sample must contain supervised actions")
128
+ return values
129
+
130
+
131
+ def near_horizon_weights(
132
+ decision_seconds: torch.Tensor,
133
+ horizon: int,
134
+ *,
135
+ tau_seconds: float,
136
+ action_hz: float = 80.0,
137
+ dtype: torch.dtype = torch.float32,
138
+ ) -> torch.Tensor:
139
+ """Return ``exp(-(t_k-t_decision)/tau)`` weights shaped ``[B,T,1]``.
140
+
141
+ Negative delays are clamped to zero. A future/edit mask must still be
142
+ supplied to the loss; this function does not decide which actions are
143
+ committed.
144
+ """
145
+
146
+ if decision_seconds.ndim != 1:
147
+ raise ValueError("decision_seconds must be [B]")
148
+ if horizon < 1:
149
+ raise ValueError("horizon must be positive")
150
+ if not math.isfinite(tau_seconds) or tau_seconds <= 0:
151
+ raise ValueError("tau_seconds must be finite and positive")
152
+ if not math.isfinite(action_hz) or action_hz <= 0:
153
+ raise ValueError("action_hz must be finite and positive")
154
+ decision = decision_seconds.to(dtype=torch.float64)
155
+ if not torch.isfinite(decision).all() or (decision < 0).any():
156
+ raise ValueError("decision_seconds must be finite and non-negative")
157
+ ticks = torch.arange(
158
+ horizon, device=decision.device, dtype=torch.float64
159
+ )[None, :, None] / action_hz
160
+ delay = (ticks - decision[:, None, None]).clamp_min(0)
161
+ return torch.exp(-delay / tau_seconds).to(dtype=dtype)
162
+
163
+
164
+ def per_sample_near_horizon_action_loss(
165
+ prediction: torch.Tensor,
166
+ target: torch.Tensor,
167
+ mask: torch.Tensor,
168
+ decision_seconds: torch.Tensor,
169
+ *,
170
+ tau_seconds: float,
171
+ action_hz: float = 80.0,
172
+ groups: ActionGroups = ACTION_GROUPS,
173
+ ) -> torch.Tensor:
174
+ """Return near-horizon, group-balanced action losses as ``[B]``.
175
+
176
+ Each group is divided by its sum of valid temporal weights, so changing
177
+ ``tau_seconds`` changes temporal emphasis without changing the loss scale.
178
+ """
179
+
180
+ weights = near_horizon_weights(
181
+ decision_seconds,
182
+ prediction.shape[1],
183
+ tau_seconds=tau_seconds,
184
+ action_hz=action_hz,
185
+ dtype=prediction.dtype,
186
+ )
187
+ if weights.device != prediction.device:
188
+ weights = weights.to(prediction.device)
189
+ values, valid = _per_sample_group_balanced_smooth_l1(
190
+ prediction,
191
+ target,
192
+ mask,
193
+ groups=groups,
194
+ timestep_weights=weights,
195
+ )
196
+ if not valid.all():
197
+ raise ValueError("every near-horizon sample must contain supervised actions")
198
+ return values
199
+
200
+
201
+ def per_sample_motion_loss(
202
+ prediction: torch.Tensor,
203
+ target: torch.Tensor,
204
+ mask: torch.Tensor,
205
+ *,
206
+ groups: ActionGroups = ACTION_GROUPS,
207
+ timestep_weights: torch.Tensor | None = None,
208
+ ) -> tuple[torch.Tensor, torch.Tensor]:
209
+ """Return adjacent-action motion losses and valid samples, both shaped ``[B]``.
210
+
211
+ A transition is supervised only when both endpoints are in ``mask``. With a
212
+ future edit mask this excludes the committed-to-first-editable boundary.
213
+ Optional ``timestep_weights`` has shape ``[B,T,1]``; transition ``k-1 -> k``
214
+ uses the weight at its later endpoint ``k``.
215
+ """
216
+
217
+ mask = _validate_action_tensors(prediction, target, mask)
218
+ weights = (
219
+ None
220
+ if timestep_weights is None
221
+ else _validate_timestep_weights(timestep_weights, prediction)[:, 1:]
222
+ )
223
+ safe_target = torch.where(mask, target, prediction.detach())
224
+ predicted_motion = prediction[:, 1:] - prediction[:, :-1]
225
+ target_motion = safe_target[:, 1:] - safe_target[:, :-1]
226
+ pair_mask = mask[:, 1:] & mask[:, :-1]
227
+ return _per_sample_group_balanced_smooth_l1(
228
+ predicted_motion,
229
+ target_motion,
230
+ pair_mask,
231
+ groups=groups,
232
+ timestep_weights=weights,
233
+ )
model/tachin_paired30_cache_20260923/code/src/tactile_editor/action_output.py ADDED
@@ -0,0 +1,71 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Model-independent action output contract for Table 1 evaluation."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass, field
6
+ from typing import Any
7
+
8
+ import torch
9
+
10
+ from .action_layout import (
11
+ ACTION_DIM,
12
+ ARTICULATION_GROUPS,
13
+ CHUNK_SIZE,
14
+ DEFAULT_ACTION_LAYOUT,
15
+ FULL_ACTION_GROUPS,
16
+ )
17
+
18
+
19
+ @dataclass(frozen=True)
20
+ class NormalizedActionOutput:
21
+ """A model prediction in the frozen VLA normalized action space.
22
+
23
+ Every baseline row (native VLA, tactile editor, or adapted model) can be
24
+ exported in this format before denormalization and physical metric scoring.
25
+ ``valid_mask`` describes available hand/action values; padding is rejected
26
+ at construction time so it cannot enter losses or Table 1 metrics.
27
+ """
28
+
29
+ actions: torch.Tensor
30
+ valid_mask: torch.Tensor
31
+ model_id: str
32
+ normalization_id: str
33
+ metadata: dict[str, Any] = field(default_factory=dict)
34
+
35
+ def __post_init__(self) -> None:
36
+ if self.actions.ndim != 3 or tuple(self.actions.shape[1:]) != (
37
+ CHUNK_SIZE,
38
+ ACTION_DIM,
39
+ ):
40
+ raise ValueError("actions must be [B,16,192]")
41
+ if self.valid_mask.shape != self.actions.shape:
42
+ raise ValueError("valid_mask must match actions")
43
+ if not self.actions.is_floating_point() or not torch.isfinite(self.actions).all():
44
+ raise ValueError("actions must be finite floating-point values")
45
+ if self.valid_mask[..., 138:].any():
46
+ raise ValueError("padding dimensions 138:192 must be invalid")
47
+ if not self.model_id or not self.normalization_id:
48
+ raise ValueError("model_id and normalization_id are required")
49
+
50
+ @property
51
+ def physical_mask(self) -> torch.Tensor:
52
+ return self.valid_mask.bool() & DEFAULT_ACTION_LAYOUT.physical_mask(
53
+ tuple(self.valid_mask.shape), device=self.valid_mask.device
54
+ )
55
+
56
+ def editable(self, *, articulation_only: bool = False) -> torch.Tensor:
57
+ groups = ARTICULATION_GROUPS if articulation_only else FULL_ACTION_GROUPS
58
+ return self.valid_mask.bool() & DEFAULT_ACTION_LAYOUT.physical_mask(
59
+ tuple(self.valid_mask.shape),
60
+ device=self.valid_mask.device,
61
+ groups=groups,
62
+ )
63
+
64
+ def denormalize(self, mean: torch.Tensor, std: torch.Tensor) -> torch.Tensor:
65
+ if tuple(mean.shape) != (ACTION_DIM,) or tuple(std.shape) != (ACTION_DIM,):
66
+ raise ValueError("normalization statistics must be [192]")
67
+ if not torch.isfinite(mean).all() or not torch.isfinite(std).all() or (std <= 0).any():
68
+ raise ValueError("normalization statistics must be finite and positive")
69
+ mean = mean.to(device=self.actions.device, dtype=self.actions.dtype)
70
+ std = std.to(device=self.actions.device, dtype=self.actions.dtype)
71
+ return self.actions * std + mean
model/tachin_paired30_cache_20260923/code/src/tactile_editor/architecture_ablation.py ADDED
@@ -0,0 +1,423 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Stage-2-only tactile conditioning architecture ablations.
2
+
3
+ The calibrated action is an immutable Stage-1 anchor. These research models
4
+ change only how frozen tactile tokens condition an incremental residual.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from dataclasses import dataclass
10
+ import math
11
+
12
+ import torch
13
+ from torch import nn
14
+
15
+ from .touch_study import IncrementalTouchEditor, StudySetting
16
+
17
+
18
+ ARCHITECTURES = ("e0_cross_attention", "e1_adaln", "e2_basis", "e3_centered_basis")
19
+
20
+
21
+ @dataclass(frozen=True)
22
+ class ArchitectureSpec:
23
+ name: str
24
+ conditioning: str
25
+ output: str
26
+ centered_coefficients: bool = False
27
+ residual_bases: int = 0
28
+
29
+
30
+ SPECS = {
31
+ "e0_cross_attention": ArchitectureSpec(
32
+ "e0_cross_attention", "gated_action_to_touch_cross_attention", "direct_delta"
33
+ ),
34
+ "e1_adaln": ArchitectureSpec(
35
+ "e1_adaln", "adaln_zero", "direct_delta"
36
+ ),
37
+ "e2_basis": ArchitectureSpec(
38
+ "e2_basis", "adaln_zero", "tactile_weighted_residual_bases", residual_bases=8
39
+ ),
40
+ "e3_centered_basis": ArchitectureSpec(
41
+ "e3_centered_basis",
42
+ "adaln_zero",
43
+ "tactile_weighted_residual_bases",
44
+ centered_coefficients=True,
45
+ residual_bases=8,
46
+ ),
47
+ }
48
+
49
+
50
+ def cross_video_phase_matched_donors(
51
+ videos: list[str],
52
+ elapsed: torch.Tensor,
53
+ hand_valid: torch.Tensor,
54
+ *,
55
+ seed: int = 20260907,
56
+ ) -> torch.Tensor:
57
+ """Build a deterministic one-to-one tactile shuffle within phase/validity groups.
58
+
59
+ Grouped cyclic rotation is a true permutation and guarantees that the donor
60
+ source video differs when no video owns more than half of a group.
61
+ """
62
+ if len(videos) != len(elapsed) or len(videos) != len(hand_valid):
63
+ raise ValueError("video, elapsed, and hand_valid lengths must match")
64
+ generator = torch.Generator().manual_seed(seed)
65
+ phases = torch.round(elapsed.detach().cpu().float() * 30).long()
66
+ validity = hand_valid.detach().cpu().bool()
67
+ donors = torch.full((len(videos),), -1, dtype=torch.long)
68
+ keys = sorted(
69
+ {
70
+ (int(phases[i]), bool(validity[i, 0]), bool(validity[i, 1]))
71
+ for i in range(len(videos))
72
+ }
73
+ )
74
+ for key in keys:
75
+ indices = [
76
+ i
77
+ for i in range(len(videos))
78
+ if (int(phases[i]), bool(validity[i, 0]), bool(validity[i, 1])) == key
79
+ ]
80
+ by_video: dict[str, list[int]] = {}
81
+ for index in indices:
82
+ by_video.setdefault(videos[index], []).append(index)
83
+ maximum = max(map(len, by_video.values()))
84
+ if maximum * 2 > len(indices):
85
+ raise ValueError(
86
+ f"cannot construct cross-video permutation for group {key}: "
87
+ f"largest video has {maximum}/{len(indices)} samples"
88
+ )
89
+ names = sorted(by_video)
90
+ names = [names[i] for i in torch.randperm(len(names), generator=generator)]
91
+ ordered = []
92
+ for name in names:
93
+ rows = by_video[name]
94
+ order = torch.randperm(len(rows), generator=generator).tolist()
95
+ ordered.extend(rows[i] for i in order)
96
+ rotated = ordered[maximum:] + ordered[:maximum]
97
+ if any(videos[source] == videos[donor] for source, donor in zip(ordered, rotated)):
98
+ raise AssertionError("grouped rotation failed to change source video")
99
+ for source, donor in zip(ordered, rotated):
100
+ donors[source] = donor
101
+ if (donors < 0).any() or len(torch.unique(donors)) != len(donors):
102
+ raise AssertionError("donor mapping must be a complete permutation")
103
+ return donors
104
+
105
+
106
+ def within_video_phase_matched_donors(
107
+ videos: list[str],
108
+ elapsed: torch.Tensor,
109
+ hand_valid: torch.Tensor,
110
+ *,
111
+ seed: int = 20260907,
112
+ ) -> torch.Tensor:
113
+ """Shuffle tactile samples inside one recording and decision phase.
114
+
115
+ This control preserves recording identity, decision phase, and the two-hand
116
+ validity pattern. It breaks the local action-touch pairing without creating
117
+ the cross-recording context shift of the primary shuffled control.
118
+ """
119
+
120
+ if len(videos) != len(elapsed) or len(videos) != len(hand_valid):
121
+ raise ValueError("video, elapsed, and hand_valid lengths must match")
122
+ generator = torch.Generator().manual_seed(seed)
123
+ phases = torch.round(elapsed.detach().cpu().float() * 30).long()
124
+ validity = hand_valid.detach().cpu().bool()
125
+ donors = torch.full((len(videos),), -1, dtype=torch.long)
126
+ groups: dict[tuple[str, int, bool, bool], list[int]] = {}
127
+ for index, video in enumerate(videos):
128
+ key = (
129
+ video,
130
+ int(phases[index]),
131
+ bool(validity[index, 0]),
132
+ bool(validity[index, 1]),
133
+ )
134
+ groups.setdefault(key, []).append(index)
135
+ for key, indices in sorted(groups.items()):
136
+ if len(indices) < 2:
137
+ raise ValueError(f"cannot shuffle singleton within-video group {key}")
138
+ order = torch.randperm(len(indices), generator=generator).tolist()
139
+ ordered = [indices[position] for position in order]
140
+ rotated = ordered[1:] + ordered[:1]
141
+ for source, donor in zip(ordered, rotated):
142
+ donors[source] = donor
143
+ if (donors < 0).any() or len(torch.unique(donors)) != len(donors):
144
+ raise AssertionError("within-video donor mapping must be a complete permutation")
145
+ if any(videos[i] != videos[int(donors[i])] for i in range(len(videos))):
146
+ raise AssertionError("within-video donor changed recording")
147
+ if (donors == torch.arange(len(donors))).any():
148
+ raise AssertionError("within-video donor mapping must change every sample")
149
+ return donors
150
+
151
+
152
+ def tactile_contact_statistics(
153
+ touch_pressure: torch.Tensor,
154
+ touch_valid: torch.Tensor,
155
+ pressure_masks: torch.Tensor,
156
+ ) -> dict[str, torch.Tensor]:
157
+ """Compute causal per-sample contact scores from two latest raw 30 Hz frames."""
158
+ if touch_pressure.ndim != 5 or tuple(touch_pressure.shape[1:]) != (8, 2, 21, 21):
159
+ raise ValueError("touch_pressure must be [N,8,2,21,21]")
160
+ if tuple(touch_valid.shape) != tuple(touch_pressure.shape[:3]):
161
+ raise ValueError("touch_valid must be [N,8,2]")
162
+ if tuple(pressure_masks.shape) != (2, 21, 21):
163
+ raise ValueError("pressure_masks must be [2,21,21]")
164
+ pressure = torch.nan_to_num(touch_pressure.detach().cpu().float()).clone()
165
+ pressure[:, :, 0] = pressure[:, :, 0].flip(-1)
166
+ masks = pressure_masks.detach().cpu().bool()
167
+ cells = torch.stack(
168
+ [pressure[:, :, hand, masks[hand]] for hand in range(2)], dim=2
169
+ )
170
+ scores = cells.topk(5, dim=-1).values.mean(-1)
171
+ pair_valid = touch_valid.detach().cpu().bool()[:, -2:].all(1)
172
+ current_valid = touch_valid.detach().cpu().bool()[:, -1]
173
+ current = scores[:, -1]
174
+ previous = scores[:, -2]
175
+ return {
176
+ "current_score_per_hand": current,
177
+ "previous_score_per_hand": previous,
178
+ "current_valid_per_hand": current_valid,
179
+ "pair_valid_per_hand": pair_valid,
180
+ }
181
+
182
+
183
+ def tactile_event_change(
184
+ touch_pressure: torch.Tensor,
185
+ touch_valid: torch.Tensor,
186
+ pressure_masks: torch.Tensor,
187
+ ) -> torch.Tensor:
188
+ """Mean absolute pressure change over valid cells and hands at the decision."""
189
+
190
+ if touch_pressure.ndim != 5 or tuple(touch_pressure.shape[1:]) != (8, 2, 21, 21):
191
+ raise ValueError("touch_pressure must be [N,8,2,21,21]")
192
+ if tuple(touch_valid.shape) != tuple(touch_pressure.shape[:3]):
193
+ raise ValueError("touch_valid must be [N,8,2]")
194
+ pressure = torch.nan_to_num(touch_pressure.detach().cpu().float()).clone()
195
+ pressure[:, :, 0] = pressure[:, :, 0].flip(-1)
196
+ masks = pressure_masks.detach().cpu().bool()
197
+ per_hand = []
198
+ for hand in range(2):
199
+ cells = pressure[:, -2:, hand, masks[hand]]
200
+ per_hand.append((cells[:, 1] - cells[:, 0]).abs().mean(-1))
201
+ change = torch.stack(per_hand, dim=1)
202
+ valid = touch_valid.detach().cpu().bool()[:, -2:].all(1)
203
+ return (change * valid).sum(1) / valid.sum(1).clamp_min(1)
204
+
205
+
206
+ def contact_slice_masks(
207
+ statistics: dict[str, torch.Tensor],
208
+ event_change: torch.Tensor,
209
+ *,
210
+ high_contact_threshold: float,
211
+ pressure_change_threshold: float,
212
+ contact_threshold: float = 0.1,
213
+ ) -> dict[str, torch.Tensor]:
214
+ """Return the five overlapping evaluation slices used by the ablation."""
215
+ current = statistics["current_score_per_hand"]
216
+ previous = statistics["previous_score_per_hand"]
217
+ current_valid = statistics["current_valid_per_hand"]
218
+ pair_valid = statistics["pair_valid_per_hand"]
219
+ now_contact = current >= contact_threshold
220
+ previous_contact = previous >= contact_threshold
221
+ transition = ((now_contact != previous_contact) & pair_valid).any(1)
222
+ changing = event_change.detach().cpu().float() >= pressure_change_threshold
223
+ stable_contact = (
224
+ (now_contact & previous_contact & pair_valid).any(1)
225
+ & ~transition
226
+ & ~changing
227
+ )
228
+ low = pair_valid.any(1) & ~(
229
+ (now_contact | previous_contact) & pair_valid
230
+ ).any(1)
231
+ high = (
232
+ (current >= high_contact_threshold) & current_valid
233
+ ).any(1)
234
+ return {
235
+ "high_contact": high,
236
+ "contact_transition": transition,
237
+ "pressure_changing": changing,
238
+ "stable_contact": stable_contact,
239
+ "low_or_no_contact": low,
240
+ }
241
+
242
+
243
+ def _modulate(x: torch.Tensor, shift: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
244
+ return x * (1 + scale[:, None]) + shift[:, None]
245
+
246
+
247
+ class AdaLNZeroBlock(nn.Module):
248
+ """One pre-norm Transformer block with zero-initialized tactile modulation."""
249
+
250
+ def __init__(self, hidden: int = 128, heads: int = 4, dropout: float = 0.1):
251
+ super().__init__()
252
+ self.norm_attention = nn.LayerNorm(hidden, elementwise_affine=False)
253
+ self.attention = nn.MultiheadAttention(
254
+ hidden, heads, dropout=dropout, batch_first=True
255
+ )
256
+ self.norm_mlp = nn.LayerNorm(hidden, elementwise_affine=False)
257
+ self.mlp = nn.Sequential(
258
+ nn.Linear(hidden, hidden * 4),
259
+ nn.GELU(),
260
+ nn.Dropout(dropout),
261
+ nn.Linear(hidden * 4, hidden),
262
+ nn.Dropout(dropout),
263
+ )
264
+ self.modulation = nn.Sequential(nn.SiLU(), nn.Linear(hidden, hidden * 6))
265
+ nn.init.zeros_(self.modulation[-1].weight)
266
+ nn.init.zeros_(self.modulation[-1].bias)
267
+
268
+ def forward(self, x: torch.Tensor, condition: torch.Tensor) -> torch.Tensor:
269
+ shift_attn, scale_attn, gate_attn, shift_mlp, scale_mlp, gate_mlp = (
270
+ self.modulation(condition).chunk(6, dim=-1)
271
+ )
272
+ q = _modulate(self.norm_attention(x), shift_attn, scale_attn)
273
+ attended = self.attention(q, q, q, need_weights=False)[0]
274
+ x = x + gate_attn[:, None] * attended
275
+ feedforward = self.mlp(_modulate(self.norm_mlp(x), shift_mlp, scale_mlp))
276
+ return x + gate_mlp[:, None] * feedforward
277
+
278
+
279
+ class AdaLNZeroTouchEditor(nn.Module):
280
+ """AdaLN-Zero direct-head or tactile-weighted-basis editor.
281
+
282
+ E2/E3 intentionally have no action-only delta output. Their residual can
283
+ only be formed by tactile coefficients weighting action-trunk bases.
284
+ """
285
+
286
+ def __init__(
287
+ self,
288
+ action_mean: torch.Tensor,
289
+ action_std: torch.Tensor,
290
+ architecture: str,
291
+ *,
292
+ hidden: int = 128,
293
+ residual_bases: int = 8,
294
+ dropout: float = 0.1,
295
+ ):
296
+ super().__init__()
297
+ if architecture not in ("e1_adaln", "e2_basis", "e3_centered_basis"):
298
+ raise ValueError(f"unsupported AdaLN architecture: {architecture}")
299
+ if residual_bases < 1:
300
+ raise ValueError("residual_bases must be positive")
301
+ self.architecture = architecture
302
+ self.hidden = hidden
303
+ self.residual_bases = residual_bases
304
+ self.centered_coefficients = architecture == "e3_centered_basis"
305
+ self.uses_basis_output = architecture in ("e2_basis", "e3_centered_basis")
306
+ self.register_buffer("action_mean", action_mean)
307
+ self.register_buffer("action_std", action_std.clamp_min(1e-3))
308
+ self.action_projection = nn.Linear(192, hidden)
309
+ self.position = nn.Parameter(torch.randn(1, 16, hidden) * 0.02)
310
+ self.time_projection = nn.Linear(1, hidden)
311
+ # Concatenation retains the fixed left/right identity of the two 64-D tokens.
312
+ self.touch_condition = nn.Linear(128, hidden)
313
+ self.trunk = AdaLNZeroBlock(hidden, heads=4, dropout=dropout)
314
+
315
+ if self.uses_basis_output:
316
+ self.basis_head = nn.Linear(hidden, residual_bases * 192)
317
+ nn.init.zeros_(self.basis_head.weight)
318
+ nn.init.zeros_(self.basis_head.bias)
319
+ self.coefficient_head = nn.Linear(
320
+ 128, residual_bases, bias=not self.centered_coefficients
321
+ )
322
+ else:
323
+ self.output = nn.Linear(hidden, 192)
324
+ nn.init.zeros_(self.output.weight)
325
+ nn.init.zeros_(self.output.bias)
326
+
327
+ @staticmethod
328
+ def _masked_dual_hand(tokens: torch.Tensor, hand_valid: torch.Tensor) -> torch.Tensor:
329
+ if tokens.ndim != 3 or tuple(tokens.shape[1:]) != (2, 64):
330
+ raise ValueError(f"tokens must be [B,2,64], got {tuple(tokens.shape)}")
331
+ if tuple(hand_valid.shape) != tuple(tokens.shape[:2]):
332
+ raise ValueError("hand_valid must match tokens' first two dimensions")
333
+ masked = tokens * hand_valid[..., None].to(tokens.dtype)
334
+ return masked.flatten(1)
335
+
336
+ def tactile_coefficients(
337
+ self,
338
+ tokens: torch.Tensor,
339
+ hand_valid: torch.Tensor,
340
+ mean_tokens: torch.Tensor,
341
+ ) -> torch.Tensor:
342
+ if not self.uses_basis_output:
343
+ raise RuntimeError("direct-head E1 has no residual-basis coefficients")
344
+ coefficient_tokens = tokens
345
+ if self.centered_coefficients:
346
+ coefficient_tokens = tokens - mean_tokens.expand_as(tokens)
347
+ return self.coefficient_head(
348
+ self._masked_dual_hand(coefficient_tokens, hand_valid)
349
+ )
350
+
351
+ def forward(
352
+ self,
353
+ anchor: torch.Tensor,
354
+ mask: torch.Tensor,
355
+ tokens: torch.Tensor,
356
+ hand_valid: torch.Tensor,
357
+ elapsed: torch.Tensor,
358
+ predicted_tokens: torch.Tensor,
359
+ mean_tokens: torch.Tensor,
360
+ *,
361
+ coefficient_override: torch.Tensor | None = None,
362
+ ) -> tuple[torch.Tensor, torch.Tensor]:
363
+ del predicted_tokens # Predicted innovation is deliberately outside this ablation.
364
+ action = self.action_projection(
365
+ (anchor.detach() - self.action_mean) / self.action_std
366
+ )
367
+ action = (
368
+ action
369
+ + self.position
370
+ + self.time_projection(elapsed.float()[:, None, None] * 10)
371
+ )
372
+ tactile = self._masked_dual_hand(tokens, hand_valid)
373
+ hidden = self.trunk(action, self.touch_condition(tactile))
374
+
375
+ if self.uses_basis_output:
376
+ bases = self.basis_head(hidden).reshape(
377
+ len(anchor), 16, self.residual_bases, 192
378
+ )
379
+ bases = bases.permute(0, 2, 1, 3)
380
+ coefficients = (
381
+ self.tactile_coefficients(tokens, hand_valid, mean_tokens)
382
+ if coefficient_override is None
383
+ else coefficient_override
384
+ )
385
+ expected = (len(anchor), self.residual_bases)
386
+ if tuple(coefficients.shape) != expected:
387
+ raise ValueError(
388
+ f"coefficients must have shape {expected}, got {tuple(coefficients.shape)}"
389
+ )
390
+ # Explicit broadcast avoids a version-specific compiled einsum
391
+ # backward while preserving the exact weighted-basis definition.
392
+ raw_delta = (coefficients[:, :, None, None] * bases).sum(1)
393
+ raw_delta = raw_delta / math.sqrt(self.residual_bases)
394
+ else:
395
+ raw_delta = self.output(hidden)
396
+
397
+ valid_sample = hand_valid.any(1)[:, None, None]
398
+ delta = raw_delta * mask.to(raw_delta.dtype) * valid_sample.to(raw_delta.dtype)
399
+ return anchor.detach() + delta, delta
400
+
401
+
402
+ def build_architecture_editor(
403
+ architecture: str,
404
+ action_mean: torch.Tensor,
405
+ action_std: torch.Tensor,
406
+ *,
407
+ hidden: int = 128,
408
+ residual_bases: int = 8,
409
+ ) -> nn.Module:
410
+ """Construct an ablation editor without changing the production defaults."""
411
+ if architecture not in ARCHITECTURES:
412
+ raise ValueError(f"unknown architecture: {architecture}")
413
+ if architecture == "e0_cross_attention":
414
+ return IncrementalTouchEditor(
415
+ action_mean, action_std, StudySetting("frozen"), hidden=hidden
416
+ )
417
+ return AdaLNZeroTouchEditor(
418
+ action_mean,
419
+ action_std,
420
+ architecture,
421
+ hidden=hidden,
422
+ residual_bases=residual_bases,
423
+ )
model/tachin_paired30_cache_20260923/code/src/tactile_editor/auxiliary_objectives.py ADDED
@@ -0,0 +1,264 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Leakage-safe auxiliary objectives for the direct tactile residual editor.
2
+
3
+ These modules are training-only. They consume causal features already emitted
4
+ by ``DirectTactileResidualEditor``; neither module adds an inference-time input
5
+ to the editor.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import math
11
+
12
+ import torch
13
+ from torch import nn
14
+ from torch.nn import functional as F
15
+
16
+
17
+ LEFT_ARTICULATION = (6, 69)
18
+ RIGHT_ARTICULATION = (75, 138)
19
+
20
+
21
+ def active_hands_from_edit_mask(edit_mask: torch.Tensor) -> torch.Tensor:
22
+ """Return ``[B,2]`` left/right articulation supervision flags."""
23
+
24
+ if edit_mask.ndim != 3 or edit_mask.shape[-1] < RIGHT_ARTICULATION[1]:
25
+ raise ValueError("edit_mask must be [B,T,A] with A >= 138")
26
+ left = (
27
+ edit_mask[..., LEFT_ARTICULATION[0] : LEFT_ARTICULATION[1]].bool().any((1, 2))
28
+ )
29
+ right = (
30
+ edit_mask[..., RIGHT_ARTICULATION[0] : RIGHT_ARTICULATION[1]].bool().any((1, 2))
31
+ )
32
+ return torch.stack((left, right), dim=1)
33
+
34
+
35
+ class ActiveHandTouchResidualInfoNCE(nn.Module):
36
+ """Align causal touch with ``target_action - anchor`` using hard groups.
37
+
38
+ A negative participates only when it has the same tactile decision tick and
39
+ active-hand code as the query. Rows in a group of size one are omitted. The
40
+ tactile projection receives only features from hands whose articulation is
41
+ editable, which prevents the inactive sensor from becoming a shortcut.
42
+ """
43
+
44
+ def __init__(
45
+ self,
46
+ *,
47
+ tactile_feature_dim: int = 128,
48
+ action_horizon: int = 16,
49
+ hidden_dim: int = 128,
50
+ embedding_dim: int = 64,
51
+ temperature: float = 0.1,
52
+ tactile_hz: float = 30.0,
53
+ ) -> None:
54
+ super().__init__()
55
+ if tactile_feature_dim < 1 or action_horizon < 1:
56
+ raise ValueError("feature dimension and action horizon must be positive")
57
+ if hidden_dim < 1 or embedding_dim < 1:
58
+ raise ValueError("hidden and embedding dimensions must be positive")
59
+ if not math.isfinite(temperature) or temperature <= 0:
60
+ raise ValueError("temperature must be finite and positive")
61
+ if not math.isfinite(tactile_hz) or tactile_hz <= 0:
62
+ raise ValueError("tactile_hz must be finite and positive")
63
+ articulation_dim = (
64
+ LEFT_ARTICULATION[1]
65
+ - LEFT_ARTICULATION[0]
66
+ + RIGHT_ARTICULATION[1]
67
+ - RIGHT_ARTICULATION[0]
68
+ )
69
+ self.tactile_feature_dim = int(tactile_feature_dim)
70
+ self.action_horizon = int(action_horizon)
71
+ self.temperature = float(temperature)
72
+ self.tactile_hz = float(tactile_hz)
73
+ self.touch_projection = nn.Sequential(
74
+ nn.LayerNorm(2 * tactile_feature_dim),
75
+ nn.Linear(2 * tactile_feature_dim, hidden_dim),
76
+ nn.GELU(),
77
+ nn.Linear(hidden_dim, embedding_dim),
78
+ )
79
+ self.residual_projection = nn.Sequential(
80
+ nn.LayerNorm(action_horizon * articulation_dim),
81
+ nn.Linear(action_horizon * articulation_dim, hidden_dim),
82
+ nn.GELU(),
83
+ nn.Linear(hidden_dim, embedding_dim),
84
+ )
85
+
86
+ def forward(
87
+ self,
88
+ tactile_features: torch.Tensor,
89
+ target_action: torch.Tensor,
90
+ anchor: torch.Tensor,
91
+ edit_mask: torch.Tensor,
92
+ decision_seconds: torch.Tensor,
93
+ ) -> torch.Tensor:
94
+ """Return sample-weighted symmetric InfoNCE over eligible groups."""
95
+
96
+ batch = len(tactile_features)
97
+ if tactile_features.shape != (batch, 2, self.tactile_feature_dim):
98
+ raise ValueError(
99
+ f"tactile_features must be [B,2,{self.tactile_feature_dim}]"
100
+ )
101
+ if target_action.shape != anchor.shape or anchor.ndim != 3:
102
+ raise ValueError(
103
+ "target_action and anchor must have matching [B,T,A] shapes"
104
+ )
105
+ if anchor.shape[:2] != (batch, self.action_horizon):
106
+ raise ValueError(
107
+ f"target_action and anchor must be [B,{self.action_horizon},A]"
108
+ )
109
+ if edit_mask.shape != anchor.shape:
110
+ raise ValueError("edit_mask must match the action tensors")
111
+ if decision_seconds.shape != (batch,):
112
+ raise ValueError("decision_seconds must be [B]")
113
+ if not torch.isfinite(decision_seconds).all():
114
+ raise ValueError("decision_seconds must be finite")
115
+
116
+ active_hands = active_hands_from_edit_mask(edit_mask)
117
+ active_code = active_hands[:, 0].long() + 2 * active_hands[:, 1].long()
118
+
119
+ touch = tactile_features * active_hands[..., None].to(tactile_features.dtype)
120
+ touch_embedding = F.normalize(self.touch_projection(touch.flatten(1)), dim=-1)
121
+
122
+ left_mask = edit_mask[..., LEFT_ARTICULATION[0] : LEFT_ARTICULATION[1]].bool()
123
+ right_mask = edit_mask[
124
+ ..., RIGHT_ARTICULATION[0] : RIGHT_ARTICULATION[1]
125
+ ].bool()
126
+ articulation_mask = torch.cat((left_mask, right_mask), dim=-1)
127
+ if not torch.isfinite(anchor).all():
128
+ raise ValueError("anchor must be finite")
129
+ if not torch.isfinite(target_action[edit_mask.bool()]).all():
130
+ raise ValueError("editable target actions must be finite")
131
+ safe_target = torch.where(edit_mask.bool(), target_action, anchor.detach())
132
+ target_residual = safe_target - anchor.detach()
133
+ articulation_residual = torch.cat(
134
+ (
135
+ target_residual[..., LEFT_ARTICULATION[0] : LEFT_ARTICULATION[1]],
136
+ target_residual[..., RIGHT_ARTICULATION[0] : RIGHT_ARTICULATION[1]],
137
+ ),
138
+ dim=-1,
139
+ ).masked_fill(~articulation_mask, 0)
140
+ residual_embedding = F.normalize(
141
+ self.residual_projection(articulation_residual.flatten(1)), dim=-1
142
+ )
143
+
144
+ tick_float = decision_seconds.to(dtype=torch.float64) * self.tactile_hz
145
+ ticks = tick_float.round().long()
146
+ if not torch.allclose(
147
+ tick_float, ticks.to(tick_float.dtype), rtol=0, atol=1e-5
148
+ ):
149
+ raise ValueError("decision_seconds must lie on the tactile tick grid")
150
+
151
+ total = (touch_embedding.sum() + residual_embedding.sum()) * 0
152
+ eligible_samples = 0
153
+ group_keys = torch.stack((ticks, active_code), dim=1)
154
+ for key in torch.unique(group_keys, dim=0):
155
+ if int(key[1]) == 0:
156
+ continue
157
+ rows = (group_keys == key).all(dim=1).nonzero(as_tuple=True)[0]
158
+ count = int(rows.numel())
159
+ if count < 2:
160
+ continue
161
+ logits = (
162
+ touch_embedding.index_select(0, rows)
163
+ @ residual_embedding.index_select(0, rows).T
164
+ ) / self.temperature
165
+ labels = torch.arange(count, device=logits.device)
166
+ group_loss = 0.5 * (
167
+ F.cross_entropy(logits, labels, reduction="sum")
168
+ + F.cross_entropy(logits.T, labels, reduction="sum")
169
+ )
170
+ total = total + group_loss
171
+ eligible_samples += count
172
+ return total / max(eligible_samples, 1)
173
+
174
+
175
+ class CausalHistoryPressureDeltaLoss(nn.Module):
176
+ """Predict all seven next-frame pressure deltas in an 8-frame history.
177
+
178
+ For transition ``t -> t+1``, the prediction uses only the causal tactile
179
+ history feature at ``t``. Raw pressure at ``t+1`` is a detached supervision
180
+ target and is never passed to the prediction head.
181
+ """
182
+
183
+ def __init__(
184
+ self,
185
+ *,
186
+ feature_dim: int = 128,
187
+ history_length: int = 8,
188
+ grid_size: int = 21,
189
+ hidden_dim: int = 128,
190
+ target_scale: float = 0.01,
191
+ ) -> None:
192
+ super().__init__()
193
+ if feature_dim < 1 or history_length < 2 or grid_size < 1 or hidden_dim < 1:
194
+ raise ValueError(
195
+ "feature dimensions must be positive and history_length >= 2"
196
+ )
197
+ if not math.isfinite(target_scale) or target_scale <= 0:
198
+ raise ValueError("target_scale must be a finite positive train-only scale")
199
+ self.feature_dim = int(feature_dim)
200
+ self.history_length = int(history_length)
201
+ self.grid_size = int(grid_size)
202
+ self.target_scale = float(target_scale)
203
+ self.prediction_head = nn.Sequential(
204
+ nn.LayerNorm(feature_dim),
205
+ nn.Linear(feature_dim, hidden_dim),
206
+ nn.GELU(),
207
+ nn.Linear(hidden_dim, grid_size * grid_size),
208
+ )
209
+
210
+ def forward(
211
+ self,
212
+ history_features: torch.Tensor,
213
+ pressure: torch.Tensor,
214
+ temporal_valid: torch.Tensor,
215
+ edit_mask: torch.Tensor,
216
+ pressure_masks: torch.Tensor,
217
+ ) -> torch.Tensor:
218
+ """Return active-hand, valid-cell masked Smooth L1 over transitions."""
219
+
220
+ batch = len(history_features)
221
+ expected_features = (batch, 2, self.history_length, self.feature_dim)
222
+ expected_pressure = (
223
+ batch,
224
+ self.history_length,
225
+ 2,
226
+ self.grid_size,
227
+ self.grid_size,
228
+ )
229
+ if history_features.shape != expected_features:
230
+ raise ValueError(f"history_features must be {expected_features}")
231
+ if pressure.shape != expected_pressure:
232
+ raise ValueError(f"pressure must be {expected_pressure}")
233
+ if temporal_valid.shape != (batch, self.history_length, 2):
234
+ raise ValueError("temporal_valid must be [B,H,2]")
235
+ if pressure_masks.shape != (2, self.grid_size, self.grid_size):
236
+ raise ValueError("pressure_masks must be [2,G,G]")
237
+ if edit_mask.ndim != 3 or len(edit_mask) != batch:
238
+ raise ValueError("edit_mask must be [B,T,A]")
239
+
240
+ # Each output at t is computed only from the causal feature at t. The
241
+ # final feature has no target inside the cached history and is unused.
242
+ prediction = self.prediction_head(history_features[:, :, :-1])
243
+ prediction = prediction.reshape(
244
+ batch, 2, self.history_length - 1, self.grid_size, self.grid_size
245
+ ).permute(0, 2, 1, 3, 4)
246
+
247
+ canonical_pressure = torch.nan_to_num(pressure.float()).clone()
248
+ canonical_pressure[:, :, 0] = canonical_pressure[:, :, 0].flip(-1)
249
+ target = (
250
+ canonical_pressure[:, 1:] - canonical_pressure[:, :-1]
251
+ ) / self.target_scale
252
+
253
+ active_hands = active_hands_from_edit_mask(edit_mask)
254
+ pair_valid = temporal_valid[:, 1:].bool() & temporal_valid[:, :-1].bool()
255
+ valid_cells = (
256
+ pair_valid[..., None, None]
257
+ & active_hands[:, None, :, None, None]
258
+ & pressure_masks.bool()[None, None].to(device=pressure.device)
259
+ )
260
+ if not bool(valid_cells.any()):
261
+ return prediction.sum() * 0
262
+ safe_target = torch.where(valid_cells, target, prediction.detach())
263
+ loss = F.smooth_l1_loss(prediction, safe_target, reduction="none")
264
+ return loss.masked_fill(~valid_cells, 0).sum() / valid_cells.sum()
model/tachin_paired30_cache_20260923/code/src/tactile_editor/config.py ADDED
@@ -0,0 +1,160 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Configuration shared by all tactile-editor variants."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import math
7
+ import warnings
8
+ from dataclasses import asdict, dataclass, field
9
+ from enum import Enum
10
+ from pathlib import Path
11
+ from typing import Any
12
+
13
+
14
+ class Variant(str, Enum):
15
+ """The four controlled integration variants."""
16
+
17
+ V0_BASE = "v0_base"
18
+ V1_POSTHOC = "v1_posthoc_residual"
19
+ V2_EXPERT_TOUCH = "v2_expert_touch"
20
+ V3_MEMORY_TOUCH = "v3_memory_conditioned_touch"
21
+
22
+
23
+ class TrainMode(str, Enum):
24
+ """Parameter policies are independent of the architecture variant."""
25
+
26
+ ADAPTER_ONLY = "adapter_only"
27
+ ENCODER_EDITOR = "encoder_editor"
28
+ EXPERT_TOP = "expert_top"
29
+ ENCODER_TOP = "encoder_top"
30
+ JOINT = "joint_finetune"
31
+
32
+
33
+ class RuntimeMode(str, Enum):
34
+ """The runtime scheduler used for action deployment."""
35
+
36
+ PIPELINED = "pipelined"
37
+
38
+
39
+ @dataclass(frozen=True)
40
+ class EncoderConfig:
41
+ model_id: str = "MIT-Media-Lab/egotouch-tactile-encoder-p1-v1"
42
+ seed: int = 20260831
43
+ history_frames: int = 8
44
+ grid_size: int = 21
45
+ embedding_dim: int = 64
46
+ canonical_hand: str = "right"
47
+ frozen: bool = True
48
+
49
+
50
+ @dataclass(frozen=True)
51
+ class FusionConfig:
52
+ hidden_dim: int = 720
53
+ action_dim: int = 192
54
+ chunk_size: int = 16
55
+ num_heads: int = 8
56
+ injection_layers: tuple[int, ...] = (3, 7, 11, 15)
57
+ posthoc_hidden_dim: int = 256
58
+ posthoc_heads: int = 8
59
+ posthoc_layers: int = 2
60
+ posthoc_dropout: float = 0.1
61
+ posthoc_edit_gate_bias: float = -4.0
62
+ posthoc_touch_gate_init: float = 0.0
63
+ posthoc_use_touch: bool = True
64
+
65
+ def __post_init__(self) -> None:
66
+ if min(self.hidden_dim, self.action_dim, self.chunk_size) <= 0:
67
+ raise ValueError("fusion dimensions must be positive")
68
+ if self.hidden_dim % self.num_heads:
69
+ raise ValueError("hidden_dim must be divisible by num_heads")
70
+ if self.posthoc_hidden_dim % self.posthoc_heads:
71
+ raise ValueError("posthoc_hidden_dim must be divisible by posthoc_heads")
72
+ if not 0.0 <= self.posthoc_dropout < 1.0:
73
+ raise ValueError("posthoc_dropout must be in [0, 1)")
74
+ if not all(
75
+ math.isfinite(x)
76
+ for x in (self.posthoc_edit_gate_bias, self.posthoc_touch_gate_init)
77
+ ):
78
+ raise ValueError("gate initialization must be finite")
79
+ if not self.injection_layers:
80
+ raise ValueError("at least one expert injection layer is required")
81
+ if min(self.injection_layers) < 0:
82
+ raise ValueError("injection layer indices must be non-negative")
83
+
84
+
85
+ @dataclass(frozen=True)
86
+ class RuntimeConfig:
87
+ mode: RuntimeMode = RuntimeMode.PIPELINED
88
+ control_hz: float = 10.0
89
+ tactile_model_hz: float = 30.0
90
+ action_hz: float = 80.0
91
+ chunk_size: int = 16
92
+ executed_steps: int = 8
93
+ compute_latency_seconds: float = 0.0
94
+
95
+ def __post_init__(self) -> None:
96
+ object.__setattr__(self, "mode", RuntimeMode(self.mode))
97
+ if min(self.control_hz, self.tactile_model_hz, self.action_hz) <= 0:
98
+ raise ValueError("all runtime rates must be positive")
99
+ if not 0 < self.executed_steps <= self.chunk_size:
100
+ raise ValueError("executed_steps must be inside the action chunk")
101
+ if self.compute_latency_seconds < 0:
102
+ raise ValueError("compute_latency_seconds must be non-negative")
103
+ control_period = 1.0 / self.control_hz
104
+ execution_period = self.executed_steps / self.action_hz
105
+ if abs(control_period - execution_period) > 1e-9:
106
+ raise ValueError(
107
+ "executed_steps/action_hz must equal the VLA control period"
108
+ )
109
+
110
+
111
+ @dataclass(frozen=True)
112
+ class EditorConfig:
113
+ variant: Variant = Variant.V0_BASE
114
+ train_mode: TrainMode = TrainMode.ADAPTER_ONLY
115
+ encoder: EncoderConfig = field(default_factory=EncoderConfig)
116
+ fusion: FusionConfig = field(default_factory=FusionConfig)
117
+ runtime: RuntimeConfig = field(default_factory=RuntimeConfig)
118
+
119
+ @classmethod
120
+ def from_dict(cls, raw: dict[str, Any]) -> "EditorConfig":
121
+ fusion = dict(raw.get("fusion", {}))
122
+ legacy = {key for key in ("max_residual", "residual_bound") if key in fusion}
123
+ if legacy:
124
+ warnings.warn(
125
+ "Residual amplitude limits were removed. Ignoring legacy fields "
126
+ f"{sorted(legacy)}. Weight shapes remain compatible, but predictions "
127
+ "can differ from the bounded checkpoint. Re-evaluation is required.",
128
+ UserWarning,
129
+ stacklevel=2,
130
+ )
131
+ for key in legacy:
132
+ fusion.pop(key)
133
+ runtime = dict(raw.get("runtime", {}))
134
+ if "vision_hz" in runtime and "control_hz" not in runtime:
135
+ runtime["control_hz"] = runtime.pop("vision_hz")
136
+ return cls(
137
+ variant=Variant(raw.get("variant", Variant.V0_BASE)),
138
+ train_mode=TrainMode(raw.get("train_mode", TrainMode.ADAPTER_ONLY)),
139
+ encoder=EncoderConfig(**raw.get("encoder", {})),
140
+ fusion=FusionConfig(
141
+ **{
142
+ **fusion,
143
+ "injection_layers": tuple(
144
+ fusion.get("injection_layers", (3, 7, 11, 15))
145
+ ),
146
+ }
147
+ ),
148
+ runtime=RuntimeConfig(**runtime),
149
+ )
150
+
151
+ @classmethod
152
+ def from_json(cls, path: str | Path) -> "EditorConfig":
153
+ return cls.from_dict(json.loads(Path(path).read_text(encoding="utf-8")))
154
+
155
+ def to_dict(self) -> dict[str, Any]:
156
+ result = asdict(self)
157
+ result["variant"] = self.variant.value
158
+ result["train_mode"] = self.train_mode.value
159
+ result["runtime"]["mode"] = self.runtime.mode.value
160
+ return result
model/tachin_paired30_cache_20260923/code/src/tactile_editor/contact_filter.py ADDED
@@ -0,0 +1,91 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Contact-aware filtering and hand-level action masking for tactile editor data.
2
+
3
+ The selector uses the mean of the five largest masked pressure values for each
4
+ hand. A row is retained when at least one supervised hand is in contact. During
5
+ loading, no-contact hands are masked so the editor cannot change their action.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import bisect
11
+ import hashlib
12
+ import json
13
+ from pathlib import Path
14
+
15
+ import numpy as np
16
+ import torch
17
+ from torch.utils.data import Dataset
18
+
19
+
20
+ DEFAULT_CONTACT_THRESHOLD = 0.25
21
+
22
+
23
+ def contact_gate(
24
+ pressure: torch.Tensor,
25
+ valid: torch.Tensor,
26
+ pressure_masks: torch.Tensor,
27
+ *,
28
+ threshold: float = DEFAULT_CONTACT_THRESHOLD,
29
+ inclusive: bool = True,
30
+ ) -> torch.Tensor:
31
+ """Return ``[batch, 2]`` contact gates from pressure history.
32
+
33
+ ``pressure`` is ``[B,T,2,H,W]`` and ``valid`` is ``[B,T,2]``. The gate is
34
+ based on the maximum over valid history frames of the masked top-five mean.
35
+ """
36
+ values = pressure.float()
37
+ masks = pressure_masks.to(device=values.device, dtype=torch.bool)
38
+ values = values.masked_fill(~masks[None, None], float("-inf"))
39
+ top5 = values.flatten(-2).topk(k=5, dim=-1).values.mean(-1)
40
+ top5 = top5.masked_fill(~valid.to(device=values.device, dtype=torch.bool), float("-inf"))
41
+ score = top5.amax(dim=1)
42
+ return score.ge(float(threshold)) if inclusive else score.gt(float(threshold))
43
+
44
+
45
+ class ContactFilteredCache(Dataset):
46
+ """A view over ``UnifiedWorldCache`` with contact-aware row selection."""
47
+
48
+ def __init__(self, base, pressure_masks: torch.Tensor, indices: np.ndarray, *, threshold: float = DEFAULT_CONTACT_THRESHOLD):
49
+ self.base = base
50
+ self.pressure_masks = pressure_masks.bool().cpu()
51
+ self.selected = np.asarray(indices, dtype=np.int64)
52
+ self.threshold = float(threshold)
53
+
54
+ def __len__(self):
55
+ return len(self.selected)
56
+
57
+ def __getitem__(self, index):
58
+ row = self.base[int(self.selected[index])]
59
+ gate = contact_gate(
60
+ row["touch_pressure"].unsqueeze(0),
61
+ row["touch_valid"].unsqueeze(0),
62
+ self.pressure_masks,
63
+ threshold=self.threshold,
64
+ )[0]
65
+ row["touch_valid"] = row["touch_valid"] & gate.unsqueeze(0)
66
+ if "future_touch_valid" in row:
67
+ row["future_touch_valid"] = row["future_touch_valid"] & gate
68
+ row["action_mask"][:, :69] &= gate[0]
69
+ row["action_mask"][:, 69:138] &= gate[1]
70
+ return row
71
+
72
+ def identity(self):
73
+ identity = self.base.identity() if hasattr(self.base, "identity") else {
74
+ "cache_root": str(getattr(self.base, "cache_root", "unknown")),
75
+ "split": getattr(self.base, "split", "unknown"),
76
+ }
77
+ identity.update(
78
+ contact_threshold=self.threshold,
79
+ selected_rows=len(self),
80
+ selection_sha256=hashlib.sha256(self.selected.tobytes()).hexdigest(),
81
+ )
82
+ return identity
83
+
84
+ def read_column(self, name: str) -> np.ndarray:
85
+ """Read a column in filtered dataset order, preserving cache APIs."""
86
+ if hasattr(self.base, "read_column"):
87
+ values = self.base.read_column(name)
88
+ else:
89
+ values = np.asarray([self.base[int(i)][name].numpy() for i in self.selected])
90
+ return values
91
+ return np.asarray(values)[self.selected]
model/tachin_paired30_cache_20260923/code/src/tactile_editor/direct_residual.py ADDED
@@ -0,0 +1,327 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """End-to-end causal tactile-to-action-residual models.
2
+
3
+ The tactile trunk is trained only by downstream action supervision. It does not
4
+ use a tactile-to-pose retrieval objective. The action branch creates residual
5
+ bases without tactile conditioning; the tactile branch creates coefficients.
6
+ They meet only in the final weighted sum.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import math
12
+
13
+ import torch
14
+ from torch import nn
15
+
16
+ from .action_layout import (
17
+ ACTION_DIM,
18
+ ARTICULATION_GROUPS,
19
+ FULL_ACTION_GROUPS,
20
+ )
21
+
22
+ DIRECT_INPUT_MODES = ("current", "current_delta", "history_delta", "history")
23
+ COEFFICIENT_SCOPES = ("global", "hand_local")
24
+
25
+
26
+ def canonical_tactile_channels(
27
+ pressure: torch.Tensor,
28
+ temporal_valid: torch.Tensor,
29
+ pressure_masks: torch.Tensor,
30
+ *,
31
+ include_delta: bool,
32
+ compact: bool = False,
33
+ flip_left: bool = True,
34
+ ) -> tuple[torch.Tensor, torch.Tensor]:
35
+ """Return canonical pressure/difference/mask channels and latest validity."""
36
+
37
+ if pressure.ndim != 5 or tuple(pressure.shape[2:]) != (2, 21, 21):
38
+ raise ValueError("pressure must be [B,T,2,21,21]")
39
+ if tuple(temporal_valid.shape) != tuple(pressure.shape[:3]):
40
+ raise ValueError("temporal_valid must be [B,T,2]")
41
+ if tuple(pressure_masks.shape) != (2, 21, 21):
42
+ raise ValueError("pressure_masks must be [2,21,21]")
43
+
44
+ value = torch.nan_to_num(pressure.float()).clone()
45
+ # EgoTouch stores the left sensor in the opposite horizontal convention.
46
+ if flip_left:
47
+ value[:, :, 0] = value[:, :, 0].flip(-1)
48
+ valid = temporal_valid.bool()
49
+ spatial_valid = (
50
+ valid[..., None, None]
51
+ & pressure_masks.bool()[None, None].to(device=value.device)
52
+ )
53
+ value = value * spatial_valid.to(value.dtype)
54
+ difference = torch.zeros_like(value)
55
+ if include_delta and value.shape[1] > 1:
56
+ pair_valid = valid[:, 1:] & valid[:, :-1]
57
+ difference[:, 1:] = (
58
+ value[:, 1:] - value[:, :-1]
59
+ ) * pair_valid[..., None, None].to(value.dtype)
60
+ if compact:
61
+ channels = torch.stack((value, spatial_valid.to(value.dtype)), dim=3)
62
+ else:
63
+ channels = torch.stack(
64
+ (value, difference, spatial_valid.to(value.dtype)), dim=3
65
+ )
66
+ return channels, valid[:, -1]
67
+
68
+
69
+ class DirectTactileTrunk(nn.Module):
70
+ """Map raw causal tactile history directly to residual-basis coefficients."""
71
+
72
+ def __init__(
73
+ self,
74
+ pressure_masks: torch.Tensor,
75
+ reference_pressure: torch.Tensor,
76
+ *,
77
+ mode: str,
78
+ coefficient_scope: str = "global",
79
+ residual_bases: int = 8,
80
+ frame_dim: int = 128,
81
+ temporal_hidden: int = 128,
82
+ dropout: float = 0.1,
83
+ flip_left: bool = True,
84
+ ) -> None:
85
+ super().__init__()
86
+ if mode not in DIRECT_INPUT_MODES:
87
+ raise ValueError(f"unknown direct tactile input mode: {mode}")
88
+ if coefficient_scope not in COEFFICIENT_SCOPES:
89
+ raise ValueError(f"unknown coefficient scope: {coefficient_scope}")
90
+ if tuple(reference_pressure.shape) != (1, 8, 2, 21, 21):
91
+ raise ValueError("reference_pressure must be [1,8,2,21,21]")
92
+ if residual_bases < 1:
93
+ raise ValueError("residual_bases must be positive")
94
+ self.mode = mode
95
+ self.flip_left = flip_left
96
+ self.coefficient_scope = coefficient_scope
97
+ self.residual_bases = residual_bases
98
+ self.register_buffer("pressure_masks", pressure_masks.bool(), persistent=True)
99
+ self.register_buffer(
100
+ "reference_pressure", reference_pressure.float(), persistent=True
101
+ )
102
+ self.compact_channels = mode == "history"
103
+ input_channels = 2 if self.compact_channels else 3
104
+ self.frame_encoder = nn.Sequential(
105
+ nn.Conv2d(input_channels, 32, 5, padding=2),
106
+ nn.GELU(),
107
+ nn.MaxPool2d(2),
108
+ nn.Conv2d(32, 32, 5, padding=2),
109
+ nn.GELU(),
110
+ nn.MaxPool2d(2),
111
+ nn.Conv2d(32, 32, 5, padding=2),
112
+ nn.GELU(),
113
+ nn.MaxPool2d(2),
114
+ nn.Flatten(),
115
+ nn.Linear(128, frame_dim),
116
+ nn.GELU(),
117
+ )
118
+ self.temporal = (
119
+ nn.GRU(
120
+ frame_dim,
121
+ temporal_hidden,
122
+ num_layers=1,
123
+ batch_first=True,
124
+ )
125
+ if mode in {"history_delta", "history"}
126
+ else None
127
+ )
128
+ output_dim = temporal_hidden if self.temporal is not None else frame_dim
129
+ coefficient_input_dim = (
130
+ output_dim * 2 if coefficient_scope == "global" else output_dim
131
+ )
132
+ self.coefficient_head = nn.Sequential(
133
+ nn.LayerNorm(coefficient_input_dim),
134
+ nn.Dropout(dropout),
135
+ nn.Linear(coefficient_input_dim, residual_bases, bias=False),
136
+ )
137
+
138
+ def _encode(
139
+ self, pressure: torch.Tensor, temporal_valid: torch.Tensor
140
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
141
+ channels, latest_valid = canonical_tactile_channels(
142
+ pressure,
143
+ temporal_valid,
144
+ self.pressure_masks,
145
+ include_delta=self.mode in {"current_delta", "history_delta"},
146
+ compact=self.compact_channels,
147
+ flip_left=self.flip_left,
148
+ )
149
+ batch, history, hands = channels.shape[:3]
150
+ frames = self.frame_encoder(
151
+ channels.reshape(batch * history * hands, channels.shape[3], 21, 21)
152
+ )
153
+ frames = frames.reshape(batch, history, hands, -1).permute(0, 2, 1, 3)
154
+ frame_valid = temporal_valid.bool().permute(0, 2, 1)[..., None]
155
+ frames = frames * frame_valid.to(frames.dtype)
156
+ if self.temporal is None:
157
+ history_features = frames
158
+ encoded = history_features[:, :, -1]
159
+ else:
160
+ sequence = frames.reshape(batch * hands, history, -1)
161
+ self.temporal.flatten_parameters()
162
+ history_features = self.temporal(sequence)[0]
163
+ history_features = history_features.reshape(batch, hands, history, -1)
164
+ encoded = history_features[:, :, -1]
165
+ return encoded, latest_valid, history_features
166
+
167
+ def forward(
168
+ self, pressure: torch.Tensor, temporal_valid: torch.Tensor
169
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
170
+ batch = len(pressure)
171
+ reference_pressure = self.reference_pressure.to(device=pressure.device).expand(
172
+ batch, -1, -1, -1, -1
173
+ )
174
+ reference_valid = torch.ones(
175
+ batch,
176
+ self.reference_pressure.shape[1],
177
+ 2,
178
+ dtype=torch.bool,
179
+ device=pressure.device,
180
+ )
181
+ # Encode matched and reference inputs in one call. Identical tensors then
182
+ # produce bitwise-identical features even when convolution algorithms are
183
+ # sensitive to batch shape.
184
+ combined_pressure = torch.cat((pressure, reference_pressure), dim=0)
185
+ combined_valid = torch.cat((temporal_valid, reference_valid), dim=0)
186
+ combined, combined_hand_valid, combined_history = self._encode(
187
+ combined_pressure, combined_valid
188
+ )
189
+ encoded, reference = combined.split(batch, dim=0)
190
+ encoded_history, reference_history = combined_history.split(batch, dim=0)
191
+ hand_valid = combined_hand_valid[:batch]
192
+ centered = (encoded - reference).mul(hand_valid[..., None].to(encoded.dtype))
193
+ history_valid = temporal_valid.bool().permute(0, 2, 1)[..., None]
194
+ centered_history = (encoded_history - reference_history).mul(
195
+ history_valid.to(encoded_history.dtype)
196
+ )
197
+ if self.coefficient_scope == "global":
198
+ coefficients = self.coefficient_head(centered.flatten(1))
199
+ active = hand_valid.any(1, keepdim=True) & (
200
+ centered.abs().sum((1, 2), keepdim=False)[:, None] > 0
201
+ )
202
+ coefficients = coefficients * active.to(coefficients.dtype)
203
+ else:
204
+ coefficients = self.coefficient_head(centered)
205
+ active = hand_valid & (centered.abs().sum(-1) > 0)
206
+ coefficients = coefficients * active[..., None].to(coefficients.dtype)
207
+ return coefficients, hand_valid, centered, centered_history
208
+
209
+
210
+ class ActionResidualBasisTrunk(nn.Module):
211
+ """Create action-conditioned bases without access to tactile inputs."""
212
+
213
+ def __init__(
214
+ self,
215
+ action_mean: torch.Tensor,
216
+ action_std: torch.Tensor,
217
+ *,
218
+ residual_bases: int = 8,
219
+ hidden: int = 128,
220
+ dropout: float = 0.1,
221
+ ) -> None:
222
+ super().__init__()
223
+ if tuple(action_mean.shape) != (192,) or tuple(action_std.shape) != (192,):
224
+ raise ValueError("action statistics must have shape [192]")
225
+ self.residual_bases = residual_bases
226
+ self.register_buffer("action_mean", action_mean.float(), persistent=True)
227
+ self.register_buffer(
228
+ "action_std", action_std.float().clamp_min(1e-3), persistent=True
229
+ )
230
+ self.action_projection = nn.Linear(192, hidden)
231
+ self.position = nn.Parameter(torch.randn(1, 16, hidden) * 0.02)
232
+ self.time_projection = nn.Linear(1, hidden)
233
+ layer = nn.TransformerEncoderLayer(
234
+ hidden,
235
+ nhead=4,
236
+ dim_feedforward=hidden * 4,
237
+ dropout=dropout,
238
+ activation="gelu",
239
+ batch_first=True,
240
+ norm_first=True,
241
+ )
242
+ self.trunk = nn.TransformerEncoder(
243
+ layer, num_layers=1, enable_nested_tensor=False
244
+ )
245
+ self.basis_head = nn.Linear(hidden, residual_bases * 192)
246
+ nn.init.zeros_(self.basis_head.weight)
247
+ nn.init.zeros_(self.basis_head.bias)
248
+
249
+ def forward(self, anchor: torch.Tensor, elapsed: torch.Tensor) -> torch.Tensor:
250
+ if tuple(anchor.shape[1:]) != (16, 192):
251
+ raise ValueError("anchor must be [B,16,192]")
252
+ hidden = self.action_projection(
253
+ (anchor.detach() - self.action_mean) / self.action_std
254
+ )
255
+ hidden = hidden + self.position + self.time_projection(
256
+ elapsed.float()[:, None, None] * 10
257
+ )
258
+ hidden = self.trunk(hidden)
259
+ bases = self.basis_head(hidden).reshape(
260
+ len(anchor), 16, self.residual_bases, 192
261
+ )
262
+ return bases.permute(0, 2, 1, 3)
263
+
264
+
265
+ class DirectTactileResidualEditor(nn.Module):
266
+ """Predict a tactile-only low-rank correction to a frozen action anchor."""
267
+
268
+ def __init__(
269
+ self,
270
+ action_mean: torch.Tensor,
271
+ action_std: torch.Tensor,
272
+ pressure_masks: torch.Tensor,
273
+ reference_pressure: torch.Tensor,
274
+ *,
275
+ mode: str,
276
+ residual_bases: int = 8,
277
+ articulation_only: bool = True,
278
+ flip_left: bool = True,
279
+ ) -> None:
280
+ super().__init__()
281
+ self.tactile = DirectTactileTrunk(
282
+ pressure_masks,
283
+ reference_pressure,
284
+ mode=mode,
285
+ residual_bases=residual_bases,
286
+ flip_left=flip_left,
287
+ )
288
+ self.action = ActionResidualBasisTrunk(
289
+ action_mean, action_std, residual_bases=residual_bases
290
+ )
291
+ component_mask = torch.zeros(1, 1, ACTION_DIM, dtype=torch.bool)
292
+ groups = ARTICULATION_GROUPS if articulation_only else FULL_ACTION_GROUPS
293
+ for start, stop in groups:
294
+ component_mask[..., start:stop] = True
295
+ self.register_buffer("component_mask", component_mask, persistent=True)
296
+ self.residual_bases = residual_bases
297
+
298
+ def forward(
299
+ self,
300
+ anchor: torch.Tensor,
301
+ edit_mask: torch.Tensor,
302
+ pressure: torch.Tensor,
303
+ temporal_valid: torch.Tensor,
304
+ elapsed: torch.Tensor,
305
+ ) -> tuple[torch.Tensor, dict[str, torch.Tensor]]:
306
+ coefficients, hand_valid, tactile_features, tactile_history_features = self.tactile(
307
+ pressure, temporal_valid
308
+ )
309
+ bases = self.action(anchor, elapsed)
310
+ raw_delta = (
311
+ coefficients[:, :, None, None] * bases
312
+ ).sum(1) / math.sqrt(self.residual_bases)
313
+ allowed = edit_mask.bool() & self.component_mask
314
+ # The action layout is [left 69 | right 69 | padding].
315
+ allowed = allowed.clone()
316
+ allowed[..., :69] &= hand_valid[:, 0, None, None]
317
+ allowed[..., 69:138] &= hand_valid[:, 1, None, None]
318
+ delta = raw_delta * allowed.to(raw_delta.dtype)
319
+ return anchor.detach() + delta, {
320
+ "effective_delta": delta,
321
+ "coefficients": coefficients,
322
+ "bases": bases,
323
+ "allowed_mask": allowed,
324
+ "hand_valid": hand_valid,
325
+ "tactile_features": tactile_features,
326
+ "tactile_history_features": tactile_history_features,
327
+ }
model/tachin_paired30_cache_20260923/code/src/tactile_editor/editor_backends.py ADDED
@@ -0,0 +1,196 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Unified tactile-editor router and backend adapters."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Any
6
+ from dataclasses import replace
7
+
8
+ import torch
9
+ from torch import nn
10
+
11
+ from .editor_interface import BaseActionContext, EditorOutput
12
+ from .direct_residual import DirectTactileResidualEditor
13
+ from .unified_world import UnifiedWorldTactileEditor
14
+
15
+
16
+ class MemoryFreeTactileEditor(nn.Module):
17
+ """Action-conditioned tactile residual editor for models without memory."""
18
+
19
+ requires_memory = False
20
+
21
+ def __init__(self, editor: DirectTactileResidualEditor) -> None:
22
+ super().__init__()
23
+ self.editor = editor
24
+
25
+ def edit(
26
+ self,
27
+ context: BaseActionContext,
28
+ pressure: torch.Tensor,
29
+ temporal_valid: torch.Tensor,
30
+ decision_seconds: torch.Tensor,
31
+ **_: Any,
32
+ ) -> EditorOutput:
33
+ if context.has_memory:
34
+ raise ValueError("MemoryFreeTactileEditor cannot consume memory_tokens")
35
+ final_action, details = self.editor(
36
+ context.base_action,
37
+ context.effective_edit_mask,
38
+ pressure,
39
+ temporal_valid,
40
+ decision_seconds,
41
+ )
42
+ return EditorOutput(
43
+ final_action=final_action,
44
+ delta_action=details["effective_delta"],
45
+ allowed_mask=details["allowed_mask"],
46
+ backend="memory_free",
47
+ auxiliary_outputs=details,
48
+ )
49
+
50
+
51
+ class NativeFinalActionEditor(nn.Module):
52
+ """Pass through a model that already performed its native tactile update.
53
+
54
+ T-Rex (and eventually a native T-WLA predictor) returns a tactile-refined
55
+ action, rather than a vision-only anchor plus reusable memory tokens. The
56
+ pass-through keeps the common ``EditorOutput`` contract without applying a
57
+ second tactile residual editor.
58
+ """
59
+
60
+ requires_memory = False
61
+
62
+ def edit(
63
+ self,
64
+ context: BaseActionContext,
65
+ pressure: torch.Tensor,
66
+ temporal_valid: torch.Tensor,
67
+ decision_seconds: torch.Tensor,
68
+ **_: Any,
69
+ ) -> EditorOutput:
70
+ del pressure, temporal_valid, decision_seconds
71
+ return EditorOutput(
72
+ final_action=context.base_action,
73
+ delta_action=torch.zeros_like(context.base_action),
74
+ allowed_mask=context.effective_edit_mask,
75
+ backend="native_tactile",
76
+ auxiliary_outputs={"native_tactile_policy": True},
77
+ )
78
+
79
+
80
+ class MemoryW0W3Editor(nn.Module):
81
+ """Adapter exposing the existing memory-conditioned W0--W3 editor."""
82
+
83
+ requires_memory = True
84
+
85
+ def __init__(self, editor: UnifiedWorldTactileEditor) -> None:
86
+ super().__init__()
87
+ self.editor = editor
88
+
89
+ def edit(
90
+ self,
91
+ context: BaseActionContext,
92
+ pressure: torch.Tensor,
93
+ temporal_valid: torch.Tensor,
94
+ decision_seconds: torch.Tensor,
95
+ **kwargs: Any,
96
+ ) -> EditorOutput:
97
+ if not context.has_memory:
98
+ raise ValueError("MemoryW0W3Editor requires memory_tokens")
99
+ # Runtime action editing cannot consume future supervision. Training
100
+ # callers can explicitly enable auxiliary world predictions.
101
+ kwargs.setdefault("compute_auxiliary", False)
102
+ output = self.editor(
103
+ base_action=context.base_action,
104
+ edit_mask=context.effective_edit_mask,
105
+ pressure=pressure,
106
+ temporal_valid=temporal_valid,
107
+ decision_seconds=decision_seconds,
108
+ final_memory=context.memory_tokens,
109
+ image_tokens=context.image_tokens,
110
+ **kwargs,
111
+ )
112
+ auxiliary = {
113
+ "tactile_features": output.tactile_features,
114
+ "hand_valid": output.hand_valid,
115
+ "tactile_velocity": output.tactile_velocity,
116
+ "tactile_target": output.tactile_target,
117
+ "tactile_valid": output.tactile_valid,
118
+ "rgb": output.rgb,
119
+ }
120
+ return EditorOutput(
121
+ final_action=output.final_action,
122
+ delta_action=output.delta_action,
123
+ allowed_mask=output.allowed_mask,
124
+ backend=f"memory_{self.editor.world_mode.value}",
125
+ auxiliary_outputs=auxiliary,
126
+ )
127
+
128
+
129
+ class TactileEditorRouter(nn.Module):
130
+ """Select an explicit shared/memory experiment, or preserve legacy auto routing."""
131
+
132
+ def __init__(
133
+ self,
134
+ memory_backend: MemoryW0W3Editor | None,
135
+ memory_free_backend: MemoryFreeTactileEditor | None,
136
+ native_final_backend: NativeFinalActionEditor | None = None,
137
+ *,
138
+ route: str = "auto",
139
+ ) -> None:
140
+ super().__init__()
141
+ if memory_backend is None and memory_free_backend is None:
142
+ raise ValueError("at least one tactile-editor backend is required")
143
+ self.memory_backend = memory_backend
144
+ self.memory_free_backend = memory_free_backend
145
+ self.native_final_backend = native_final_backend or NativeFinalActionEditor()
146
+ if route not in ("auto", "shared", "memory_ablation"):
147
+ raise ValueError("route must be auto, shared, or memory_ablation")
148
+ self.route = route
149
+
150
+ def edit(
151
+ self,
152
+ context: BaseActionContext,
153
+ pressure: torch.Tensor,
154
+ temporal_valid: torch.Tensor,
155
+ decision_seconds: torch.Tensor,
156
+ **kwargs: Any,
157
+ ) -> EditorOutput:
158
+ if self.route == "shared":
159
+ if self.memory_free_backend is None:
160
+ raise ValueError("shared route requires a memory-free backend")
161
+ # The base retains its native memory. Only the editor loses access.
162
+ context = replace(context, memory_tokens=None, image_tokens=None)
163
+ return self.memory_free_backend.edit(
164
+ context, pressure, temporal_valid, decision_seconds, **kwargs
165
+ )
166
+ if self.route == "memory_ablation":
167
+ if context.model_id != "twla" or not context.has_memory or self.memory_backend is None:
168
+ raise ValueError("memory_ablation requires twla, memory tokens, and a memory backend")
169
+ return self.memory_backend.edit(
170
+ context, pressure, temporal_valid, decision_seconds, **kwargs
171
+ )
172
+ if context.metadata.get("action_is_final", False):
173
+ return self.native_final_backend.edit(
174
+ context, pressure, temporal_valid, decision_seconds, **kwargs
175
+ )
176
+ if context.has_memory:
177
+ if self.memory_backend is None:
178
+ raise ValueError("memory_tokens were supplied but no memory backend is configured")
179
+ return self.memory_backend.edit(
180
+ context, pressure, temporal_valid, decision_seconds, **kwargs
181
+ )
182
+ if self.memory_free_backend is None:
183
+ raise ValueError("no memory-free backend is configured for this context")
184
+ return self.memory_free_backend.edit(
185
+ context, pressure, temporal_valid, decision_seconds, **kwargs
186
+ )
187
+
188
+ def forward(
189
+ self,
190
+ context: BaseActionContext,
191
+ pressure: torch.Tensor,
192
+ temporal_valid: torch.Tensor,
193
+ decision_seconds: torch.Tensor,
194
+ **kwargs: Any,
195
+ ) -> EditorOutput:
196
+ return self.edit(context, pressure, temporal_valid, decision_seconds, **kwargs)
model/tachin_paired30_cache_20260923/code/src/tactile_editor/editor_interface.py ADDED
@@ -0,0 +1,96 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Shared model-to-tactile-editor interface.
2
+
3
+ VLA implementations differ in their internal memory representation. They all
4
+ expose the same normalized action context here, while editor backends decide
5
+ whether to use optional memory/image tokens.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from dataclasses import dataclass, field
11
+ from typing import Any, Protocol
12
+
13
+ import torch
14
+
15
+ from .action_layout import ACTION_DIM, CHUNK_SIZE, DEFAULT_ACTION_LAYOUT
16
+
17
+
18
+ @dataclass(frozen=True)
19
+ class BaseActionContext:
20
+ """Frozen output of a model-specific VLA adapter."""
21
+
22
+ base_action: torch.Tensor
23
+ action_mask: torch.Tensor
24
+ model_id: str
25
+ normalization_id: str
26
+ memory_tokens: torch.Tensor | None = None
27
+ image_tokens: torch.Tensor | None = None
28
+ edit_mask: torch.Tensor | None = None
29
+ metadata: dict[str, Any] = field(default_factory=dict)
30
+
31
+ def __post_init__(self) -> None:
32
+ DEFAULT_ACTION_LAYOUT.validate(self.base_action, name="base_action")
33
+ if self.action_mask.shape != self.base_action.shape:
34
+ raise ValueError("action_mask must match base_action")
35
+ if self.edit_mask is not None and self.edit_mask.shape != self.base_action.shape:
36
+ raise ValueError("edit_mask must match base_action")
37
+ if self.action_mask[..., 138:].any():
38
+ raise ValueError("action_mask cannot mark padding dimensions valid")
39
+ if self.edit_mask is not None and self.edit_mask[..., 138:].any():
40
+ raise ValueError("edit_mask cannot mark padding dimensions editable")
41
+ if not self.base_action.is_floating_point() or not torch.isfinite(self.base_action).all():
42
+ raise ValueError("base_action must be finite floating-point values")
43
+ if self.memory_tokens is not None:
44
+ if self.memory_tokens.ndim != 3 or self.memory_tokens.shape[0] != self.base_action.shape[0]:
45
+ raise ValueError("memory_tokens must be [B,N,D]")
46
+ if self.image_tokens is not None and self.image_tokens.shape[0] != self.base_action.shape[0]:
47
+ raise ValueError("image_tokens batch size must match base_action")
48
+ if not self.model_id or not self.normalization_id:
49
+ raise ValueError("model_id and normalization_id are required")
50
+
51
+ @property
52
+ def effective_edit_mask(self) -> torch.Tensor:
53
+ mask = self.action_mask.bool()
54
+ if self.edit_mask is not None:
55
+ mask = mask & self.edit_mask.bool()
56
+ return mask
57
+
58
+ @property
59
+ def has_memory(self) -> bool:
60
+ return self.memory_tokens is not None
61
+
62
+
63
+ @dataclass(frozen=True)
64
+ class EditorOutput:
65
+ """Common output of every tactile editor backend."""
66
+
67
+ final_action: torch.Tensor
68
+ delta_action: torch.Tensor
69
+ allowed_mask: torch.Tensor
70
+ backend: str
71
+ auxiliary_outputs: dict[str, Any] = field(default_factory=dict)
72
+
73
+ def __post_init__(self) -> None:
74
+ if self.final_action.shape != self.delta_action.shape:
75
+ raise ValueError("final_action and delta_action must match")
76
+ if self.allowed_mask.shape != self.final_action.shape:
77
+ raise ValueError("allowed_mask must match final_action")
78
+ DEFAULT_ACTION_LAYOUT.validate(self.final_action, name="final_action")
79
+ if not self.backend:
80
+ raise ValueError("backend name is required")
81
+
82
+
83
+ class TactileEditorBackend(Protocol):
84
+ """Structural protocol implemented by memory and memory-free backends."""
85
+
86
+ requires_memory: bool
87
+
88
+ def edit(
89
+ self,
90
+ context: BaseActionContext,
91
+ pressure: torch.Tensor,
92
+ temporal_valid: torch.Tensor,
93
+ decision_seconds: torch.Tensor,
94
+ **kwargs: Any,
95
+ ) -> EditorOutput: ...
96
+
model/tachin_paired30_cache_20260923/code/src/tactile_editor/encoder.py ADDED
@@ -0,0 +1,209 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """The stable bridge from two EgoTouch glove streams to two tactile tokens."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass
6
+ from pathlib import Path
7
+
8
+ import torch
9
+ from torch import nn
10
+
11
+
12
+ @dataclass
13
+ class TouchFeatures:
14
+ tokens: torch.Tensor # [B, 2, 64], left then right
15
+ valid: torch.Tensor # [B, 2]
16
+
17
+
18
+ def _selected_encoder_api():
19
+ try:
20
+ from selected_encoder import ( # type: ignore
21
+ DEFAULT_MODEL_ID,
22
+ DEFAULT_SEED,
23
+ SelectedTactileEncoder,
24
+ canonicalize_raw_touch,
25
+ )
26
+ except ImportError as exc: # pragma: no cover - exercised by packaging smoke
27
+ raise ImportError(
28
+ "Selected EgoTouch encoder is unavailable. Add the repository's Tactile "
29
+ "directory to PYTHONPATH before importing tactile_editor."
30
+ ) from exc
31
+ return (
32
+ DEFAULT_MODEL_ID,
33
+ DEFAULT_SEED,
34
+ SelectedTactileEncoder,
35
+ canonicalize_raw_touch,
36
+ )
37
+
38
+
39
+ def load_huggingface_encoder(
40
+ *,
41
+ model_id: str = "MIT-Media-Lab/egotouch-tactile-encoder-p1-v1",
42
+ seed: int = 20260831,
43
+ cache_dir: str | Path | None = None,
44
+ device: str | torch.device = "cpu",
45
+ history_frames: int = 8,
46
+ ) -> nn.Module:
47
+ """Download only the standalone tactile weights and masks, then load strictly."""
48
+
49
+ try:
50
+ from huggingface_hub import snapshot_download
51
+ except ImportError as exc: # pragma: no cover - optional network path
52
+ raise ImportError(
53
+ "huggingface_hub is required for remote encoder loading"
54
+ ) from exc
55
+ _, _, selected_cls, _ = _selected_encoder_api()
56
+ if cache_dir is not None:
57
+ Path(cache_dir).mkdir(parents=True, exist_ok=True)
58
+ root = snapshot_download(
59
+ repo_id=model_id,
60
+ cache_dir=str(cache_dir) if cache_dir is not None else None,
61
+ allow_patterns=[
62
+ "pressure_masks.npy",
63
+ f"seed-{seed}/tactile_encoder.safetensors",
64
+ ],
65
+ )
66
+ return selected_cls.from_pretrained_dir(
67
+ root, seed=seed, device=device, history_frames=history_frames
68
+ )
69
+
70
+
71
+ def load_formal_t2p_encoder(
72
+ checkpoint: str | Path,
73
+ *,
74
+ history_frames: int,
75
+ device: str | torch.device = "cpu",
76
+ ) -> nn.Module:
77
+ """Load only the tactile tower from a formal P1 T2P checkpoint.
78
+
79
+ The pose tower and contrastive temperature are training-only components.
80
+ The returned module has the same normalized 64-D runtime contract as the
81
+ published selected encoder.
82
+ """
83
+
84
+ _, _, selected_cls, _ = _selected_encoder_api()
85
+ from egotouch_formal.model import TactileEncoder
86
+
87
+ payload = torch.load(checkpoint, map_location="cpu", weights_only=False)
88
+ config = payload.get("config", {})
89
+ if config.get("task") != "p1":
90
+ raise ValueError(f"formal checkpoint must be P1, got {config.get('task')!r}")
91
+ trained_touch_history = int(config.get("tactile_history_frames", 8))
92
+ if trained_touch_history != int(history_frames):
93
+ raise ValueError(
94
+ f"checkpoint history is {trained_touch_history}, requested {history_frames}"
95
+ )
96
+ source = payload.get("model")
97
+ if not isinstance(source, dict):
98
+ raise ValueError("formal checkpoint has no model state")
99
+ tactile_state = {
100
+ key.removeprefix("tactile."): value
101
+ for key, value in source.items()
102
+ if key.startswith("tactile.")
103
+ }
104
+ encoder = TactileEncoder(embed_dim=int(config.get("embed_dim", 64)))
105
+ encoder.load_state_dict(tactile_state, strict=True)
106
+ pressure_masks = source.get("pressure_masks")
107
+ if pressure_masks is None:
108
+ raise ValueError("formal checkpoint does not contain pressure_masks")
109
+ return selected_cls(
110
+ encoder,
111
+ pressure_masks,
112
+ history_frames=history_frames,
113
+ ).to(device)
114
+
115
+
116
+ def initialize_random_t2p_encoder(
117
+ pressure_masks: torch.Tensor,
118
+ *,
119
+ history_frames: int,
120
+ device: str | torch.device = "cpu",
121
+ ) -> nn.Module:
122
+ """Construct the P1 tactile architecture without pretrained weights."""
123
+
124
+ _, _, selected_cls, _ = _selected_encoder_api()
125
+ from egotouch_formal.model import TactileEncoder
126
+
127
+ return selected_cls(
128
+ TactileEncoder(embed_dim=64),
129
+ pressure_masks,
130
+ history_frames=history_frames,
131
+ ).to(device)
132
+
133
+
134
+ class DualHandTactileEncoder(nn.Module):
135
+ """Run one shared encoder over left/right causal histories.
136
+
137
+ Raw input keeps the online cache contract ``[B,8,2,21,21]``. The wrapped
138
+ checkpoint sees the latest ``history_frames`` as ``[B*2,T,21,21]`` plus a
139
+ side id. Left pressure is flipped exactly once when ``canonicalized=False``.
140
+ """
141
+
142
+ def __init__(self, encoder: nn.Module, *, freeze: bool = True) -> None:
143
+ super().__init__()
144
+ self.encoder = encoder
145
+ self.history_frames = int(getattr(encoder, "history_frames", 8))
146
+ if not 1 <= self.history_frames <= 8:
147
+ raise ValueError("editor bridge supports history_frames in [1,8]")
148
+ self.set_frozen(freeze)
149
+
150
+ def set_frozen(self, frozen: bool) -> None:
151
+ for parameter in self.encoder.parameters():
152
+ parameter.requires_grad = not frozen
153
+ if frozen:
154
+ self.encoder.eval()
155
+
156
+ def train(self, mode: bool = True):
157
+ super().train(mode)
158
+ if not any(parameter.requires_grad for parameter in self.encoder.parameters()):
159
+ self.encoder.eval()
160
+ return self
161
+
162
+ def forward(
163
+ self,
164
+ pressure: torch.Tensor,
165
+ temporal_valid: torch.Tensor | None = None,
166
+ *,
167
+ canonicalized: bool = False,
168
+ ) -> TouchFeatures:
169
+ expected = (8, 2, 21, 21)
170
+ if pressure.ndim != 5 or tuple(pressure.shape[1:]) != expected:
171
+ raise ValueError(
172
+ f"pressure must be [B,8,2,21,21], got {tuple(pressure.shape)}"
173
+ )
174
+ b = pressure.shape[0]
175
+ if temporal_valid is None:
176
+ temporal_valid = torch.ones(
177
+ b, 8, 2, dtype=torch.bool, device=pressure.device
178
+ )
179
+ if tuple(temporal_valid.shape) != (b, 8, 2):
180
+ raise ValueError(
181
+ f"temporal_valid must be {(b, 8, 2)}, got {tuple(temporal_valid.shape)}"
182
+ )
183
+
184
+ # Keep the most recent causal frames. [B,T,H,W] for each hand becomes
185
+ # [B*2,T,H,W], ordered L0,R0,L1,R1,...
186
+ pressure = pressure[:, -self.history_frames :]
187
+ temporal_valid = temporal_valid[:, -self.history_frames :]
188
+ flat = pressure.permute(0, 2, 1, 3, 4).reshape(
189
+ b * 2, self.history_frames, 21, 21
190
+ )
191
+ valid_flat = temporal_valid.permute(0, 2, 1).reshape(
192
+ b * 2, self.history_frames
193
+ )
194
+ flat = flat * valid_flat[:, :, None, None].to(flat.dtype)
195
+ side = torch.arange(2, device=pressure.device).repeat(b)
196
+ if not canonicalized:
197
+ _, _, _, canonicalize = _selected_encoder_api()
198
+ flat = canonicalize(flat, side)
199
+ pressure_masks = getattr(self.encoder, "pressure_masks", None)
200
+ if pressure_masks is not None:
201
+ spatial_valid = pressure_masks[side.to(pressure_masks.device)]
202
+ flat = flat * spatial_valid[:, None].to(
203
+ device=flat.device, dtype=flat.dtype
204
+ )
205
+
206
+ tokens = self.encoder(flat, side).reshape(b, 2, -1)
207
+ hand_valid = valid_flat.any(dim=1).reshape(b, 2)
208
+ tokens = tokens * hand_valid.unsqueeze(-1).to(tokens.dtype)
209
+ return TouchFeatures(tokens=tokens, valid=hand_valid)
model/tachin_paired30_cache_20260923/code/src/tactile_editor/full_cache.py ADDED
@@ -0,0 +1,86 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Shared reader for the sharded full EgoTouch editor replay cache."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import bisect
6
+ import json
7
+ from pathlib import Path
8
+
9
+ import numpy as np
10
+ import torch
11
+ from torch.utils.data import Dataset
12
+
13
+
14
+ ARRAY_NAMES = (
15
+ "base_action",
16
+ "target_action",
17
+ "action_mask",
18
+ "touch_pressure",
19
+ "touch_valid",
20
+ "decision_seconds",
21
+ "recording",
22
+ )
23
+
24
+
25
+ class ShardedEditorDataset(Dataset):
26
+ """Memory-map complete replay shards without copying the full cache."""
27
+
28
+ def __init__(
29
+ self,
30
+ cache_root: str | Path,
31
+ split: str,
32
+ *,
33
+ expected_shards: int = 8,
34
+ ) -> None:
35
+ self.cache_root = Path(cache_root)
36
+ self.split = split
37
+ self.shards: list[tuple[int, dict[str, np.ndarray]]] = []
38
+ self.ends: list[int] = []
39
+ total = 0
40
+ for root in sorted(self.cache_root.glob("shard_*")):
41
+ metadata_path = root / split / "metadata.json"
42
+ if not metadata_path.is_file():
43
+ continue
44
+ metadata = json.loads(metadata_path.read_text())
45
+ count = int(metadata["valid_samples"])
46
+ arrays = {
47
+ name: np.load(root / split / f"{name}.npy", mmap_mode="r")
48
+ for name in ARRAY_NAMES
49
+ }
50
+ if any(len(value) < count for value in arrays.values()):
51
+ raise ValueError(f"shard arrays shorter than metadata: {root / split}")
52
+ self.shards.append((count, arrays))
53
+ total += count
54
+ self.ends.append(total)
55
+ if len(self.shards) != expected_shards:
56
+ raise RuntimeError(
57
+ f"expected {expected_shards} complete cache shards, found {len(self.shards)}"
58
+ )
59
+
60
+ def __len__(self) -> int:
61
+ return self.ends[-1]
62
+
63
+ def __getitem__(self, index: int) -> dict[str, torch.Tensor]:
64
+ if index < 0:
65
+ index += len(self)
66
+ if not 0 <= index < len(self):
67
+ raise IndexError(index)
68
+ shard = bisect.bisect_right(self.ends, index)
69
+ start = 0 if shard == 0 else self.ends[shard - 1]
70
+ row = index - start
71
+ arrays = self.shards[shard][1]
72
+ result = {
73
+ name: torch.from_numpy(np.array(value[row], copy=True))
74
+ for name, value in arrays.items()
75
+ }
76
+ result["index"] = torch.tensor(index, dtype=torch.long)
77
+ return result
78
+
79
+ def read_column(self, name: str) -> np.ndarray:
80
+ """Materialize one valid column in global dataset order."""
81
+
82
+ if name not in ARRAY_NAMES:
83
+ raise KeyError(name)
84
+ return np.concatenate(
85
+ [np.asarray(arrays[name][:count]) for count, arrays in self.shards], axis=0
86
+ )
model/tachin_paired30_cache_20260923/code/src/tactile_editor/fusion.py ADDED
@@ -0,0 +1,211 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """V1/V2/V3 fusion modules. V0 owns no tactile parameters."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass
6
+
7
+ import torch
8
+ from torch import nn
9
+
10
+ from .config import FusionConfig, Variant
11
+ from .encoder import TouchFeatures
12
+
13
+
14
+ def _safe_key_padding(valid: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
15
+ """Return an MHA mask without all-masked rows and the original row validity."""
16
+
17
+ row_valid = valid.any(dim=1)
18
+ key_padding = ~valid.to(torch.bool)
19
+ if bool((~row_valid).any()):
20
+ key_padding = key_padding.clone()
21
+ key_padding[~row_valid, 0] = False
22
+ return key_padding, row_valid
23
+
24
+
25
+ class GatedCrossAttention(nn.Module):
26
+ """Residual cross-attention that is an exact identity at initialization."""
27
+
28
+ def __init__(self, hidden_dim: int, num_heads: int) -> None:
29
+ super().__init__()
30
+ self.query_norm = nn.LayerNorm(hidden_dim)
31
+ self.memory_norm = nn.LayerNorm(hidden_dim)
32
+ self.attention = nn.MultiheadAttention(hidden_dim, num_heads, batch_first=True)
33
+ self.gate = nn.Parameter(torch.zeros(()))
34
+
35
+ def forward(
36
+ self, query: torch.Tensor, memory: torch.Tensor, memory_valid: torch.Tensor
37
+ ) -> torch.Tensor:
38
+ key_padding, row_valid = _safe_key_padding(memory_valid)
39
+ delta, _ = self.attention(
40
+ self.query_norm(query),
41
+ self.memory_norm(memory),
42
+ self.memory_norm(memory),
43
+ key_padding_mask=key_padding,
44
+ need_weights=False,
45
+ )
46
+ delta = delta * row_valid[:, None, None].to(delta.dtype)
47
+ return query + torch.tanh(self.gate) * delta
48
+
49
+
50
+ class PosthocResidualEditor(nn.Module):
51
+ """V1: correct a completed normalized action chunk with current touch."""
52
+
53
+ def __init__(self, cfg: FusionConfig, tactile_dim: int = 64) -> None:
54
+ super().__init__()
55
+ h = cfg.posthoc_hidden_dim
56
+ self.cfg = cfg
57
+ self.action_projection = nn.Linear(cfg.action_dim, h)
58
+ self.touch_projection = nn.Linear(tactile_dim, h)
59
+ self.hand_embedding = nn.Parameter(torch.zeros(1, 2, h))
60
+ nn.init.normal_(self.hand_embedding, std=0.02)
61
+ self.position = nn.Parameter(torch.zeros(1, cfg.chunk_size, h))
62
+ nn.init.normal_(self.position, std=0.02)
63
+ self.touch_attention = GatedCrossAttention(h, cfg.posthoc_heads)
64
+ nn.init.constant_(self.touch_attention.gate, cfg.posthoc_touch_gate_init)
65
+ layer = nn.TransformerEncoderLayer(
66
+ h,
67
+ cfg.posthoc_heads,
68
+ dim_feedforward=4 * h,
69
+ batch_first=True,
70
+ norm_first=True,
71
+ )
72
+ self.trunk = nn.TransformerEncoder(
73
+ layer, num_layers=cfg.posthoc_layers, enable_nested_tensor=False
74
+ )
75
+ self.delta_head = nn.Linear(h, cfg.action_dim)
76
+ self.edit_gate = nn.Linear(h, cfg.action_dim)
77
+ nn.init.zeros_(self.delta_head.weight)
78
+ nn.init.zeros_(self.delta_head.bias)
79
+ nn.init.zeros_(self.edit_gate.weight)
80
+ nn.init.constant_(self.edit_gate.bias, cfg.posthoc_edit_gate_bias)
81
+
82
+ def forward(
83
+ self,
84
+ base_action: torch.Tensor,
85
+ touch: TouchFeatures,
86
+ edit_mask: torch.Tensor,
87
+ ) -> tuple[torch.Tensor, dict[str, torch.Tensor]]:
88
+ expected = (self.cfg.chunk_size, self.cfg.action_dim)
89
+ if base_action.ndim != 3 or tuple(base_action.shape[1:]) != expected:
90
+ raise ValueError(f"base_action must be [B,{expected[0]},{expected[1]}]")
91
+ if edit_mask.shape != base_action.shape:
92
+ raise ValueError("edit_mask must have the same shape as base_action")
93
+ action = self.action_projection(base_action) + self.position
94
+ if self.cfg.posthoc_use_touch:
95
+ tactile = self.touch_projection(touch.tokens) + self.hand_embedding
96
+ action = self.touch_attention(action, tactile, touch.valid)
97
+ hidden = self.trunk(action)
98
+ raw_delta = self.delta_head(hidden)
99
+ gate = torch.sigmoid(self.edit_gate(hidden))
100
+ # The post-hoc editor is also the action-only control path when a touch
101
+ # frame is unavailable. ``GatedCrossAttention`` already returns the
102
+ # unchanged action stream for an all-invalid tactile row, so gating the
103
+ # residual a second time here would silently disable the control model
104
+ # and make missing-touch samples contribute no learning signal.
105
+ effective = raw_delta * gate * edit_mask.to(raw_delta.dtype)
106
+ return base_action + effective, {
107
+ "raw_delta": raw_delta,
108
+ "gate": gate,
109
+ "effective_delta": effective,
110
+ }
111
+
112
+
113
+ class CenteredArticulationTouchEditor(nn.Module):
114
+ """Legacy centered touch-only editor kept for prior ablation checkpoints."""
115
+
116
+ ARTICULATION_GROUPS = ((6, 69), (75, 138))
117
+
118
+ def __init__(self, cfg: FusionConfig, tactile_dim: int = 64) -> None:
119
+ super().__init__()
120
+ if cfg.action_dim < self.ARTICULATION_GROUPS[-1][1]:
121
+ raise ValueError("action_dim is too small for bimanual MANO articulation")
122
+ if getattr(cfg, "posthoc_dropout", 0.0) != 0.0:
123
+ raise ValueError("centered subtraction requires posthoc_dropout=0")
124
+ self.cfg = cfg
125
+ self.residual = PosthocResidualEditor(cfg, tactile_dim=tactile_dim)
126
+ component_mask = torch.zeros(1, 1, cfg.action_dim, dtype=torch.bool)
127
+ for start, stop in self.ARTICULATION_GROUPS:
128
+ component_mask[..., start:stop] = True
129
+ self.register_buffer("component_mask", component_mask, persistent=False)
130
+
131
+ def forward(
132
+ self,
133
+ calibrated_action: torch.Tensor,
134
+ touch: TouchFeatures,
135
+ edit_mask: torch.Tensor,
136
+ ) -> tuple[torch.Tensor, dict[str, torch.Tensor]]:
137
+ _, matched = self.residual(calibrated_action, touch, edit_mask)
138
+ reference = TouchFeatures(
139
+ tokens=torch.zeros_like(touch.tokens),
140
+ valid=torch.zeros_like(touch.valid),
141
+ )
142
+ _, action_only = self.residual(calibrated_action, reference, edit_mask)
143
+ has_touch = touch.valid.any(dim=1)[:, None, None]
144
+ allowed = edit_mask.bool() & self.component_mask & has_touch
145
+ touch_delta = matched["effective_delta"] - action_only["effective_delta"]
146
+ effective = touch_delta * allowed.to(touch_delta.dtype)
147
+ return calibrated_action + effective, {
148
+ "matched_delta": matched["effective_delta"],
149
+ "action_only_delta": action_only["effective_delta"],
150
+ "effective_delta": effective,
151
+ "allowed_mask": allowed,
152
+ }
153
+
154
+
155
+ @dataclass
156
+ class ExpertFusionContext:
157
+ touch: torch.Tensor
158
+ touch_valid: torch.Tensor
159
+
160
+
161
+ class ExpertTouchFusion(nn.Module):
162
+ """V2/V3: inject touch into selected layers of Method D's action stream."""
163
+
164
+ def __init__(
165
+ self, variant: Variant, cfg: FusionConfig, tactile_dim: int = 64
166
+ ) -> None:
167
+ super().__init__()
168
+ if variant not in {Variant.V2_EXPERT_TOUCH, Variant.V3_MEMORY_TOUCH}:
169
+ raise ValueError(f"expert fusion does not implement {variant.value}")
170
+ self.variant = variant
171
+ self.cfg = cfg
172
+ self.touch_projection = nn.Linear(tactile_dim, cfg.hidden_dim)
173
+ self.hand_embedding = nn.Parameter(torch.zeros(1, 2, cfg.hidden_dim))
174
+ nn.init.normal_(self.hand_embedding, std=0.02)
175
+ self.action_from_touch = GatedCrossAttention(cfg.hidden_dim, cfg.num_heads)
176
+ self.touch_from_memory = (
177
+ nn.MultiheadAttention(cfg.hidden_dim, cfg.num_heads, batch_first=True)
178
+ if variant == Variant.V3_MEMORY_TOUCH
179
+ else None
180
+ )
181
+ self.touch_memory_norm = nn.LayerNorm(cfg.hidden_dim)
182
+
183
+ def prepare(
184
+ self, touch: TouchFeatures, expert_memory: torch.Tensor
185
+ ) -> ExpertFusionContext:
186
+ tactile = self.touch_projection(touch.tokens) + self.hand_embedding
187
+ tactile = tactile * touch.valid.unsqueeze(-1).to(tactile.dtype)
188
+ if self.touch_from_memory is not None:
189
+ query = self.touch_memory_norm(tactile)
190
+ conditioned, _ = self.touch_from_memory(
191
+ query, expert_memory, expert_memory, need_weights=False
192
+ )
193
+ tactile = (tactile + conditioned) * touch.valid.unsqueeze(-1).to(
194
+ tactile.dtype
195
+ )
196
+ return ExpertFusionContext(touch=tactile, touch_valid=touch.valid)
197
+
198
+ def forward(
199
+ self, layer_index: int, action_hidden: torch.Tensor, ctx: ExpertFusionContext
200
+ ) -> torch.Tensor:
201
+ if layer_index not in self.cfg.injection_layers:
202
+ return action_hidden
203
+ return self.action_from_touch(action_hidden, ctx.touch, ctx.touch_valid)
204
+
205
+
206
+ def build_fusion(variant: Variant, cfg: FusionConfig) -> nn.Module | None:
207
+ if variant == Variant.V0_BASE:
208
+ return None
209
+ if variant == Variant.V1_POSTHOC:
210
+ return PosthocResidualEditor(cfg)
211
+ return ExpertTouchFusion(variant, cfg)
model/tachin_paired30_cache_20260923/code/src/tactile_editor/informative_data.py ADDED
@@ -0,0 +1,177 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Leakage-safe selection of samples with active-hand tactile events."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import torch
6
+
7
+ from tactile_editor.architecture_ablation import tactile_contact_statistics
8
+ from tactile_editor.training import build_future_action_mask
9
+
10
+
11
+ LEFT_ARTICULATION = (6, 69)
12
+ RIGHT_ARTICULATION = (75, 138)
13
+
14
+
15
+ def supervised_hands(
16
+ action_mask: torch.Tensor,
17
+ decision_seconds: torch.Tensor,
18
+ ) -> torch.Tensor:
19
+ """Return the hands that have editable articulation after each decision."""
20
+
21
+ future = build_future_action_mask(
22
+ action_mask.bool(), decision_seconds.to(dtype=torch.float64)
23
+ )
24
+ left = future[..., LEFT_ARTICULATION[0] : LEFT_ARTICULATION[1]].any((1, 2))
25
+ right = future[..., RIGHT_ARTICULATION[0] : RIGHT_ARTICULATION[1]].any((1, 2))
26
+ return torch.stack((left, right), dim=1).cpu()
27
+
28
+
29
+ def tactile_event_change_per_hand(
30
+ touch_pressure: torch.Tensor,
31
+ touch_valid: torch.Tensor,
32
+ pressure_masks: torch.Tensor,
33
+ ) -> tuple[torch.Tensor, torch.Tensor]:
34
+ """Return latest-pair pressure change and validity for each hand."""
35
+
36
+ if touch_pressure.ndim != 5 or tuple(touch_pressure.shape[1:]) != (8, 2, 21, 21):
37
+ raise ValueError("touch_pressure must be [N,8,2,21,21]")
38
+ if tuple(touch_valid.shape) != tuple(touch_pressure.shape[:3]):
39
+ raise ValueError("touch_valid must be [N,8,2]")
40
+ if tuple(pressure_masks.shape) != (2, 21, 21):
41
+ raise ValueError("pressure_masks must be [2,21,21]")
42
+ pressure = torch.nan_to_num(touch_pressure.detach().cpu().float()).clone()
43
+ pressure[:, :, 0] = pressure[:, :, 0].flip(-1)
44
+ masks = pressure_masks.detach().cpu().bool()
45
+ changes = []
46
+ for hand in range(2):
47
+ cells = pressure[:, -2:, hand, masks[hand]]
48
+ changes.append((cells[:, 1] - cells[:, 0]).abs().mean(-1))
49
+ pair_valid = touch_valid.detach().cpu().bool()[:, -2:].all(1)
50
+ change = torch.stack(changes, dim=1).masked_fill(~pair_valid, 0)
51
+ return change, pair_valid
52
+
53
+
54
+ def tactile_history_dynamics_per_hand(
55
+ touch_pressure: torch.Tensor,
56
+ touch_valid: torch.Tensor,
57
+ pressure_masks: torch.Tensor,
58
+ *,
59
+ contact_threshold: float = 0.1,
60
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
61
+ """Return change, transition, pair validity, and contact for each hand."""
62
+
63
+ if touch_pressure.ndim != 5 or tuple(touch_pressure.shape[1:]) != (8, 2, 21, 21):
64
+ raise ValueError("touch_pressure must be [N,8,2,21,21]")
65
+ if tuple(touch_valid.shape) != tuple(touch_pressure.shape[:3]):
66
+ raise ValueError("touch_valid must be [N,8,2]")
67
+ if tuple(pressure_masks.shape) != (2, 21, 21):
68
+ raise ValueError("pressure_masks must be [2,21,21]")
69
+ pressure = torch.nan_to_num(touch_pressure.detach().cpu().float()).clone()
70
+ pressure[:, :, 0] = pressure[:, :, 0].flip(-1)
71
+ valid = touch_valid.detach().cpu().bool()
72
+ masks = pressure_masks.detach().cpu().bool()
73
+ maximum_changes = []
74
+ transitions = []
75
+ has_pairs = []
76
+ has_contacts = []
77
+ for hand in range(2):
78
+ cells = pressure[:, :, hand, masks[hand]]
79
+ pair_valid = valid[:, 1:, hand] & valid[:, :-1, hand]
80
+ changes = (cells[:, 1:] - cells[:, :-1]).abs().mean(-1)
81
+ maximum = changes.masked_fill(~pair_valid, -torch.inf).max(1).values
82
+ has_pair = pair_valid.any(1)
83
+ maximum = maximum.masked_fill(~has_pair, 0)
84
+ contact = cells.topk(5, dim=-1).values.mean(-1) >= contact_threshold
85
+ transition = ((contact[:, 1:] != contact[:, :-1]) & pair_valid).any(1)
86
+ maximum_changes.append(maximum)
87
+ transitions.append(transition)
88
+ has_pairs.append(has_pair)
89
+ has_contacts.append((contact & valid[:, :, hand]).any(1))
90
+ return (
91
+ torch.stack(maximum_changes, dim=1),
92
+ torch.stack(transitions, dim=1),
93
+ torch.stack(has_pairs, dim=1),
94
+ torch.stack(has_contacts, dim=1),
95
+ )
96
+
97
+
98
+ def active_hand_selection(
99
+ touch_pressure: torch.Tensor,
100
+ touch_valid: torch.Tensor,
101
+ action_mask: torch.Tensor,
102
+ decision_seconds: torch.Tensor,
103
+ pressure_masks: torch.Tensor,
104
+ *,
105
+ pressure_change_threshold: float,
106
+ contact_threshold: float = 0.1,
107
+ ) -> dict[str, torch.Tensor]:
108
+ """Classify rows without reading the target action or its prediction error."""
109
+
110
+ active = supervised_hands(action_mask, decision_seconds)
111
+ contact = tactile_contact_statistics(
112
+ touch_pressure, touch_valid, pressure_masks
113
+ )
114
+ change, pair_valid = tactile_event_change_per_hand(
115
+ touch_pressure, touch_valid, pressure_masks
116
+ )
117
+ now_contact = contact["current_score_per_hand"] >= contact_threshold
118
+ previous_contact = contact["previous_score_per_hand"] >= contact_threshold
119
+ active_pair = active & pair_valid
120
+ transition_per_hand = (now_contact != previous_contact) & active_pair
121
+ changing_per_hand = (change >= pressure_change_threshold) & active_pair
122
+ dynamic_per_hand = transition_per_hand | changing_per_hand
123
+ contact_per_hand = (now_contact | previous_contact) & active_pair
124
+ supervised = active.any(1)
125
+ dynamic = dynamic_per_hand.any(1)
126
+ stable_contact = contact_per_hand.any(1) & ~dynamic
127
+ low_or_no_contact = active_pair.any(1) & ~contact_per_hand.any(1) & ~dynamic
128
+ return {
129
+ "supervised": supervised,
130
+ "active_hands": active,
131
+ "active_pair_valid": active_pair,
132
+ "active_event_change": change,
133
+ "active_transition_per_hand": transition_per_hand,
134
+ "active_changing_per_hand": changing_per_hand,
135
+ "active_dynamic": dynamic,
136
+ "active_stable_contact": stable_contact,
137
+ "active_low_or_no_contact": low_or_no_contact,
138
+ }
139
+
140
+
141
+ def active_hand_history_selection(
142
+ touch_pressure: torch.Tensor,
143
+ touch_valid: torch.Tensor,
144
+ action_mask: torch.Tensor,
145
+ decision_seconds: torch.Tensor,
146
+ pressure_masks: torch.Tensor,
147
+ *,
148
+ pressure_change_threshold: float,
149
+ contact_threshold: float = 0.1,
150
+ ) -> dict[str, torch.Tensor]:
151
+ """Classify rows from the full causal history of the supervised hand."""
152
+
153
+ active = supervised_hands(action_mask, decision_seconds)
154
+ maximum_change, transition, has_pair, has_contact = tactile_history_dynamics_per_hand(
155
+ touch_pressure,
156
+ touch_valid,
157
+ pressure_masks,
158
+ contact_threshold=contact_threshold,
159
+ )
160
+ active_pair = active & has_pair
161
+ changing = (maximum_change >= pressure_change_threshold) & active_pair
162
+ dynamic_per_hand = (transition & active_pair) | changing
163
+ dynamic = dynamic_per_hand.any(1)
164
+ contact_per_hand = has_contact & active_pair
165
+ stable_contact = contact_per_hand.any(1) & ~dynamic
166
+ low_or_no_contact = active_pair.any(1) & ~contact_per_hand.any(1) & ~dynamic
167
+ return {
168
+ "supervised": active.any(1),
169
+ "active_hands": active,
170
+ "active_pair_valid": active_pair,
171
+ "active_event_change": maximum_change,
172
+ "active_transition_per_hand": transition & active_pair,
173
+ "active_changing_per_hand": changing,
174
+ "active_dynamic": dynamic,
175
+ "active_stable_contact": stable_contact,
176
+ "active_low_or_no_contact": low_or_no_contact,
177
+ }
model/tachin_paired30_cache_20260923/code/src/tactile_editor/memoryvla_baseline.py ADDED
@@ -0,0 +1,104 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Restore human MemoryVLA policies or explicitly initialize a new human policy."""
2
+ from __future__ import annotations
3
+
4
+ import copy
5
+ import hashlib
6
+ from pathlib import Path
7
+
8
+ import torch
9
+
10
+
11
+ def tokenizer_fingerprint(directory):
12
+ """Identify the exact config and tokenizer assets used by the frozen VLM."""
13
+ root = Path(directory)
14
+ files = sorted(p for p in root.iterdir() if p.is_file() and
15
+ (p.name.startswith('tokenizer') or p.name in ('config.json', 'special_tokens_map.json')))
16
+ if not files or not (root / 'config.json').is_file():
17
+ raise FileNotFoundError(f"missing Llama config/tokenizer files in {root}")
18
+ return {p.name: hashlib.sha256(p.read_bytes()).hexdigest() for p in files}
19
+
20
+
21
+ def validate_action_stats(model):
22
+ if not bool(model.stats_loaded):
23
+ raise ValueError("MemoryVLA checkpoint has no human action normalization")
24
+ for name in ('action_mean', 'action_std'):
25
+ value = getattr(model, name)
26
+ if value.shape != (192,) or not torch.isfinite(value).all():
27
+ raise ValueError(f"invalid MemoryVLA {name}")
28
+ if (model.action_std <= 0).any():
29
+ raise ValueError("MemoryVLA action_std must be positive")
30
+
31
+
32
+ def initialize_memoryvla(pretrained_checkpoint, llama_directory, *, action_mean, action_std,
33
+ overrides=None, device='cpu', seed=0):
34
+ """Initialize human I/O from CogACT or native MemoryVLA, ready for base training.
35
+
36
+ New I/O weights have not learned human actions. This is not a Table 1 base.
37
+ Statistics must be supplied from human training data (synthetic only for smoke tests).
38
+ """
39
+ from ._baselines.memoryvla_official.config import from_json
40
+ from ._baselines.memoryvla_official.model import MemoryVLAModel, build_tokenizer
41
+ values = copy.deepcopy(overrides or {})
42
+ values.setdefault('vlm', {}).update(pretrained_checkpoint=str(pretrained_checkpoint),
43
+ llama_local_dir=str(llama_directory), freeze_vlm=True)
44
+ values.setdefault('data', {})['stats_path'] = 'provided://human-training-statistics'
45
+ cfg = from_json(values)
46
+ # Validate assets and prompt before allocating the large model.
47
+ assets = tokenizer_fingerprint(llama_directory)
48
+ tokenizer = build_tokenizer(cfg)
49
+ probe = tokenizer.prompt('Move the hand.')
50
+ expected = tokenizer.tok(probe.removesuffix(' </s>'), add_special_tokens=True).input_ids + [29871, 2]
51
+ if tokenizer.tok(probe, add_special_tokens=True).input_ids != expected:
52
+ raise ValueError('Llama tokenizer differs from the MemoryVLA EOS prompt convention')
53
+ with torch.random.fork_rng(devices=[]):
54
+ torch.manual_seed(seed)
55
+ model = MemoryVLAModel(cfg, load_weights=True)
56
+ model._table1_tokenizer = tokenizer
57
+ model.set_normalisation(action_mean=action_mean, action_std=action_std)
58
+ validate_action_stats(model)
59
+ model.trainable_params_setup()
60
+ model.to(device)
61
+ model._human_action_trained = False
62
+ info = {'model_id': 'memoryvla', 'layout': 'keypoints', 'human_action_trained': False,
63
+ 'initialization_seed': seed, 'pretrained_checkpoint': str(Path(pretrained_checkpoint).resolve()),
64
+ 'tokenizer_assets': assets, 'load_report': model.load_report}
65
+ return model, cfg, info
66
+
67
+
68
+ def load_memoryvla_human_checkpoint(checkpoint, *, pretrained_checkpoint, llama_directory,
69
+ device='cpu'):
70
+ """Restore collaborator checkpoints, including the separately stored frozen VLM.
71
+
72
+ The original VLM checkpoint must match the one used during base training.
73
+ All saved trainable parameters and normalization buffers must match exactly.
74
+ """
75
+ payload = torch.load(checkpoint, map_location='cpu', mmap=True, weights_only=False)
76
+ run = payload['config']
77
+ if run.get('model') != 'memoryvla_official':
78
+ raise ValueError('expected a collaborator memoryvla_official human checkpoint')
79
+ if payload.get('human_action_trained') is False:
80
+ raise ValueError('untrained human I/O cannot serve as a Table 1 baseline')
81
+ sd = payload['model']
82
+ if not bool(sd.get('stats_loaded', False)):
83
+ raise ValueError('checkpoint has no human action normalization')
84
+ model, cfg, info = initialize_memoryvla(
85
+ pretrained_checkpoint, llama_directory, action_mean=sd['action_mean'], action_std=sd['action_std'],
86
+ overrides=run.get('config', {}), device='cpu')
87
+ model.load_state_dict(sd, strict=True)
88
+ validate_action_stats(model)
89
+ for name in ('action_mean', 'action_std'):
90
+ if name in payload:
91
+ reference = torch.as_tensor(payload[name], dtype=torch.float32)
92
+ if not torch.isfinite(reference).all() or (name == 'action_std' and (reference < 0).any()):
93
+ raise ValueError(f'invalid top-level {name}')
94
+ # The trainer saves raw statistics above, but clamps constant channels
95
+ # to 1e-6 when installing the model buffers.
96
+ if name == 'action_std':
97
+ reference = reference.clamp_min(1e-6)
98
+ torch.testing.assert_close(getattr(model, name), reference, rtol=0, atol=0)
99
+ model.requires_grad_(False).eval().to(device)
100
+ model._human_action_trained = True
101
+ info.update(checkpoint=str(Path(checkpoint).resolve()), step=int(payload['step']),
102
+ human_action_trained=True, strict_load=True, frozen_vlm_external=True,
103
+ state_tensors=len(sd))
104
+ return model, cfg, info
model/tachin_paired30_cache_20260923/code/src/tactile_editor/model_adapters.py ADDED
@@ -0,0 +1,287 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Model adapters that normalize heterogeneous VLA outputs to BaseActionContext.
2
+
3
+ The adapters intentionally do not know dataset layouts. A dataset/inference
4
+ loader supplies the observation object; the adapter only calls the model and
5
+ normalizes its returned action/memory fields.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from collections.abc import Callable, Mapping
11
+ from dataclasses import dataclass, replace
12
+ from typing import Any
13
+
14
+ import torch
15
+
16
+ from .action_layout import ACTION_DIM, CHUNK_SIZE, DEFAULT_ACTION_LAYOUT
17
+ from .editor_interface import BaseActionContext
18
+
19
+
20
+ ActionPredictor = Callable[..., Any]
21
+
22
+
23
+ @dataclass(frozen=True)
24
+ class AdapterSpec:
25
+ """Output convention for one base model checkpoint."""
26
+
27
+ model_id: str
28
+ normalization_id: str
29
+ output_is_normalized: bool = True
30
+ action_mean: torch.Tensor | None = None
31
+ action_std: torch.Tensor | None = None
32
+ action_key: str = "base_action"
33
+ mask_key: str = "action_mask"
34
+ memory_key: str = "memory_tokens"
35
+ image_key: str = "image_tokens"
36
+
37
+ def __post_init__(self) -> None:
38
+ if not self.model_id or not self.normalization_id:
39
+ raise ValueError("model_id and normalization_id are required")
40
+ if not self.output_is_normalized:
41
+ if self.action_mean is None or self.action_std is None:
42
+ raise ValueError("raw action adapters require action_mean and action_std")
43
+ for name, value in (("action_mean", self.action_mean), ("action_std", self.action_std)):
44
+ if value is not None and tuple(value.shape) != (ACTION_DIM,):
45
+ raise ValueError(f"{name} must have shape [192]")
46
+ if self.action_std is not None and (self.action_std <= 0).any():
47
+ raise ValueError("action_std must be positive")
48
+
49
+
50
+ def _first(mapping: Mapping[str, Any], names: tuple[str, ...]) -> Any:
51
+ for name in names:
52
+ if name in mapping:
53
+ return mapping[name]
54
+ return None
55
+
56
+
57
+ class ModelAdapter:
58
+ """Call a model and convert its output to a validated BaseActionContext."""
59
+
60
+ ACTION_NAMES = ("base_action", "action", "actions", "action_chunk", "norm_action", "normalized_action")
61
+ MASK_NAMES = ("action_mask", "mask", "valid_mask")
62
+ MEMORY_NAMES = ("memory_tokens", "final_memory", "memory", "m_f")
63
+ IMAGE_NAMES = ("image_tokens", "vision_tokens", "image_features")
64
+
65
+ def __init__(
66
+ self,
67
+ model: Any,
68
+ spec: AdapterSpec,
69
+ *,
70
+ predictor: ActionPredictor | None = None,
71
+ action_converter: Callable[[torch.Tensor], torch.Tensor] | None = None,
72
+ method_names: tuple[str, ...] = (
73
+ "predict_base_action",
74
+ "sample_action_context",
75
+ "select_action_context",
76
+ "predict_action",
77
+ "sample_action",
78
+ "select_action",
79
+ ),
80
+ ) -> None:
81
+ self.model = model
82
+ self.spec = spec
83
+ self.predictor = predictor
84
+ self.action_converter = action_converter
85
+ self.method_names = method_names
86
+
87
+ def _predict(self, observation: Any, **model_kwargs: Any) -> Any:
88
+ if self.predictor is not None:
89
+ return self.predictor(observation, **model_kwargs)
90
+ if callable(self.model) and not any(hasattr(self.model, name) for name in self.method_names):
91
+ return self.model(observation, **model_kwargs)
92
+ for name in self.method_names:
93
+ method = getattr(self.model, name, None)
94
+ if method is not None:
95
+ return method(observation, **model_kwargs)
96
+ raise AttributeError(
97
+ f"{self.spec.model_id} adapter found no supported inference method; "
98
+ f"expected one of {self.method_names}"
99
+ )
100
+
101
+ def _unpack(self, result: Any) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor | None, dict[str, Any]]:
102
+ if isinstance(result, BaseActionContext):
103
+ return result.base_action, result.memory_tokens, result.image_tokens, {
104
+ "action_mask": result.action_mask,
105
+ "edit_mask": result.edit_mask,
106
+ "metadata": result.metadata,
107
+ }
108
+ if isinstance(result, Mapping):
109
+ action = _first(result, (self.spec.action_key, *self.ACTION_NAMES))
110
+ memory = _first(result, (self.spec.memory_key, *self.MEMORY_NAMES))
111
+ image = _first(result, (self.spec.image_key, *self.IMAGE_NAMES))
112
+ extras = dict(result)
113
+ return action, memory, image, extras
114
+ if isinstance(result, (tuple, list)):
115
+ if not result:
116
+ raise ValueError("model adapter received an empty output tuple")
117
+ action = result[0]
118
+ memory = result[1] if len(result) > 1 else None
119
+ image = result[2] if len(result) > 2 else None
120
+ return action, memory, image, {}
121
+ return result, None, None, {}
122
+
123
+ def __call__(
124
+ self,
125
+ observation: Any,
126
+ *,
127
+ action_mask: torch.Tensor | None = None,
128
+ edit_mask: torch.Tensor | None = None,
129
+ **model_kwargs: Any,
130
+ ) -> BaseActionContext:
131
+ result = self._predict(observation, **model_kwargs)
132
+ action, memory, image, extras = self._unpack(result)
133
+ if action is None:
134
+ raise ValueError(
135
+ f"{self.spec.model_id} predictor did not return an action; "
136
+ f"expected one of {(self.spec.action_key, *self.ACTION_NAMES)}"
137
+ )
138
+ if not isinstance(action, torch.Tensor):
139
+ action = torch.as_tensor(action)
140
+ if self.action_converter is not None:
141
+ action = self.action_converter(action)
142
+ if action.ndim != 3 or tuple(action.shape[1:]) != (CHUNK_SIZE, ACTION_DIM):
143
+ raise ValueError(
144
+ f"{self.spec.model_id} must emit [B,{CHUNK_SIZE},{ACTION_DIM}], "
145
+ f"got {tuple(action.shape)}; provide an explicit action converter"
146
+ )
147
+ action = action.float()
148
+ if not self.spec.output_is_normalized:
149
+ mean = self.spec.action_mean.to(action.device, action.dtype)
150
+ std = self.spec.action_std.to(action.device, action.dtype)
151
+ action = (action - mean) / std
152
+
153
+ supplied_mask = _first(extras, (self.spec.mask_key, *self.MASK_NAMES))
154
+ mask = action_mask if action_mask is not None else supplied_mask
155
+ if mask is None:
156
+ mask = DEFAULT_ACTION_LAYOUT.physical_mask(
157
+ tuple(action.shape), device=action.device
158
+ )
159
+ mask = mask.to(device=action.device, dtype=torch.bool)
160
+ supplied_edit = _first(extras, ("edit_mask", "future_mask", "editable_mask"))
161
+ if edit_mask is None:
162
+ edit_mask = supplied_edit
163
+ return BaseActionContext(
164
+ base_action=action,
165
+ action_mask=mask,
166
+ edit_mask=edit_mask,
167
+ model_id=self.spec.model_id,
168
+ normalization_id=self.spec.normalization_id,
169
+ memory_tokens=memory,
170
+ image_tokens=image,
171
+ metadata={"adapter_output": extras},
172
+ )
173
+
174
+
175
+ class SmolVLAAdapter(ModelAdapter):
176
+ """Adapter for the local V4-A/B/C/D SmolVLA-derived models."""
177
+
178
+ def __init__(self, model: Any, spec: AdapterSpec, **kwargs: Any) -> None:
179
+ super().__init__(model, spec, **kwargs)
180
+
181
+
182
+ class VITRAAdapter(ModelAdapter):
183
+ """Adapter for VITRA or VITRA student action predictors."""
184
+
185
+
186
+ class TRexAdapter(ModelAdapter):
187
+ """Adapter for the native T-Rex slow/fast predictor.
188
+
189
+ T-Rex does not expose the ``[B, 16, 192]`` hand-action contract used by
190
+ this repository. Its released inference server returns a denormalized
191
+ native chunk, normally ``[8, 62]`` or ``[16, 62]`` (31 dimensions per
192
+ arm: 9-D delta EEF pose + 22-D hand target). The Table 1 evaluator uses
193
+ a two-hand MANO action space, so converting that native output is a
194
+ task-specific operation and must be supplied through ``action_converter``.
195
+
196
+ The adapter nevertheless handles the complete interface around that
197
+ operation: unbatched server responses, the ``actions`` response key,
198
+ native-shape validation, optional raw-action normalization, and metadata
199
+ recording. If a predictor already emits the canonical action, no
200
+ converter is needed.
201
+ """
202
+
203
+ NATIVE_ACTION_DIMS = (29, 31, 58, 62)
204
+ NATIVE_HORIZONS = (8, 16)
205
+
206
+ def __init__(self, model: Any, spec: AdapterSpec, **kwargs: Any) -> None:
207
+ native_converter = kwargs.pop("action_converter", None)
208
+
209
+ def convert(action: torch.Tensor) -> torch.Tensor:
210
+ # The ZMQ server returns a single chunk as [T, D]; model adapters
211
+ # use a batch dimension everywhere else.
212
+ if action.ndim == 2:
213
+ action = action.unsqueeze(0)
214
+
215
+ if action.ndim == 3 and tuple(action.shape[1:]) == (CHUNK_SIZE, ACTION_DIM):
216
+ return action
217
+
218
+ native_shape = tuple(action.shape[-2:]) if action.ndim == 3 else None
219
+ if native_shape is None or native_shape[0] not in self.NATIVE_HORIZONS or native_shape[1] not in self.NATIVE_ACTION_DIMS:
220
+ raise ValueError(
221
+ "T-Rex predictor must emit canonical [B,16,192] or a "
222
+ "native [T,D] / [B,T,D] chunk with T in "
223
+ f"{self.NATIVE_HORIZONS} and D in {self.NATIVE_ACTION_DIMS}; "
224
+ f"got {tuple(action.shape)}"
225
+ )
226
+ if native_converter is None:
227
+ raise ValueError(
228
+ "T-Rex emits native EEF/hand actions "
229
+ f"{native_shape}; provide action_converter to map them to "
230
+ "the canonical [B,16,192] MANO action space"
231
+ )
232
+ converted = native_converter(action)
233
+ if not isinstance(converted, torch.Tensor):
234
+ converted = torch.as_tensor(converted, device=action.device)
235
+ if converted.ndim == 2:
236
+ converted = converted.unsqueeze(0)
237
+ return converted
238
+
239
+ super().__init__(model, spec, action_converter=convert, **kwargs)
240
+
241
+ def __call__(
242
+ self,
243
+ observation: Any,
244
+ *,
245
+ action_mask: torch.Tensor | None = None,
246
+ edit_mask: torch.Tensor | None = None,
247
+ ) -> BaseActionContext:
248
+ context = super().__call__(
249
+ observation, action_mask=action_mask, edit_mask=edit_mask
250
+ )
251
+ metadata = dict(context.metadata)
252
+ metadata.update(
253
+ {
254
+ "native_interface": "trex_slow_fast_cascaded_flow",
255
+ "native_action_dims": self.NATIVE_ACTION_DIMS,
256
+ "native_action_horizons": self.NATIVE_HORIZONS,
257
+ "has_external_memory_tokens": False,
258
+ "action_is_final": True,
259
+ }
260
+ )
261
+ # T-Rex's cached KV is an internal inference state, not a reusable
262
+ # memory-token tensor. Keeping it absent routes the model through the
263
+ # memory-free editor when a separate editor is explicitly requested.
264
+ return replace(context, memory_tokens=None, metadata=metadata)
265
+
266
+
267
+ class TWLAAdapter(ModelAdapter):
268
+ """Adapter for the local T-WLA/V4-D checkpoint family.
269
+
270
+ The current T-WLA checkpoint exposes ``select_action_context`` and returns
271
+ the canonical full action together with ``final_memory``/``m_f``. This is
272
+ intentionally a named adapter so Table 1 configuration can distinguish
273
+ T-WLA from the MemoryVLA comparison row while sharing the same validated
274
+ contract.
275
+ """
276
+
277
+
278
+ class MemoryVLAAdapter(SmolVLAAdapter):
279
+ """Adapter configuration for the official MemoryVLA baseline.
280
+
281
+ Official MemoryVLA checkpoints emit robot-native end-effector actions and
282
+ keep cognitive/perceptual memories inside the policy. They do not emit
283
+ the canonical human-MANO action or reusable ``[B,N,D]`` memory tensor.
284
+ Therefore this adapter must be constructed with a predictor that includes
285
+ the task-specific human action head; it must not silently reuse the local
286
+ V4-D/T-WLA checkpoint.
287
+ """
model/tachin_paired30_cache_20260923/code/src/tactile_editor/native_tachin.py ADDED
@@ -0,0 +1,201 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Native 15-region / 880-cell Tachin tactile editors.
2
+
3
+ This module deliberately does not reshape the glove into the unrelated
4
+ EgoTouch 21x21 grid. One shared spatial encoder is applied to each native
5
+ sensor region, followed by sensor and temporal attention.
6
+ """
7
+ from __future__ import annotations
8
+
9
+ from dataclasses import dataclass
10
+ import math
11
+
12
+ import torch
13
+ from torch import nn
14
+
15
+ from .action_layout import FULL_HAND_GROUPS
16
+ from .direct_residual import ActionResidualBasisTrunk
17
+ from .rgb_world_editor import FrozenRGBWorldResidual, RGBWorldResidualOutput
18
+ from .single_stage_world import (TactileWorldModel, gather_action_window,
19
+ tactile_flow_pair)
20
+ from .unified_world import DirectMemoryActionEditor
21
+
22
+
23
+ SENSOR_SHAPES = ((16, 10), (4, 8), (13, 8), (9, 8), (3, 8), (4, 8),
24
+ (9, 8), (3, 8), (4, 8), (9, 8), (3, 8), (4, 8),
25
+ (9, 8), (3, 8), (13, 8))
26
+ SENSOR_OFFSETS = (0, 160, 192, 296, 368, 392, 424, 496, 520, 552, 624,
27
+ 648, 680, 752, 776, 880)
28
+
29
+
30
+ class NativeTachinEncoder(nn.Module):
31
+ """Encode masked native cells into one 64-D token per hand."""
32
+
33
+ def __init__(self, pressure_mean: torch.Tensor, pressure_scale: torch.Tensor,
34
+ hidden_dim: int = 64) -> None:
35
+ super().__init__()
36
+ if tuple(pressure_mean.shape) != (2, 880) or tuple(pressure_scale.shape) != (2, 880):
37
+ raise ValueError("Tachin pressure statistics must be [2,880]")
38
+ self.register_buffer("pressure_mean", pressure_mean.float(), persistent=True)
39
+ self.register_buffer("pressure_scale", pressure_scale.float().clamp_min(1e-6), persistent=True)
40
+ self.spatial = nn.Sequential(
41
+ nn.Conv2d(2, 32, 3, padding=1), nn.GELU(),
42
+ nn.Conv2d(32, hidden_dim, 3, padding=1), nn.GELU(),
43
+ )
44
+ self.sensor_embedding = nn.Parameter(torch.randn(1, 15, hidden_dim) * 0.02)
45
+ sensor_layer = nn.TransformerEncoderLayer(hidden_dim, 4, hidden_dim * 4,
46
+ dropout=0.1, activation="gelu",
47
+ batch_first=True, norm_first=True)
48
+ self.sensor_blocks = nn.TransformerEncoder(sensor_layer, 1,
49
+ enable_nested_tensor=False)
50
+ self.temporal = nn.GRU(hidden_dim, hidden_dim, batch_first=True)
51
+ self.norm = nn.LayerNorm(hidden_dim)
52
+
53
+ def _encode(self, normalized: torch.Tensor, valid: torch.Tensor) -> torch.Tensor:
54
+ batch, history, hands = normalized.shape[:3]
55
+ sensor_tokens, sensor_valid = [], []
56
+ for sensor, ((height, width), start, stop) in enumerate(
57
+ zip(SENSOR_SHAPES, SENSOR_OFFSETS[:-1], SENSOR_OFFSETS[1:])):
58
+ values = normalized[..., start:stop].reshape(batch * history * hands, 1, height, width)
59
+ masks = valid[..., start:stop].reshape(batch * history * hands, 1, height, width)
60
+ features = self.spatial(torch.cat((values, masks.to(values.dtype)), 1))
61
+ denominator = masks.sum((2, 3)).clamp_min(1).to(features.dtype)
62
+ pooled = (features * masks.to(features.dtype)).sum((2, 3)) / denominator
63
+ sensor_tokens.append(pooled)
64
+ sensor_valid.append(masks.any((1, 2, 3)))
65
+ tokens = torch.stack(sensor_tokens, 1) + self.sensor_embedding
66
+ sensor_mask = torch.stack(sensor_valid, 1)
67
+ safe_mask = sensor_mask.clone()
68
+ safe_mask[~safe_mask.any(1), 0] = True
69
+ encoded = self.sensor_blocks(tokens, src_key_padding_mask=~safe_mask)
70
+ pooled = (encoded * safe_mask[..., None]).sum(1) / safe_mask.sum(1, keepdim=True).clamp_min(1)
71
+ pooled = pooled.reshape(batch, history, hands, -1).permute(0, 2, 1, 3)
72
+ sequence = pooled.reshape(batch * hands, history, -1)
73
+ self.temporal.flatten_parameters()
74
+ temporal = self.temporal(sequence)[0].reshape(batch, hands, history, -1)
75
+ return self.norm(temporal[:, :, -1])
76
+
77
+ def forward(self, pressure: torch.Tensor, cell_valid: torch.Tensor,
78
+ sample_valid: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
79
+ if tuple(pressure.shape[1:]) != (8, 2, 880):
80
+ raise ValueError("pressure must be [B,8,2,880]")
81
+ if cell_valid.shape != pressure.shape or sample_valid.shape != pressure.shape[:3]:
82
+ raise ValueError("Tachin tactile masks have the wrong shape")
83
+ normalized = (torch.nan_to_num(pressure.float()) - self.pressure_mean) / self.pressure_scale
84
+ valid = cell_valid.bool() & sample_valid.bool()[..., None]
85
+ normalized = normalized * valid.to(normalized.dtype)
86
+ # Center against an exact train-mean reference with the same missing-cell
87
+ # pattern. Zero tactile evidence therefore produces an exact zero token.
88
+ combined = self._encode(torch.cat((normalized, torch.zeros_like(normalized)), 0),
89
+ torch.cat((valid, valid), 0))
90
+ actual, reference = combined.chunk(2, 0)
91
+ hand_valid = sample_valid[:, -1].bool() & cell_valid[:, -1].any(-1)
92
+ output = (actual - reference) * hand_valid[..., None].to(actual.dtype)
93
+ return output, hand_valid
94
+
95
+
96
+ def allowed_mask(edit_mask: torch.Tensor, hand_valid: torch.Tensor) -> torch.Tensor:
97
+ allowed = edit_mask.bool().clone()
98
+ allowed[..., 138:] = False
99
+ allowed[..., :69] &= hand_valid[:, 0, None, None]
100
+ allowed[..., 69:138] &= hand_valid[:, 1, None, None]
101
+ return allowed
102
+
103
+
104
+ class NativeResidualEditor(nn.Module):
105
+ """The identical memory-free editor used for SmolVLA, VITRA, and T-Rex."""
106
+
107
+ def __init__(self, encoder: NativeTachinEncoder, residual_bases: int = 8) -> None:
108
+ super().__init__()
109
+ self.encoder = encoder
110
+ self.action = ActionResidualBasisTrunk(torch.zeros(192), torch.ones(192),
111
+ residual_bases=residual_bases)
112
+ self.coefficients = nn.Sequential(nn.LayerNorm(128), nn.Linear(128, residual_bases, bias=False))
113
+ self.residual_bases = residual_bases
114
+
115
+ def forward(self, base_action, edit_mask, pressure, cell_valid, sample_valid):
116
+ tactile, hand_valid = self.encoder(pressure, cell_valid, sample_valid)
117
+ coefficients = self.coefficients(tactile.flatten(1))
118
+ # LayerNorm has trainable affine parameters. Without an explicit
119
+ # centered-feature gate, its learned bias can create a global action
120
+ # correction even when pressure is exactly the train-mean reference.
121
+ # Match the EgoTouch editor contract: no tactile evidence means an
122
+ # exact frozen-base plan, including after training.
123
+ active = hand_valid.any(1, keepdim=True) & (
124
+ tactile.abs().sum((1, 2), keepdim=False)[:, None] > 0
125
+ )
126
+ coefficients = coefficients * active.to(coefficients.dtype)
127
+ bases = self.action(base_action, torch.zeros(len(base_action), device=base_action.device))
128
+ raw_delta = (coefficients[:, :, None, None] * bases).sum(1) / math.sqrt(self.residual_bases)
129
+ allowed = allowed_mask(edit_mask, hand_valid)
130
+ delta = raw_delta * allowed.to(raw_delta.dtype)
131
+ return base_action.detach() + delta, delta, allowed, tactile, hand_valid
132
+
133
+
134
+ @dataclass
135
+ class NativeW3Output:
136
+ final_action: torch.Tensor
137
+ delta_action: torch.Tensor
138
+ allowed_mask: torch.Tensor
139
+ tactile_features: torch.Tensor
140
+ hand_valid: torch.Tensor
141
+ tactile_velocity: torch.Tensor | None = None
142
+ tactile_target: torch.Tensor | None = None
143
+ tactile_valid: torch.Tensor | None = None
144
+ rgb: RGBWorldResidualOutput | None = None
145
+
146
+
147
+ class NativeTachinW3Editor(nn.Module):
148
+ """Real W3: memory action editor plus RGB and tactile world objectives."""
149
+
150
+ def __init__(self, encoder: NativeTachinEncoder, pressure_mean: torch.Tensor,
151
+ pressure_scale: torch.Tensor) -> None:
152
+ super().__init__()
153
+ self.encoder = encoder
154
+ self.register_buffer("pressure_mean", pressure_mean.float(), persistent=True)
155
+ self.register_buffer("pressure_scale", pressure_scale.float().clamp_min(1e-6), persistent=True)
156
+ self.action = DirectMemoryActionEditor(torch.zeros(192), torch.ones(192),
157
+ tactile_dim=64, articulation_only=False)
158
+ self.tactile_world = TactileWorldModel(tactile_dim=64, cells=880)
159
+ self.rgb_world = FrozenRGBWorldResidual(tactile_dim=64, memory_rows=8,
160
+ condition_dim=2304, hidden_dim=256)
161
+
162
+ def forward(self, *, base_action, edit_mask, pressure, cell_valid, sample_valid,
163
+ final_memory, future_pressure=None, future_cell_valid=None,
164
+ tactile_tau=None, tactile_noise=None, rgb_world_head=None,
165
+ image_tokens=None, clean_latent=None, rgb_valid=None,
166
+ rgb_tau=None, rgb_noise=None, compute_auxiliary=True):
167
+ tactile, hand_valid = self.encoder(pressure, cell_valid, sample_valid)
168
+ final, delta, allowed = self.action(base_action, final_memory, tactile, hand_valid,
169
+ torch.zeros(len(base_action), device=base_action.device),
170
+ edit_mask)
171
+ tactile_velocity = tactile_target = tactile_valid = None
172
+ rgb = None
173
+ if compute_auxiliary:
174
+ required = (future_pressure, future_cell_valid, tactile_tau, tactile_noise,
175
+ rgb_world_head, image_tokens, clean_latent, rgb_valid, rgb_tau, rgb_noise)
176
+ if any(value is None for value in required):
177
+ raise ValueError("W3 auxiliary inputs are incomplete")
178
+ tactile_valid = future_cell_valid.bool()
179
+ clean = ((future_pressure.float() - self.pressure_mean) / self.pressure_scale)
180
+ clean = clean * tactile_valid.to(clean.dtype)
181
+ noisy, tactile_target = tactile_flow_pair(clean, tactile_valid, tactile_tau, tactile_noise)
182
+ action_window, action_valid = gather_action_window(final.detach(),
183
+ torch.zeros(len(final), device=final.device), steps=8)
184
+ tactile_velocity = self.tactile_world.velocity(noisy, tactile_tau,
185
+ final_memory=final_memory.detach(), tactile_features=tactile,
186
+ hand_valid=hand_valid, action_window=action_window, action_valid=action_valid)
187
+ rgb = self.rgb_world(world_head=rgb_world_head, image_tokens=image_tokens,
188
+ final_memory=final_memory.detach(), tactile_features=tactile,
189
+ hand_valid=hand_valid, clean_latent=clean_latent, world_valid=rgb_valid,
190
+ tau=rgb_tau, noise=rgb_noise)
191
+ return NativeW3Output(final, delta, allowed, tactile, hand_valid,
192
+ tactile_velocity, tactile_target, tactile_valid, rgb)
193
+
194
+ def optimizer_groups(self, action_lr=1e-4, encoder_lr=1e-5,
195
+ world_adapter_lr=1e-4, world_block_lr=1e-4):
196
+ return [
197
+ {"params": self.action.parameters(), "lr": action_lr, "name": "action"},
198
+ {"params": self.encoder.parameters(), "lr": encoder_lr, "name": "encoder"},
199
+ {"params": self.tactile_world.parameters(), "lr": world_block_lr, "name": "tactile_world"},
200
+ {"params": self.rgb_world.parameters(), "lr": world_adapter_lr, "name": "rgb_world"},
201
+ ]
model/tachin_paired30_cache_20260923/code/src/tactile_editor/rgb_world_editor.py ADDED
@@ -0,0 +1,419 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Frozen-RGB-world regularization for a Stage-2 tactile action editor.
2
+
3
+ The RGB world model is a teacher and is never stored inside this module. It predicts
4
+ rectified-flow velocity from the current image tokens and Method-D memory. A
5
+ reference-centred tactile history may perturb only the eight memory-condition rows;
6
+ the frozen SANA then turns that perturbation into a latent-velocity residual.
7
+
8
+ The action path is equally strict: action/memory features create residual bases and
9
+ per-hand tactile features create their coefficients. There is no action-only Stage-2
10
+ residual path, so reference tactile is an exact frozen Stage-1 identity.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ from dataclasses import dataclass
16
+ import math
17
+
18
+ import torch
19
+ from torch import nn
20
+
21
+ from .action_objectives import per_sample_action_loss
22
+ from .action_layout import ARTICULATION_GROUPS, FULL_ACTION_GROUPS, FULL_HAND_GROUPS
23
+ from .direct_residual import DirectTactileTrunk
24
+ from .single_stage_world import MemoryConditionedActionBasis
25
+ from .training import ACTION_GROUPS
26
+
27
+
28
+ def _check_shape(value: torch.Tensor, expected: tuple[int | None, ...], name: str) -> None:
29
+ if value.ndim != len(expected):
30
+ raise ValueError(f"{name} must have {len(expected)} dimensions")
31
+ for actual, wanted in zip(value.shape, expected):
32
+ if wanted is not None and actual != wanted:
33
+ raise ValueError(f"{name} must have shape {expected}, got {tuple(value.shape)}")
34
+
35
+
36
+ class TactileMemoryConditionModulator(nn.Module):
37
+ """Map two hand-local tactile features to the eight W0 memory rows.
38
+
39
+ Both projections are bias-free and the output layers are zero-initialized. Thus a
40
+ zero/reference tactile feature produces an exact zero condition delta, including at
41
+ initialization and after training.
42
+ """
43
+
44
+ def __init__(self, tactile_dim: int = 128, memory_rows: int = 8,
45
+ condition_dim: int = 2304, hidden_dim: int = 256) -> None:
46
+ super().__init__()
47
+ self.memory_rows = memory_rows
48
+ self.condition_dim = condition_dim
49
+ self.input = nn.ModuleList(
50
+ nn.Linear(tactile_dim, hidden_dim, bias=False) for _ in range(2)
51
+ )
52
+ self.output = nn.ModuleList(
53
+ nn.Linear(hidden_dim, memory_rows * condition_dim, bias=False)
54
+ for _ in range(2)
55
+ )
56
+ for layer in self.output:
57
+ nn.init.zeros_(layer.weight)
58
+
59
+ def forward(self, tactile_features: torch.Tensor,
60
+ hand_valid: torch.Tensor) -> torch.Tensor:
61
+ _check_shape(tactile_features, (None, 2, None), "tactile_features")
62
+ if hand_valid.shape != tactile_features.shape[:2]:
63
+ raise ValueError("hand_valid must be [B,2]")
64
+ pieces = []
65
+ for hand in range(2):
66
+ feature = tactile_features[:, hand]
67
+ hidden = torch.nn.functional.gelu(self.input[hand](feature))
68
+ delta = self.output[hand](hidden).reshape(
69
+ len(feature), self.memory_rows, self.condition_dim
70
+ )
71
+ pieces.append(delta * hand_valid[:, hand, None, None].to(delta.dtype))
72
+ return pieces[0] + pieces[1]
73
+
74
+
75
+ @dataclass
76
+ class RGBWorldResidualOutput:
77
+ predicted_residual: torch.Tensor
78
+ target_residual: torch.Tensor
79
+ baseline_velocity: torch.Tensor
80
+ tactile_velocity: torch.Tensor
81
+ valid: torch.Tensor
82
+
83
+
84
+ class FrozenRGBWorldResidual(nn.Module):
85
+ """Trainable tactile condition in front of a caller-owned, frozen RGB W0."""
86
+
87
+ def __init__(self, tactile_dim: int = 128, memory_rows: int = 8,
88
+ condition_dim: int = 2304, hidden_dim: int = 256) -> None:
89
+ super().__init__()
90
+ self.modulator = TactileMemoryConditionModulator(
91
+ tactile_dim=tactile_dim,
92
+ memory_rows=memory_rows,
93
+ condition_dim=condition_dim,
94
+ hidden_dim=hidden_dim,
95
+ )
96
+
97
+ def forward(
98
+ self,
99
+ *,
100
+ world_head,
101
+ image_tokens: torch.Tensor,
102
+ final_memory: torch.Tensor,
103
+ tactile_features: torch.Tensor,
104
+ hand_valid: torch.Tensor,
105
+ clean_latent: torch.Tensor,
106
+ world_valid: torch.Tensor,
107
+ tau: torch.Tensor,
108
+ noise: torch.Tensor,
109
+ ) -> RGBWorldResidualOutput:
110
+ _check_shape(image_tokens, (None, None, None), "image_tokens")
111
+ _check_shape(final_memory, (len(image_tokens), 8, None), "final_memory")
112
+ _check_shape(clean_latent, (len(image_tokens), None, None, None, None), "clean_latent")
113
+ b, horizons = clean_latent.shape[:2]
114
+ if world_valid.shape != (b, horizons):
115
+ raise ValueError("world_valid must be [B,H]")
116
+ if tau.shape != (b, horizons):
117
+ raise ValueError("tau must be [B,H]")
118
+ if noise.shape != clean_latent.shape:
119
+ raise ValueError("noise must match clean_latent")
120
+
121
+ # Replace invalid values before arithmetic: NaN * 0 is still NaN.
122
+ valid5 = world_valid[..., None, None, None]
123
+ clean = torch.where(valid5, clean_latent.float(), torch.zeros_like(clean_latent.float()))
124
+ eps = torch.where(valid5, noise.float(), torch.zeros_like(noise.float()))
125
+ mixed = (1.0 - tau[..., None, None, None].float()) * clean
126
+ mixed = mixed + tau[..., None, None, None].float() * eps
127
+ target_velocity = eps - clean
128
+
129
+ adapter_dtype = next(world_head.adapter.parameters()).dtype
130
+ base_condition = world_head.build_condition(
131
+ image_tokens.to(adapter_dtype), final_memory.to(adapter_dtype)
132
+ )
133
+ memory_rows = self.modulator.memory_rows
134
+ if base_condition.shape[1] < memory_rows:
135
+ raise ValueError("world condition has fewer rows than the memory modulator")
136
+ delta = self.modulator(tactile_features, hand_valid).to(base_condition.dtype)
137
+ tactile_condition = base_condition.clone()
138
+ tactile_condition[:, -memory_rows:] = tactile_condition[:, -memory_rows:] + delta
139
+
140
+ adapter_horizons = getattr(world_head.adapter.cfg, "num_horizons", 1)
141
+ base_rows, tactile_rows = [], []
142
+ for horizon in range(horizons):
143
+ if adapter_horizons == horizons:
144
+ base_rows.append(world_head.adapter.for_horizon(base_condition, horizon))
145
+ tactile_rows.append(world_head.adapter.for_horizon(tactile_condition, horizon))
146
+ elif adapter_horizons == 1:
147
+ base_rows.append(base_condition)
148
+ tactile_rows.append(tactile_condition)
149
+ else:
150
+ raise ValueError(
151
+ "world adapter horizon count must be one or match clean_latent"
152
+ )
153
+ base_flat = torch.cat(base_rows, dim=0)
154
+ tactile_flat = torch.cat(tactile_rows, dim=0)
155
+
156
+ # Horizon-major flattening is required by W0's modulation horizon embedding.
157
+ flatten = lambda value: value.transpose(0, 1).reshape( # noqa: E731
158
+ horizons * b, *value.shape[2:]
159
+ )
160
+ mixed_flat = flatten(mixed).to(base_flat.dtype)
161
+ tau_flat = tau.transpose(0, 1).reshape(horizons * b).to(base_flat.dtype)
162
+
163
+ with torch.no_grad():
164
+ baseline = world_head.predict_velocity(mixed_flat, tau_flat, base_flat)
165
+ # W0 parameters are frozen, but this call keeps autograd with respect to the
166
+ # tactile-conditioned input.
167
+ tactile_velocity = world_head.predict_velocity(
168
+ mixed_flat, tau_flat, tactile_flat
169
+ )
170
+ target_flat = flatten(target_velocity)
171
+ predicted_residual = tactile_velocity.float() - baseline.float().detach()
172
+ target_residual = target_flat.float() - baseline.float().detach()
173
+ return RGBWorldResidualOutput(
174
+ predicted_residual=predicted_residual,
175
+ target_residual=target_residual.detach(),
176
+ baseline_velocity=baseline.detach(),
177
+ tactile_velocity=tactile_velocity,
178
+ valid=world_valid.transpose(0, 1).reshape(horizons * b),
179
+ )
180
+
181
+
182
+ def roi_weighted_world_loss(output: RGBWorldResidualOutput,
183
+ roi: torch.Tensor,
184
+ *, background_weight: float = 0.1) -> torch.Tensor:
185
+ """MSE on latent velocity with hand ROI weight 1 and background weight 0.1."""
186
+ if not 0 <= background_weight <= 1:
187
+ raise ValueError("background_weight must be in [0,1]")
188
+ if roi.ndim != 4:
189
+ raise ValueError("roi must be [B,H,Hlatent,Wlatent]")
190
+ flat_roi = roi.transpose(0, 1).reshape(-1, *roi.shape[2:]).bool()
191
+ if flat_roi.shape[0] != output.predicted_residual.shape[0]:
192
+ raise ValueError("roi rows must match world residual rows")
193
+ if flat_roi.shape[1:] != output.predicted_residual.shape[2:]:
194
+ raise ValueError("roi spatial shape must match world latent")
195
+ valid = output.valid.bool()
196
+ if not valid.any():
197
+ raise ValueError("the batch has no valid RGB world targets")
198
+ weight = torch.full_like(flat_roi, background_weight, dtype=torch.float32)
199
+ weight = torch.where(flat_roi, torch.ones_like(weight), weight)
200
+ error = (output.predicted_residual - output.target_residual).square()
201
+ weighted = error * weight[:, None].to(error.device)
202
+ per_row = weighted.flatten(1).sum(1) / (
203
+ weight.flatten(1).sum(1).to(error.device) * error.shape[1]
204
+ ).clamp_min(1)
205
+ return per_row[valid].mean()
206
+
207
+
208
+ @dataclass
209
+ class Stage2Output:
210
+ final_action: torch.Tensor
211
+ delta_action: torch.Tensor
212
+ coefficients: torch.Tensor
213
+ tactile_features: torch.Tensor
214
+ tactile_history_features: torch.Tensor
215
+ hand_valid: torch.Tensor
216
+ allowed_mask: torch.Tensor
217
+ world: RGBWorldResidualOutput | None
218
+
219
+
220
+ class RGBWorldRegularizedTactileEditor(nn.Module):
221
+ """Strict per-hand Stage-2 action editor with an RGB-world auxiliary branch."""
222
+
223
+ def __init__(
224
+ self,
225
+ *,
226
+ action_mean: torch.Tensor,
227
+ action_std: torch.Tensor,
228
+ pressure_masks: torch.Tensor,
229
+ reference_pressure: torch.Tensor,
230
+ residual_bases: int = 8,
231
+ articulation_only: bool = False,
232
+ memory_dim: int = 960,
233
+ hidden_dim: int = 256,
234
+ world_condition_dim: int = 2304,
235
+ ) -> None:
236
+ super().__init__()
237
+ self.tactile = DirectTactileTrunk(
238
+ pressure_masks,
239
+ reference_pressure,
240
+ mode="history_delta",
241
+ coefficient_scope="hand_local",
242
+ residual_bases=residual_bases,
243
+ )
244
+ self.action = MemoryConditionedActionBasis(
245
+ action_mean,
246
+ action_std,
247
+ memory_dim=memory_dim,
248
+ hidden_dim=hidden_dim,
249
+ residual_bases=2 * residual_bases,
250
+ )
251
+ self.rgb_world = FrozenRGBWorldResidual(
252
+ tactile_dim=128,
253
+ memory_rows=8,
254
+ condition_dim=world_condition_dim,
255
+ hidden_dim=hidden_dim,
256
+ )
257
+ self.residual_bases = residual_bases
258
+ self.articulation_only = articulation_only
259
+ component = torch.zeros(1, 2, 1, 192, dtype=torch.bool)
260
+ groups = (
261
+ ARTICULATION_GROUPS if articulation_only else FULL_HAND_GROUPS
262
+ )
263
+ for hand, (start, stop) in enumerate(groups):
264
+ component[:, hand, :, start:stop] = True
265
+ self.register_buffer("hand_component_mask", component, persistent=True)
266
+
267
+ def forward(
268
+ self,
269
+ *,
270
+ anchor_action: torch.Tensor,
271
+ edit_mask: torch.Tensor,
272
+ pressure: torch.Tensor,
273
+ temporal_valid: torch.Tensor,
274
+ decision_seconds: torch.Tensor,
275
+ final_memory: torch.Tensor,
276
+ world_head=None,
277
+ image_tokens: torch.Tensor | None = None,
278
+ clean_latent: torch.Tensor | None = None,
279
+ world_valid: torch.Tensor | None = None,
280
+ tau: torch.Tensor | None = None,
281
+ noise: torch.Tensor | None = None,
282
+ ) -> Stage2Output:
283
+ coefficients, hand_valid, tactile_features, tactile_history = self.tactile(
284
+ pressure, temporal_valid
285
+ )
286
+ all_bases = self.action(anchor_action, final_memory, decision_seconds)
287
+ bases = all_bases.reshape(
288
+ len(anchor_action), 2, self.residual_bases, 16, 192
289
+ )
290
+ per_hand = (coefficients[..., None, None] * bases).sum(2) / math.sqrt(
291
+ self.residual_bases
292
+ )
293
+ hand_allowed = edit_mask[:, None].bool() & self.hand_component_mask
294
+ hand_allowed = hand_allowed & hand_valid[:, :, None, None]
295
+ delta = (per_hand * hand_allowed.to(per_hand.dtype)).sum(1)
296
+ final_action = anchor_action.detach() + delta
297
+
298
+ supplied = (
299
+ world_head is not None,
300
+ image_tokens is not None,
301
+ clean_latent is not None,
302
+ world_valid is not None,
303
+ tau is not None,
304
+ noise is not None,
305
+ )
306
+ if any(supplied) and not all(supplied):
307
+ raise ValueError("all RGB world inputs must be supplied together")
308
+ world = None
309
+ if all(supplied):
310
+ world = self.rgb_world(
311
+ world_head=world_head,
312
+ image_tokens=image_tokens,
313
+ final_memory=final_memory.detach(),
314
+ tactile_features=tactile_features,
315
+ hand_valid=hand_valid,
316
+ clean_latent=clean_latent,
317
+ world_valid=world_valid,
318
+ tau=tau,
319
+ noise=noise,
320
+ )
321
+ return Stage2Output(
322
+ final_action=final_action,
323
+ delta_action=delta,
324
+ coefficients=coefficients,
325
+ tactile_features=tactile_features,
326
+ tactile_history_features=tactile_history,
327
+ hand_valid=hand_valid,
328
+ allowed_mask=hand_allowed.any(1),
329
+ world=world,
330
+ )
331
+
332
+ def objective(
333
+ self,
334
+ output: Stage2Output,
335
+ target_action: torch.Tensor,
336
+ edit_mask: torch.Tensor,
337
+ *,
338
+ roi: torch.Tensor | None,
339
+ lambda_world: float,
340
+ lambda_delta: float = 1e-4,
341
+ lambda_coefficient: float = 1e-5,
342
+ background_weight: float = 0.1,
343
+ ) -> tuple[torch.Tensor, dict[str, torch.Tensor]]:
344
+ if min(lambda_world, lambda_delta, lambda_coefficient) < 0:
345
+ raise ValueError("loss weights must be non-negative")
346
+ mask = edit_mask.bool() & output.allowed_mask
347
+ present = mask.any((1, 2))
348
+ if not present.any():
349
+ raise ValueError("the batch has no tactile-editable action target")
350
+ groups = ARTICULATION_GROUPS if self.articulation_only else FULL_ACTION_GROUPS
351
+ action = per_sample_action_loss(
352
+ output.final_action[present], target_action[present], mask[present], groups=groups
353
+ ).mean()
354
+ delta_l2 = output.delta_action.float().square().masked_fill(~mask, 0).sum()
355
+ delta_l2 = delta_l2 / mask.sum().clamp_min(1)
356
+ coefficient_l2 = output.coefficients.float().square().mean()
357
+ if lambda_world:
358
+ if output.world is None or roi is None:
359
+ raise ValueError("RGB world output and roi are required")
360
+ world = roi_weighted_world_loss(
361
+ output.world, roi, background_weight=background_weight
362
+ )
363
+ else:
364
+ world = action.new_zeros(())
365
+ total = action + lambda_world * world
366
+ total = total + lambda_delta * delta_l2 + lambda_coefficient * coefficient_l2
367
+ return total, {
368
+ "loss": total.detach(),
369
+ "action_loss": action.detach(),
370
+ "world_loss": world.detach(),
371
+ "delta_l2": delta_l2.detach(),
372
+ "coefficient_l2": coefficient_l2.detach(),
373
+ }
374
+
375
+ def optimizer_groups(
376
+ self,
377
+ *,
378
+ action_lr: float = 1e-4,
379
+ tactile_gru_lr: float = 1e-5,
380
+ tactile_cnn_lr: float = 3e-6,
381
+ world_modulator_lr: float = 1e-4,
382
+ ) -> list[dict]:
383
+ groups = [
384
+ {
385
+ "name": "action",
386
+ "params": [
387
+ *self.action.parameters(),
388
+ *self.tactile.coefficient_head.parameters(),
389
+ ],
390
+ "lr": action_lr,
391
+ },
392
+ {
393
+ "name": "tactile.cnn",
394
+ "params": list(self.tactile.frame_encoder.parameters()),
395
+ "lr": tactile_cnn_lr,
396
+ },
397
+ {
398
+ "name": "world.modulator",
399
+ "params": list(self.rgb_world.parameters()),
400
+ "lr": world_modulator_lr,
401
+ },
402
+ ]
403
+ if self.tactile.temporal is not None:
404
+ groups.append(
405
+ {
406
+ "name": "tactile.gru",
407
+ "params": list(self.tactile.temporal.parameters()),
408
+ "lr": tactile_gru_lr,
409
+ }
410
+ )
411
+ return groups
412
+
413
+
414
+ def ramp_world_weight(step: int, maximum: float, warmup_steps: int) -> float:
415
+ if maximum < 0 or warmup_steps < 0:
416
+ raise ValueError("maximum and warmup_steps must be non-negative")
417
+ if warmup_steps == 0:
418
+ return float(maximum)
419
+ return float(maximum) * min(1.0, max(0.0, step / warmup_steps))
model/tachin_paired30_cache_20260923/code/src/tactile_editor/runtime.py ADDED
@@ -0,0 +1,376 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Timestamp-driven tactile buffers and pipelined VLA action control."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import bisect
6
+ import math
7
+ from dataclasses import dataclass, field
8
+ from typing import Callable
9
+
10
+ import torch
11
+
12
+
13
+ class TactileRingBuffer:
14
+ """Causally resample a native-rate stream to the encoder's 30 Hz contract."""
15
+
16
+ def __init__(self, history_frames: int = 8, model_hz: float = 30.0) -> None:
17
+ if history_frames <= 0 or model_hz <= 0:
18
+ raise ValueError("history_frames and model_hz must be positive")
19
+ self.history_frames = int(history_frames)
20
+ self.model_hz = float(model_hz)
21
+ self._timestamps: list[float] = []
22
+ self._pressure: list[torch.Tensor] = []
23
+ self._valid: list[torch.Tensor] = []
24
+ self.episode_id: int | str | None = None
25
+
26
+ @property
27
+ def sample_count(self) -> int:
28
+ return len(self._timestamps)
29
+
30
+ def reset(self, episode_id: int | str | None = None) -> None:
31
+ self._timestamps.clear()
32
+ self._pressure.clear()
33
+ self._valid.clear()
34
+ self.episode_id = episode_id
35
+
36
+ def append(
37
+ self,
38
+ timestamp: float,
39
+ pressure: torch.Tensor,
40
+ valid: torch.Tensor | None = None,
41
+ *,
42
+ episode_id: int | str | None = None,
43
+ ) -> None:
44
+ if not math.isfinite(timestamp):
45
+ raise ValueError("tactile timestamp must be finite")
46
+ if tuple(pressure.shape) != (2, 21, 21):
47
+ raise ValueError("one tactile frame must be [2,21,21]")
48
+ if self.episode_id is not None and episode_id != self.episode_id:
49
+ self.reset(episode_id)
50
+ elif self.episode_id is None:
51
+ self.episode_id = episode_id
52
+ if self._timestamps and timestamp <= self._timestamps[-1]:
53
+ raise ValueError("tactile timestamps must be strictly increasing")
54
+ if valid is None:
55
+ valid = torch.ones(2, dtype=torch.bool, device=pressure.device)
56
+ if tuple(valid.shape) != (2,):
57
+ raise ValueError("tactile validity must be [2]")
58
+ self._timestamps.append(float(timestamp))
59
+ self._pressure.append(pressure.detach().clone())
60
+ self._valid.append(
61
+ valid.to(device=pressure.device, dtype=torch.bool).detach().clone()
62
+ )
63
+ # Retain one sample before the useful horizon. That sample is needed for
64
+ # causal zero-order hold when a target time falls between sensor frames.
65
+ horizon = (self.history_frames + 1) / self.model_hz
66
+ first_useful = bisect.bisect_left(self._timestamps, float(timestamp) - horizon)
67
+ remove_count = max(0, first_useful - 1)
68
+ if remove_count:
69
+ del self._timestamps[:remove_count]
70
+ del self._pressure[:remove_count]
71
+ del self._valid[:remove_count]
72
+
73
+ def window(self, timestamp: float) -> tuple[torch.Tensor, torch.Tensor]:
74
+ """Return [1,8,2,21,21] and [1,8,2], using no future sample."""
75
+
76
+ if not self._timestamps:
77
+ raise RuntimeError("cannot build a tactile window from an empty buffer")
78
+ frames: list[torch.Tensor] = []
79
+ valids: list[torch.Tensor] = []
80
+ for offset in range(self.history_frames - 1, -1, -1):
81
+ target = float(timestamp) - offset / self.model_hz
82
+ index = bisect.bisect_right(self._timestamps, target + 1e-9) - 1
83
+ if index < 0:
84
+ # Repeat the earliest value for a stable tensor, but mark it padded.
85
+ frames.append(self._pressure[0])
86
+ valids.append(torch.zeros_like(self._valid[0]))
87
+ else:
88
+ frames.append(self._pressure[index])
89
+ valids.append(self._valid[index])
90
+ return torch.stack(frames)[None], torch.stack(valids)[None]
91
+
92
+
93
+ @dataclass
94
+ class ActionPlan:
95
+ """A revision-safe full chunk. Committed actions are immutable."""
96
+
97
+ actions: torch.Tensor # [T,A], normally normalized inside the editor
98
+ started_at: float
99
+ action_hz: float
100
+ plan_id: int
101
+ action_mask: torch.Tensor | None = None # [T,A], valid Method-D action slots
102
+ _base_actions: torch.Tensor = field(init=False, repr=False)
103
+
104
+ def __post_init__(self) -> None:
105
+ if self.actions.ndim != 2:
106
+ raise ValueError("actions must be [T,A]")
107
+ if self.action_hz <= 0:
108
+ raise ValueError("action_hz must be positive")
109
+ self.actions = self.actions.detach().clone()
110
+ self._base_actions = self.actions.clone()
111
+ if self.action_mask is not None:
112
+ if self.action_mask.shape != self.actions.shape:
113
+ raise ValueError("action_mask and actions must have the same shape")
114
+ self.action_mask = (
115
+ self.action_mask.to(device=self.actions.device, dtype=torch.bool)
116
+ .detach()
117
+ .clone()
118
+ )
119
+
120
+ @property
121
+ def base_actions(self) -> torch.Tensor:
122
+ """Original installed VLA output; callers cannot mutate the snapshot."""
123
+ return self._base_actions.clone()
124
+
125
+ def committed_steps(self, now: float, compute_latency_seconds: float = 0.0) -> int:
126
+ elapsed = max(0.0, float(now) - self.started_at + compute_latency_seconds)
127
+ # Subtract a small tolerance so exact action boundaries stay exact after
128
+ # floating-point subtraction (1.05 - 1.0 can be greater than 0.05).
129
+ return min(
130
+ len(self.actions), max(0, math.ceil(elapsed * self.action_hz - 1e-9))
131
+ )
132
+
133
+ def editable_mask(
134
+ self, now: float, compute_latency_seconds: float = 0.0
135
+ ) -> torch.Tensor:
136
+ start = self.committed_steps(now, compute_latency_seconds)
137
+ result = torch.zeros_like(self.actions, dtype=torch.bool)
138
+ result[start:] = True
139
+ if self.action_mask is not None:
140
+ result &= self.action_mask
141
+ return result
142
+
143
+ def apply_revision(
144
+ self,
145
+ candidate: torch.Tensor,
146
+ *,
147
+ now: float,
148
+ compute_latency_seconds: float = 0.0,
149
+ expected_plan_id: int | None = None,
150
+ ) -> torch.Tensor:
151
+ if candidate.shape != self.actions.shape:
152
+ raise ValueError("candidate and plan action shapes differ")
153
+ if expected_plan_id is not None and expected_plan_id != self.plan_id:
154
+ raise RuntimeError("stale tactile result targets an obsolete action plan")
155
+ mask = self.editable_mask(now, compute_latency_seconds)
156
+ self.actions = torch.where(mask, candidate, self.actions)
157
+ return self.actions
158
+
159
+
160
+ RevisionFn = Callable[
161
+ [torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor], torch.Tensor
162
+ ]
163
+
164
+
165
+ def next_action_boundary(timestamp: float, action_hz: float) -> float:
166
+ """Return the first 80 Hz-style action boundary at or after a timestamp."""
167
+
168
+ if action_hz <= 0:
169
+ raise ValueError("action_hz must be positive")
170
+ if not math.isfinite(timestamp):
171
+ raise ValueError("timestamp must be finite")
172
+ tick = math.ceil(float(timestamp) * action_hz - 1e-9)
173
+ return tick / action_hz
174
+
175
+
176
+ @dataclass(frozen=True)
177
+ class InferenceRequest:
178
+ """One VLA request with its camera capture timestamp."""
179
+
180
+ request_id: int
181
+ captured_at: float
182
+ episode_id: int | str
183
+
184
+
185
+ @dataclass(frozen=True)
186
+ class ScheduledActionPlan:
187
+ """A completed plan waiting for an atomic action-boundary install."""
188
+
189
+ request_id: int
190
+ plan: ActionPlan
191
+ ready_at: float
192
+ install_at: float
193
+
194
+
195
+ class PipelinedController:
196
+ """Overlap VLA inference with execution and install each new action queue."""
197
+
198
+ def __init__(
199
+ self,
200
+ *,
201
+ control_hz: float = 10.0,
202
+ action_hz: float = 80.0,
203
+ tactile_model_hz: float = 30.0,
204
+ history_frames: int = 8,
205
+ compute_latency_seconds: float = 0.0,
206
+ ) -> None:
207
+ if control_hz <= 0 or action_hz <= 0 or tactile_model_hz <= 0:
208
+ raise ValueError("runtime rates must be positive")
209
+ if history_frames <= 0:
210
+ raise ValueError("history_frames must be positive")
211
+ if compute_latency_seconds < 0:
212
+ raise ValueError("compute_latency_seconds must be non-negative")
213
+ self.control_hz = float(control_hz)
214
+ self.action_hz = float(action_hz)
215
+ self.compute_latency_seconds = float(compute_latency_seconds)
216
+ self.touch = TactileRingBuffer(history_frames, tactile_model_hz)
217
+ self.plan: ActionPlan | None = None
218
+ self.active_schedule: ScheduledActionPlan | None = None
219
+ self.pending_schedule: ScheduledActionPlan | None = None
220
+ self.inflight_request: InferenceRequest | None = None
221
+ self._episode_id: int | str | None = None
222
+ self._last_capture_at: float | None = None
223
+ self._next_request_id = 0
224
+ self._next_plan_id = 0
225
+
226
+ def reset(self, episode_id: int | str | None = None) -> None:
227
+ """Clear every temporal state at an episode boundary."""
228
+
229
+ self.touch.reset(episode_id)
230
+ self.plan = None
231
+ self.active_schedule = None
232
+ self.pending_schedule = None
233
+ self.inflight_request = None
234
+ self._episode_id = episode_id
235
+ self._last_capture_at = None
236
+
237
+ def _activate_ready_plan(self, timestamp: float) -> None:
238
+ pending = self.pending_schedule
239
+ if pending is not None and float(timestamp) + 1e-9 >= pending.install_at:
240
+ self.active_schedule = pending
241
+ self.plan = pending.plan
242
+ self.pending_schedule = None
243
+
244
+ def begin_inference(
245
+ self, *, captured_at: float, episode_id: int | str
246
+ ) -> InferenceRequest:
247
+ """Start a VLA request while the current action plan remains active."""
248
+
249
+ if not math.isfinite(captured_at):
250
+ raise ValueError("capture timestamp must be finite")
251
+ if self._episode_id is not None and episode_id != self._episode_id:
252
+ self.reset(episode_id)
253
+ elif self._episode_id is None:
254
+ self.reset(episode_id)
255
+ self._activate_ready_plan(captured_at)
256
+ if self.inflight_request is not None:
257
+ raise RuntimeError("one VLA inference request is already in flight")
258
+ if self._last_capture_at is not None:
259
+ minimum = self._last_capture_at + 1.0 / self.control_hz
260
+ if captured_at < minimum - 1e-9:
261
+ raise RuntimeError(
262
+ "VLA capture requests exceed the configured control rate"
263
+ )
264
+ self._next_request_id += 1
265
+ request = InferenceRequest(
266
+ self._next_request_id, float(captured_at), episode_id
267
+ )
268
+ self.inflight_request = request
269
+ self._last_capture_at = float(captured_at)
270
+ return request
271
+
272
+ def complete_inference(
273
+ self,
274
+ full_action: torch.Tensor,
275
+ *,
276
+ request_id: int,
277
+ ready_at: float,
278
+ episode_id: int | str,
279
+ action_mask: torch.Tensor | None = None,
280
+ ) -> ScheduledActionPlan:
281
+ """Schedule an unedited VLA chunk for the next action boundary.
282
+
283
+ Pass the frozen backbone output, not a previously revised V1 output.
284
+ Subsequent touch updates always predict residuals from this snapshot.
285
+ """
286
+
287
+ request = self.inflight_request
288
+ if request is None or request.request_id != request_id:
289
+ raise RuntimeError("stale or unknown VLA inference result")
290
+ if request.episode_id != episode_id or self._episode_id != episode_id:
291
+ raise RuntimeError("VLA inference result belongs to an obsolete episode")
292
+ if not math.isfinite(ready_at) or ready_at < request.captured_at:
293
+ raise ValueError("ready_at must be finite and no earlier than captured_at")
294
+ self._activate_ready_plan(ready_at)
295
+ if self.pending_schedule is not None:
296
+ raise RuntimeError(
297
+ "the previous VLA plan has not reached its install boundary"
298
+ )
299
+ install_at = next_action_boundary(ready_at, self.action_hz)
300
+ self._next_plan_id += 1
301
+ plan = ActionPlan(
302
+ full_action.detach().clone(),
303
+ started_at=install_at,
304
+ action_hz=self.action_hz,
305
+ plan_id=self._next_plan_id,
306
+ action_mask=action_mask,
307
+ )
308
+ self.inflight_request = None
309
+ scheduled = ScheduledActionPlan(
310
+ request_id=request_id,
311
+ plan=plan,
312
+ ready_at=float(ready_at),
313
+ install_at=install_at,
314
+ )
315
+ self.pending_schedule = scheduled
316
+ self._activate_ready_plan(ready_at)
317
+ return scheduled
318
+
319
+ def update_touch(
320
+ self,
321
+ pressure: torch.Tensor,
322
+ *,
323
+ timestamp: float,
324
+ episode_id: int | str,
325
+ valid: torch.Tensor | None = None,
326
+ revise: RevisionFn | None = None,
327
+ ) -> ActionPlan | None:
328
+ """Append one sensor frame and optionally operate a fast V1 revision."""
329
+
330
+ if self._episode_id is not None and episode_id != self._episode_id:
331
+ self.reset(episode_id)
332
+ elif self._episode_id is None:
333
+ self.reset(episode_id)
334
+ self.touch.append(timestamp, pressure, valid, episode_id=episode_id)
335
+ self._activate_ready_plan(timestamp)
336
+ if self.plan is None or revise is None:
337
+ return self.plan
338
+ # Capture the exact plan targeted by this callback. A new VLA result may
339
+ # replace ``self.plan`` while a slow tactile revision is still running.
340
+ target_plan = self.plan
341
+ expected_plan_id = target_plan.plan_id
342
+ window, window_valid = self.touch.window(timestamp)
343
+ edit_mask = target_plan.editable_mask(timestamp, self.compute_latency_seconds)
344
+ candidate = revise(
345
+ target_plan.base_actions[None], edit_mask[None], window, window_valid
346
+ )
347
+ if tuple(candidate.shape) != (1, *target_plan.actions.shape):
348
+ raise ValueError("revision callback must return [1,T,A]")
349
+ if self.plan is not target_plan or self.plan.plan_id != expected_plan_id:
350
+ raise RuntimeError("stale tactile result targets an obsolete action plan")
351
+ target_plan.apply_revision(
352
+ candidate[0],
353
+ now=timestamp,
354
+ compute_latency_seconds=self.compute_latency_seconds,
355
+ expected_plan_id=expected_plan_id,
356
+ )
357
+ return target_plan
358
+
359
+ def next_action(self, timestamp: float) -> torch.Tensor:
360
+ """Return the next action relative to the plan installation time."""
361
+
362
+ self._activate_ready_plan(timestamp)
363
+ if self.plan is None:
364
+ raise RuntimeError(
365
+ "no active VLA action plan is available; use a safe hold"
366
+ )
367
+ index = math.floor(
368
+ (float(timestamp) - self.plan.started_at) * self.action_hz + 1e-9
369
+ )
370
+ if index < 0:
371
+ raise RuntimeError("the active VLA plan starts in the future")
372
+ if index >= len(self.plan.actions):
373
+ raise RuntimeError(
374
+ "the full action-chunk latency buffer is exhausted; use a safe hold"
375
+ )
376
+ return self.plan.actions[index]
model/tachin_paired30_cache_20260923/code/src/tactile_editor/touch_study.py ADDED
@@ -0,0 +1,196 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Experimental incremental editors. These do not change the deployment defaults.
2
+
3
+ The immutable anchor is a frozen, calibrated VLA plan. All methods use the same
4
+ uncapped head and output mask. Modality dropout only affects conditioning.
5
+ """
6
+
7
+ from dataclasses import dataclass
8
+ import hashlib
9
+ import json
10
+ import math
11
+ from pathlib import Path
12
+
13
+ import torch
14
+ from torch import nn
15
+ from torch.nn import functional as F
16
+
17
+ from .fusion import GatedCrossAttention
18
+ from .training import ACTION_GROUPS
19
+
20
+
21
+ @dataclass(frozen=True)
22
+ class StudySetting:
23
+ name: str
24
+ encoder_lr: float = 0.0
25
+ constant: bool = False
26
+ condition_dropout: float = 0.0
27
+ touch_dropout: float = 0.0
28
+ dropout: float = 0.1
29
+ teacher_weight: float = 0.0
30
+ dynamics_weight: float = 0.0
31
+ innovation: bool = False
32
+ event_sampling: bool = False
33
+ gradient_balance: bool = False
34
+
35
+
36
+ SETTINGS = {
37
+ s.name: s
38
+ for s in (
39
+ StudySetting("constant", constant=True),
40
+ StudySetting("frozen"),
41
+ StudySetting("finetune", encoder_lr=1e-5),
42
+ StudySetting("ordinary_dropout", encoder_lr=1e-5, dropout=0.3),
43
+ StudySetting("condition_dropout", encoder_lr=1e-5, condition_dropout=0.2),
44
+ StudySetting("touch_dropout", encoder_lr=1e-5, touch_dropout=0.2),
45
+ StudySetting("teacher", encoder_lr=1e-5, teacher_weight=1.0),
46
+ StudySetting("dynamics", encoder_lr=1e-5, dynamics_weight=0.1),
47
+ StudySetting("innovation", innovation=True),
48
+ StudySetting("event_sampling", event_sampling=True),
49
+ StudySetting("gradient_balance", encoder_lr=1e-5, gradient_balance=True),
50
+ )
51
+ }
52
+
53
+
54
+ def digest(path):
55
+ """Return a streaming SHA-256 digest for provenance records."""
56
+ value = hashlib.sha256()
57
+ with Path(path).open("rb") as source:
58
+ for block in iter(lambda: source.read(8 * 1024 * 1024), b""):
59
+ value.update(block)
60
+ return value.hexdigest()
61
+
62
+
63
+ def write_json(path, value):
64
+ """Atomically replace a JSON result file."""
65
+ path = Path(path)
66
+ temporary = path.with_suffix(path.suffix + ".tmp")
67
+ temporary.write_text(json.dumps(value, indent=2, default=str) + "\n")
68
+ temporary.replace(path)
69
+
70
+
71
+ def learning_rate_factor(step, horizon, warmup):
72
+ """Linear warmup followed by cosine decay to ten percent."""
73
+ if horizon < 1 or not 0 <= warmup < horizon:
74
+ raise ValueError("schedule horizon must exceed nonnegative warmup")
75
+ if step < warmup:
76
+ return (step + 1) / warmup
77
+ progress = min(1.0, max(0.0, (step - warmup) / (horizon - warmup)))
78
+ return 0.1 + 0.9 * (1 + math.cos(math.pi * progress)) / 2
79
+
80
+
81
+ def per_sample_action_loss(prediction, target, mask):
82
+ """Same group/sample weighting as v1_residual_objective, without reduction."""
83
+ safe_target = torch.where(mask, target, prediction.detach())
84
+ error = F.smooth_l1_loss(prediction, safe_target, reduction="none")
85
+ values, present = [], []
86
+ for lo, hi in ACTION_GROUPS:
87
+ count = mask[..., lo:hi].sum((1, 2))
88
+ values.append(
89
+ error[..., lo:hi].masked_fill(~mask[..., lo:hi], 0).sum((1, 2))
90
+ / count.clamp_min(1)
91
+ )
92
+ present.append(count > 0)
93
+ occupied = torch.stack(present, 1)
94
+ if not occupied.any(1).all():
95
+ raise ValueError("every study sample must contain supervised actions")
96
+ return torch.stack(values, 1).sum(1) / occupied.sum(1)
97
+
98
+
99
+ def masked_dynamics_loss(predicted, target, valid):
100
+ """Exclude missing/ambiguous future labels before arithmetic, including NaNs."""
101
+ safe = torch.where(valid[..., None], target, predicted.detach())
102
+ values = F.smooth_l1_loss(predicted, safe, reduction="none").mean(-1)
103
+ return values.masked_fill(~valid, 0).sum() / valid.sum().clamp_min(1)
104
+
105
+
106
+ class IncrementalTouchEditor(nn.Module):
107
+ def __init__(self, action_mean, action_std, setting, hidden=128):
108
+ super().__init__()
109
+ self.setting = setting
110
+ self.register_buffer("action_mean", action_mean)
111
+ self.register_buffer("action_std", action_std.clamp_min(1e-3))
112
+ self.action_projection = nn.Linear(192, hidden)
113
+ # All settings reserve identical capacity for the optional innovation.
114
+ self.touch_projection = nn.Linear(128, hidden)
115
+ self.position = nn.Parameter(torch.randn(1, 16, hidden) * 0.02)
116
+ self.hand_embedding = nn.Parameter(torch.randn(1, 2, hidden) * 0.02)
117
+ self.time_projection = nn.Linear(1, hidden)
118
+ self.attention = GatedCrossAttention(hidden, 4)
119
+ nn.init.constant_(self.attention.gate, 0.1)
120
+ layer = nn.TransformerEncoderLayer(
121
+ hidden,
122
+ 4,
123
+ hidden * 4,
124
+ dropout=setting.dropout,
125
+ batch_first=True,
126
+ norm_first=True,
127
+ )
128
+ self.trunk = nn.TransformerEncoder(layer, 1, enable_nested_tensor=False)
129
+ self.output = nn.Linear(hidden, 192)
130
+ nn.init.zeros_(self.output.weight)
131
+ nn.init.zeros_(self.output.bias)
132
+
133
+ def forward(
134
+ self, anchor, mask, tokens, hand_valid, elapsed, predicted_tokens, mean_tokens
135
+ ):
136
+ condition = self.action_projection(
137
+ (anchor.detach() - self.action_mean) / self.action_std
138
+ )
139
+ if self.training and self.setting.condition_dropout:
140
+ # Block-mask plan steps, not residual anchors, targets or masks.
141
+ keep = (
142
+ torch.rand((*condition.shape[:2], 1), device=anchor.device)
143
+ >= self.setting.condition_dropout
144
+ )
145
+ condition = condition * keep
146
+ if self.setting.constant:
147
+ tokens = mean_tokens.expand_as(tokens)
148
+ if self.training and self.setting.touch_dropout:
149
+ keep = (
150
+ torch.rand((len(tokens), 1, 1), device=tokens.device)
151
+ >= self.setting.touch_dropout
152
+ )
153
+ tokens = torch.where(keep, tokens, mean_tokens.expand_as(tokens))
154
+ innovation = (
155
+ tokens - predicted_tokens
156
+ if self.setting.innovation
157
+ else torch.zeros_like(tokens)
158
+ )
159
+ memory = (
160
+ self.touch_projection(torch.cat((tokens, innovation), -1))
161
+ + self.hand_embedding
162
+ )
163
+ query = (
164
+ condition
165
+ + self.position
166
+ + self.time_projection(elapsed.float()[:, None, None] * 10)
167
+ )
168
+ hidden = self.trunk(self.attention(query, memory, hand_valid))
169
+ raw_delta = self.output(hidden)
170
+ delta = raw_delta * mask * hand_valid.any(1)[:, None, None]
171
+ return anchor.detach() + delta, delta
172
+
173
+
174
+ def balance_condition_gradients(model):
175
+ """Exploratory regression adaptation, NOT a reproduction of OGM-GE.
176
+
177
+ Only attenuate action-projection gradients when their norm exceeds the touch
178
+ projection norm. No VLA updates, classification scores, or added GE noise.
179
+ """
180
+
181
+ def norm(module):
182
+ return sum(
183
+ (
184
+ p.grad.detach().square().sum()
185
+ for p in module.parameters()
186
+ if p.grad is not None
187
+ ),
188
+ model.position.new_zeros(()),
189
+ ).sqrt()
190
+
191
+ action, touch = norm(model.action_projection), norm(model.touch_projection)
192
+ factor = (touch / action.clamp_min(1e-12)).clamp(0.1, 1.0)
193
+ for p in model.action_projection.parameters():
194
+ if p.grad is not None:
195
+ p.grad.mul_(factor)
196
+ return float(factor), float(action), float(touch)
model/tachin_paired30_cache_20260923/code/src/tactile_editor/trainability.py ADDED
@@ -0,0 +1,99 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Explicit trainable-component policies for controlled experiments."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from torch import nn
6
+
7
+ from .config import TrainMode, Variant
8
+ from .model import TactileMethodD, component_report
9
+
10
+
11
+ def _set(module: nn.Module | None, enabled: bool) -> None:
12
+ if module is None:
13
+ return
14
+ for parameter in module.parameters():
15
+ parameter.requires_grad = enabled
16
+
17
+
18
+ def configure_trainable_components(
19
+ model: TactileMethodD,
20
+ mode: TrainMode | str | None = None,
21
+ *,
22
+ expert_top_layers: int = 4,
23
+ ) -> dict[str, dict[str, int]]:
24
+ """Apply one parameter policy and return an auditable count by component."""
25
+
26
+ selected = TrainMode(mode or model.config.train_mode)
27
+ _set(model.method_d, False)
28
+ _set(model.tactile_encoder, False)
29
+ _set(model.fusion, False)
30
+
31
+ if selected == TrainMode.ADAPTER_ONLY:
32
+ _set(model.fusion, model.variant != Variant.V0_BASE)
33
+ elif selected == TrainMode.ENCODER_EDITOR:
34
+ if model.variant != Variant.V1_POSTHOC or model.tactile_encoder is None:
35
+ raise ValueError("encoder_editor requires V1 and a tactile encoder")
36
+ _set(model.tactile_encoder, True)
37
+ _set(model.fusion, True)
38
+ elif selected == TrainMode.EXPERT_TOP:
39
+ if expert_top_layers <= 0:
40
+ raise ValueError("expert_top_layers must be positive")
41
+ if expert_top_layers > len(model.method_d.expert.layers):
42
+ raise ValueError("expert_top_layers exceeds the expert depth")
43
+ _set(model.fusion, model.variant != Variant.V0_BASE)
44
+ layers = model.method_d.expert.layers
45
+ for layer in layers[-expert_top_layers:]:
46
+ _set(layer, True)
47
+ _set(model.method_d.expert.norm_out, True)
48
+ _set(model.method_d.expert.action_out_proj, True)
49
+ elif selected == TrainMode.ENCODER_TOP:
50
+ _set(model.fusion, model.variant != Variant.V0_BASE)
51
+ if model.tactile_encoder is None:
52
+ raise ValueError("encoder_top is not meaningful for V0")
53
+ # The published wrapper owns SelectedTactileEncoder as `.encoder`; that in
54
+ # turn owns the actual TactileEncoder as another `.encoder`.
55
+ selected_encoder = model.tactile_encoder.encoder
56
+ core = getattr(selected_encoder, "encoder", selected_encoder)
57
+ if not hasattr(core, "projection") or not hasattr(core, "gru"):
58
+ raise TypeError("encoder_top expects the published CNN+BiGRU encoder")
59
+ _set(core.projection, True)
60
+ for name, parameter in core.gru.named_parameters():
61
+ parameter.requires_grad = "_l1" in name
62
+ elif selected == TrainMode.JOINT:
63
+ _set(model.method_d, True)
64
+ _set(model.tactile_encoder, model.tactile_encoder is not None)
65
+ _set(model.fusion, model.fusion is not None)
66
+ else: # pragma: no cover - Enum makes this unreachable
67
+ raise ValueError(selected)
68
+ if not any(p.requires_grad for p in model.method_d.parameters()):
69
+ model.method_d.eval()
70
+ return component_report(model)
71
+
72
+
73
+ def encoder_editor_parameter_groups(
74
+ model: TactileMethodD, *, editor_lr: float, encoder_lr: float
75
+ ) -> list[dict]:
76
+ """Two disjoint optimizer groups; refuse accidental backbone updates."""
77
+ import math
78
+
79
+ if any(not math.isfinite(lr) or lr <= 0 for lr in (editor_lr, encoder_lr)):
80
+ raise ValueError("learning rates must be finite and positive")
81
+ if any(p.requires_grad for p in model.method_d.parameters()):
82
+ raise ValueError("Method D must remain frozen")
83
+ groups = []
84
+ for name, module, lr in (
85
+ ("editor", model.fusion, editor_lr),
86
+ ("encoder", model.tactile_encoder, encoder_lr),
87
+ ):
88
+ params = (
89
+ []
90
+ if module is None
91
+ else [p for p in module.parameters() if p.requires_grad]
92
+ )
93
+ if not params:
94
+ raise ValueError(f"{name} has no trainable parameters")
95
+ groups.append({"name": name, "params": params, "lr": lr})
96
+ ids = [id(p) for group in groups for p in group["params"]]
97
+ if len(ids) != len(set(ids)):
98
+ raise ValueError("optimizer groups overlap")
99
+ return groups
model/tachin_paired30_cache_20260923/code/src/tactile_editor/training.py ADDED
@@ -0,0 +1,180 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """V1 training primitives derived from the Kai-final VLA timebase."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import math
6
+ from dataclasses import dataclass
7
+
8
+ import torch
9
+ import torch.nn.functional as F
10
+
11
+ from .action_layout import FULL_ACTION_GROUPS
12
+ from .model import TactileMethodD
13
+
14
+
15
+ # Backwards-compatible name used by existing objectives and scripts.
16
+ ACTION_GROUPS = FULL_ACTION_GROUPS
17
+
18
+
19
+ def balanced_masked_mean(
20
+ values: torch.Tensor,
21
+ mask: torch.Tensor,
22
+ groups: tuple[tuple[int, int], ...] = ACTION_GROUPS,
23
+ ) -> torch.Tensor:
24
+ """Mean over values per group, present groups per sample, then valid samples.
25
+
26
+ Empty groups/samples are omitted; an entirely empty batch is an error.
27
+ Invalid values are selected away before reduction (including NaN padding).
28
+ """
29
+ if values.ndim != 3 or mask.shape != values.shape:
30
+ raise ValueError("values and mask must have matching [B,T,A] shapes")
31
+ occupied: set[int] = set()
32
+ means, present = [], []
33
+ for start, end in groups:
34
+ if not 0 <= start < end <= values.shape[-1]:
35
+ raise ValueError("action group outside the action dimension")
36
+ indices = set(range(start, end))
37
+ if occupied & indices:
38
+ raise ValueError("action groups must not overlap")
39
+ occupied |= indices
40
+ selected = mask[..., start:end].bool()
41
+ count = selected.sum((1, 2))
42
+ means.append(
43
+ values[..., start:end].masked_fill(~selected, 0).sum((1, 2))
44
+ / count.clamp_min(1)
45
+ )
46
+ present.append(count > 0)
47
+ if not groups:
48
+ raise ValueError("at least one action group is required")
49
+ valid = torch.stack(present, dim=1)
50
+ per_sample = torch.stack(means, dim=1).sum(1) / valid.sum(1).clamp_min(1)
51
+ nonempty = valid.any(1)
52
+ if not nonempty.any():
53
+ raise ValueError("the batch has no editable action values")
54
+ return per_sample[nonempty].mean()
55
+
56
+
57
+ @dataclass(frozen=True)
58
+ class V1TrainingBatch:
59
+ """One cached Method-D plan and its synchronized tactile supervision."""
60
+
61
+ base_action: torch.Tensor # [B,16,192], normalized frozen-VLA output
62
+ target_action: torch.Tensor # [B,16,192], normalized ground truth
63
+ action_mask: torch.Tensor # [B,16,192]
64
+ touch_pressure: torch.Tensor # [B,8,2,21,21]
65
+ touch_valid: torch.Tensor # [B,8,2]
66
+ decision_after_start_seconds: torch.Tensor # [B], offline plan-relative time
67
+
68
+ def validate(self) -> None:
69
+ if self.base_action.ndim != 3:
70
+ raise ValueError("base_action must be [B,T,A]")
71
+ if self.target_action.shape != self.base_action.shape:
72
+ raise ValueError("target_action and base_action must have the same shape")
73
+ if self.action_mask.shape != self.base_action.shape:
74
+ raise ValueError("action_mask and base_action must have the same shape")
75
+ batch = len(self.base_action)
76
+ if self.touch_pressure.shape != (batch, 8, 2, 21, 21):
77
+ raise ValueError("touch_pressure must be [B,8,2,21,21]")
78
+ if self.touch_valid.shape != (batch, 8, 2):
79
+ raise ValueError("touch_valid must be [B,8,2]")
80
+ if self.decision_after_start_seconds.shape != (batch,):
81
+ raise ValueError("decision_after_start_seconds must be [B]")
82
+
83
+
84
+ def build_future_action_mask(
85
+ action_mask: torch.Tensor,
86
+ decision_after_start_seconds: torch.Tensor,
87
+ *,
88
+ action_hz: float = 80.0,
89
+ ) -> torch.Tensor:
90
+ """Offline mask at the tactile decision time; no compute-latency model."""
91
+
92
+ if action_mask.ndim != 3:
93
+ raise ValueError("action_mask must be [B,T,A]")
94
+ if decision_after_start_seconds.shape != (len(action_mask),):
95
+ raise ValueError("decision_after_start_seconds must be [B]")
96
+ if not math.isfinite(action_hz) or action_hz <= 0:
97
+ raise ValueError("action_hz must be finite and positive")
98
+ ready = decision_after_start_seconds.to(
99
+ device=action_mask.device, dtype=torch.float64
100
+ )
101
+ if not torch.isfinite(ready).all() or (ready < 0).any():
102
+ raise ValueError("decision times must be finite and non-negative")
103
+ # Float32 seconds can put an exact tick a few ulps above its integer index.
104
+ tolerance = 1e-6 if decision_after_start_seconds.dtype != torch.float64 else 1e-9
105
+ committed = torch.ceil(ready * action_hz - tolerance).long()
106
+ committed = committed.clamp(max=action_mask.shape[1])
107
+ steps = torch.arange(action_mask.shape[1], device=action_mask.device)[None, :, None]
108
+ return action_mask.to(dtype=torch.bool) & (steps >= committed[:, None, None])
109
+
110
+
111
+ def v1_residual_objective(
112
+ model: TactileMethodD,
113
+ batch: V1TrainingBatch,
114
+ *,
115
+ action_hz: float = 80.0,
116
+ delta_l2_weight: float = 1e-4,
117
+ action_groups: tuple[tuple[int, int], ...] = ACTION_GROUPS,
118
+ ) -> tuple[torch.Tensor, dict[str, torch.Tensor]]:
119
+ """Train V1 on only the action positions that deployment can still change."""
120
+
121
+ batch.validate()
122
+ if delta_l2_weight < 0:
123
+ raise ValueError("delta_l2_weight must be non-negative")
124
+ edit_mask = build_future_action_mask(
125
+ batch.action_mask,
126
+ batch.decision_after_start_seconds,
127
+ action_hz=action_hz,
128
+ )
129
+ # Exclude all dimensions outside the specified physical groups, even if the
130
+ # supplied mask accidentally marks padding as valid.
131
+ component_mask = torch.zeros_like(edit_mask)
132
+ for start, end in action_groups:
133
+ if not 0 <= start < end <= edit_mask.shape[-1]:
134
+ raise ValueError("action group outside the action dimension")
135
+ component_mask[..., start:end] = True
136
+ edit_mask &= component_mask
137
+ base = batch.base_action.detach()
138
+ target = batch.target_action.detach()
139
+ if not torch.isfinite(base).all():
140
+ raise ValueError("base_action must be finite, including padding")
141
+ if not torch.isfinite(target[edit_mask]).all():
142
+ raise ValueError("editable target_action must be finite")
143
+ # Invalid GT can contain NaN; remove it before Smooth L1 so backward is safe.
144
+ target = torch.where(edit_mask, target, base)
145
+ revised = model.edit_existing_plan(
146
+ base,
147
+ edit_mask,
148
+ batch.touch_pressure,
149
+ batch.touch_valid,
150
+ canonicalized_touch=False,
151
+ )
152
+ per_value = F.smooth_l1_loss(revised, target, reduction="none")
153
+ action_loss = balanced_masked_mean(per_value, edit_mask, action_groups)
154
+ delta = revised - base
155
+ delta_l2 = balanced_masked_mean(delta.square(), edit_mask, action_groups)
156
+ loss = action_loss + delta_l2_weight * delta_l2
157
+ with torch.no_grad():
158
+ base_error = balanced_masked_mean(
159
+ F.smooth_l1_loss(base, target, reduction="none"), edit_mask, action_groups
160
+ )
161
+ metrics = {
162
+ "loss": loss.detach(),
163
+ "action_loss": action_loss.detach(),
164
+ "delta_l2": delta_l2.detach(),
165
+ "base_action_loss": base_error.detach(),
166
+ "editable_fraction": edit_mask.float().mean().detach(),
167
+ "supervised_samples": edit_mask.any(dim=(1, 2)).sum().detach(),
168
+ }
169
+ return loss, metrics
170
+
171
+
172
+ def nominal_revision_times(
173
+ *, tactile_hz: float = 30.0, control_hz: float = 10.0
174
+ ) -> tuple[float, ...]:
175
+ """Return tactile decision times inside one VLA control period."""
176
+
177
+ if tactile_hz <= 0 or control_hz <= 0:
178
+ raise ValueError("rates must be positive")
179
+ count = max(1, math.ceil(tactile_hz / control_hz))
180
+ return tuple(index / tactile_hz for index in range(count))
model/tachin_paired30_cache_20260923/code/src/tactile_editor/unified_cache.py ADDED
@@ -0,0 +1,122 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Memory-mapped cache shared by the W0--W3 comparison."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import bisect
6
+ import hashlib
7
+ import json
8
+ from pathlib import Path
9
+
10
+ import numpy as np
11
+ import torch
12
+ from torch.utils.data import Dataset
13
+
14
+
15
+ COMMON_ARRAYS = (
16
+ "base_action",
17
+ "target_action",
18
+ "action_mask",
19
+ "touch_pressure",
20
+ "touch_valid",
21
+ "decision_seconds",
22
+ "recording",
23
+ "final_memory",
24
+ "future_touch_pressure",
25
+ "future_touch_valid",
26
+ "future_source_frame",
27
+ )
28
+
29
+ RGB_ARRAYS = (
30
+ "image_tokens",
31
+ "clean_latent",
32
+ "rgb_valid",
33
+ "future_roi",
34
+ )
35
+
36
+
37
+ class UnifiedWorldCache(Dataset):
38
+ """Read one immutable row set for every W0--W3 experiment.
39
+
40
+ A cache row is admitted only if it has an editable tactile observation, an
41
+ exact +0.1-s tactile target, and valid RGB targets/ROI. Even W0 and W2 use
42
+ this same intersection so their reported action scores are paired with W1
43
+ and W3 sample by sample.
44
+ """
45
+
46
+ def __init__(
47
+ self,
48
+ root: str | Path,
49
+ split: str,
50
+ *,
51
+ include_rgb: bool,
52
+ expected_shards: int = 8,
53
+ ) -> None:
54
+ self.root = Path(root)
55
+ self.split = split
56
+ self.include_rgb = include_rgb
57
+ self.names = COMMON_ARRAYS + (RGB_ARRAYS if include_rgb else ())
58
+ self.shards: list[tuple[int, dict[str, np.ndarray]]] = []
59
+ self.ends: list[int] = []
60
+ self.metadata: list[dict] = []
61
+ total = 0
62
+ for shard_root in sorted(self.root.glob("shard_*")):
63
+ split_root = shard_root / split
64
+ metadata_path = split_root / "metadata.json"
65
+ if not metadata_path.is_file():
66
+ continue
67
+ metadata = json.loads(metadata_path.read_text())
68
+ if int(metadata.get("cache_schema", -1)) != 3:
69
+ raise RuntimeError(f"not a unified W0--W3 cache: {split_root}")
70
+ if metadata.get("decision_ticks") != [0]:
71
+ raise RuntimeError("W0--W3 RGB comparison requires decision tick 0")
72
+ count = int(metadata["valid_samples"])
73
+ missing = [name for name in self.names if not (split_root / f"{name}.npy").is_file()]
74
+ if missing:
75
+ raise RuntimeError(f"missing {missing} in {split_root}")
76
+ arrays = {
77
+ name: np.load(split_root / f"{name}.npy", mmap_mode="r")
78
+ for name in self.names
79
+ }
80
+ if any(len(array) < count for array in arrays.values()):
81
+ raise RuntimeError(f"array shorter than metadata in {split_root}")
82
+ self.shards.append((count, arrays))
83
+ self.metadata.append(metadata)
84
+ total += count
85
+ self.ends.append(total)
86
+ if len(self.shards) != expected_shards:
87
+ raise RuntimeError(
88
+ f"expected {expected_shards} cache shards, found {len(self.shards)}"
89
+ )
90
+ fingerprints = {item["contract_fingerprint"] for item in self.metadata}
91
+ if len(fingerprints) != 1:
92
+ raise RuntimeError("cache shards have different data/model contracts")
93
+ self.contract_fingerprint = fingerprints.pop()
94
+
95
+ def __len__(self) -> int:
96
+ return self.ends[-1]
97
+
98
+ def __getitem__(self, index: int) -> dict[str, torch.Tensor]:
99
+ if index < 0:
100
+ index += len(self)
101
+ if not 0 <= index < len(self):
102
+ raise IndexError(index)
103
+ shard = bisect.bisect_right(self.ends, index)
104
+ start = 0 if shard == 0 else self.ends[shard - 1]
105
+ arrays = self.shards[shard][1]
106
+ result = {
107
+ name: torch.from_numpy(np.array(array[index - start], copy=True))
108
+ for name, array in arrays.items()
109
+ }
110
+ result["index"] = torch.tensor(index, dtype=torch.long)
111
+ return result
112
+
113
+ def identity(self) -> dict:
114
+ counts = [count for count, _ in self.shards]
115
+ payload = {
116
+ "root": str(self.root.resolve()),
117
+ "split": self.split,
118
+ "counts": counts,
119
+ "contract_fingerprint": self.contract_fingerprint,
120
+ }
121
+ canonical = json.dumps(payload, sort_keys=True).encode()
122
+ return {**payload, "identity_sha256": hashlib.sha256(canonical).hexdigest()}
model/tachin_paired30_cache_20260923/code/src/tactile_editor/visual_dynamics_pilot.py ADDED
@@ -0,0 +1,215 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Cheap frozen-backbone pilot for visual world-model design choices.
2
+
3
+ This module is deliberately smaller than T-WLA. It consumes cached final-memory
4
+ tokens from the fixed ten-domain 110k checkpoint and trains one shared bottleneck
5
+ that serves both an action residual head and a visual-dynamics head. It is a
6
+ screening experiment: a positive result justifies an expensive end-to-end run; a
7
+ negative result prevents spending that budget.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ from dataclasses import dataclass
13
+
14
+ import torch
15
+ from torch import nn
16
+
17
+ from .action_layout import DEFAULT_ACTION_LAYOUT, FULL_ACTION_GROUPS, VALID_ACTION_DIM
18
+ from .action_objectives import per_sample_action_loss
19
+
20
+
21
+ @dataclass
22
+ class VisualDynamicsOutput:
23
+ action: torch.Tensor
24
+ delta: torch.Tensor
25
+ shared: torch.Tensor
26
+ future_latent: torch.Tensor | None
27
+
28
+
29
+ class VisualDynamicsPilot(nn.Module):
30
+ """Shared memory adapter with optional action-conditioned visual dynamics.
31
+
32
+ ``world_condition`` is one of:
33
+
34
+ * ``none``: action-only control; no visual head is evaluated.
35
+ * ``memory``: the visual head reads the shared memory bottleneck only.
36
+ * ``predicted_action``: it additionally reads the differentiable action
37
+ prediction, so the visual loss can directly regulate the action head.
38
+ """
39
+
40
+ def __init__(
41
+ self,
42
+ *,
43
+ memory_tokens: int = 8,
44
+ memory_dim: int = 960,
45
+ hidden_dim: int = 384,
46
+ action_hidden_dim: int = 128,
47
+ time_dim: int = 32,
48
+ horizons: int = 2,
49
+ latent_channels: int = 32,
50
+ latent_size: int = 16,
51
+ residual_limit: float = 0.5,
52
+ world_condition: str = "none",
53
+ world_step_ranges: tuple[tuple[int, int], ...] | None = None,
54
+ ) -> None:
55
+ super().__init__()
56
+ if world_condition not in {
57
+ "none", "memory", "predicted_action", "predicted_action_only",
58
+ }:
59
+ raise ValueError(f"unknown world_condition: {world_condition}")
60
+ if residual_limit <= 0:
61
+ raise ValueError("residual_limit must be positive")
62
+ self.world_condition = world_condition
63
+ self.residual_limit = float(residual_limit)
64
+ self.horizons = horizons
65
+ self.latent_channels = latent_channels
66
+ self.latent_size = latent_size
67
+ self.world_step_ranges = world_step_ranges or tuple(
68
+ (0, steps) for steps in (8, 16)[:horizons]
69
+ )
70
+ if len(self.world_step_ranges) != horizons:
71
+ raise ValueError("one world action-step range is required per horizon")
72
+ if any(start < 0 or stop <= start or stop > 16 for start, stop in self.world_step_ranges):
73
+ raise ValueError("world action-step ranges must satisfy 0 <= start < stop <= 16")
74
+
75
+ self.memory_norm = nn.LayerNorm(memory_dim)
76
+ self.memory_encoder = nn.Sequential(
77
+ nn.Linear(memory_tokens * memory_dim, hidden_dim),
78
+ nn.GELU(),
79
+ nn.LayerNorm(hidden_dim),
80
+ nn.Linear(hidden_dim, hidden_dim),
81
+ nn.GELU(),
82
+ )
83
+ self.action_time = nn.Parameter(torch.zeros(16, time_dim))
84
+ self.action_head = nn.Sequential(
85
+ nn.Linear(hidden_dim + time_dim + VALID_ACTION_DIM, hidden_dim),
86
+ nn.GELU(),
87
+ nn.Linear(hidden_dim, VALID_ACTION_DIM),
88
+ )
89
+
90
+ if world_condition != "none":
91
+ self.action_encoder = nn.Sequential(
92
+ nn.Linear(VALID_ACTION_DIM, action_hidden_dim),
93
+ nn.GELU(),
94
+ nn.Linear(action_hidden_dim, action_hidden_dim),
95
+ )
96
+ self.horizon_embedding = nn.Parameter(torch.zeros(horizons, time_dim))
97
+ self.world_decoder = nn.Sequential(
98
+ nn.Linear(hidden_dim + action_hidden_dim + time_dim, 512),
99
+ nn.GELU(),
100
+ nn.Linear(512, latent_channels * latent_size * latent_size),
101
+ )
102
+
103
+ def forward(
104
+ self,
105
+ base_action: torch.Tensor,
106
+ action_mask: torch.Tensor,
107
+ final_memory: torch.Tensor,
108
+ *,
109
+ compute_world: bool = True,
110
+ ) -> VisualDynamicsOutput:
111
+ DEFAULT_ACTION_LAYOUT.validate(base_action, name="base_action")
112
+ DEFAULT_ACTION_LAYOUT.validate(action_mask, name="action_mask")
113
+ if final_memory.shape != (len(base_action), 8, 960):
114
+ raise ValueError("final_memory must be [B,8,960]")
115
+
116
+ memory = self.memory_norm(final_memory.float()).flatten(1)
117
+ shared = self.memory_encoder(memory)
118
+ times = self.action_time[None].expand(len(base_action), -1, -1)
119
+ shared_steps = shared[:, None].expand(-1, 16, -1)
120
+ action_input = torch.cat(
121
+ [base_action[..., :VALID_ACTION_DIM].float(), shared_steps, times], dim=-1
122
+ )
123
+ raw_delta = self.residual_limit * torch.tanh(self.action_head(action_input))
124
+ physical_mask = action_mask.bool().clone()
125
+ physical_mask[..., VALID_ACTION_DIM:] = False
126
+ delta = torch.zeros_like(base_action.float())
127
+ delta[..., :VALID_ACTION_DIM] = raw_delta
128
+ delta = delta * physical_mask.to(delta.dtype)
129
+ action = base_action.float().detach() + delta
130
+
131
+ future = None
132
+ if self.world_condition != "none" and compute_world:
133
+ if self.world_condition in {"predicted_action", "predicted_action_only"}:
134
+ step_features = self.action_encoder(action[..., :VALID_ACTION_DIM])
135
+ valid_steps = physical_mask[..., :VALID_ACTION_DIM].any(-1)
136
+ else:
137
+ step_features = torch.zeros(
138
+ len(action), 16, self.action_encoder[-1].out_features,
139
+ device=action.device, dtype=shared.dtype,
140
+ )
141
+ valid_steps = torch.ones(
142
+ len(action), 16, device=action.device, dtype=torch.bool
143
+ )
144
+
145
+ predictions = []
146
+ for horizon, (start, stop) in enumerate(self.world_step_ranges):
147
+ valid = valid_steps[:, start:stop].to(step_features.dtype)
148
+ context = (
149
+ step_features[:, start:stop] * valid[..., None]
150
+ ).sum(1) / valid.sum(1, keepdim=True).clamp_min(1)
151
+ world_memory = (
152
+ torch.zeros_like(shared)
153
+ if self.world_condition == "predicted_action_only"
154
+ else shared
155
+ )
156
+ condition = torch.cat(
157
+ [
158
+ world_memory,
159
+ context,
160
+ self.horizon_embedding[horizon][None].expand(len(action), -1),
161
+ ],
162
+ dim=-1,
163
+ )
164
+ predictions.append(
165
+ self.world_decoder(condition).reshape(
166
+ len(action), self.latent_channels, self.latent_size, self.latent_size
167
+ )
168
+ )
169
+ future = torch.stack(predictions, dim=1)
170
+ return VisualDynamicsOutput(action, delta, shared, future)
171
+
172
+
173
+ def visual_dynamics_losses(
174
+ output: VisualDynamicsOutput,
175
+ target_action: torch.Tensor,
176
+ action_mask: torch.Tensor,
177
+ *,
178
+ future_latent: torch.Tensor | None = None,
179
+ future_valid: torch.Tensor | None = None,
180
+ future_roi: torch.Tensor | None = None,
181
+ background_weight: float = 0.05,
182
+ delta_l2_weight: float = 1e-4,
183
+ ) -> dict[str, torch.Tensor]:
184
+ """Return action and ROI-weighted visual losses without combining them."""
185
+
186
+ mask = action_mask.bool().clone()
187
+ mask[..., VALID_ACTION_DIM:] = False
188
+ action = per_sample_action_loss(
189
+ output.action, target_action.float(), mask, groups=FULL_ACTION_GROUPS
190
+ ).mean()
191
+ delta_l2 = output.delta.square().masked_fill(~mask, 0).sum() / mask.sum().clamp_min(1)
192
+ action_total = action + delta_l2_weight * delta_l2
193
+
194
+ world = action.new_zeros(())
195
+ if output.future_latent is not None:
196
+ if future_latent is None or future_valid is None or future_roi is None:
197
+ raise ValueError("future latent, validity, and ROI are required")
198
+ if future_latent.shape != output.future_latent.shape:
199
+ raise ValueError("future latent shape does not match the prediction")
200
+ if not 0 <= background_weight <= 1:
201
+ raise ValueError("background_weight must be in [0,1]")
202
+ valid = future_valid.bool()
203
+ roi = future_roi.bool()
204
+ weights = background_weight + (1.0 - background_weight) * roi.float()
205
+ weights = weights * valid[:, :, None, None].float()
206
+ squared = (output.future_latent.float() - future_latent.float()).square()
207
+ numerator = (squared * weights[:, :, None]).sum()
208
+ denominator = weights.sum() * squared.shape[2]
209
+ world = numerator / denominator.clamp_min(1)
210
+ return {
211
+ "action": action,
212
+ "action_total": action_total,
213
+ "delta_l2": delta_l2,
214
+ "world": world,
215
+ }
model/tachin_paired30_cache_20260923/code/src/tactile_editor/world_residual_probe.py ADDED
@@ -0,0 +1,362 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Leakage-safe statistics for the Task-35 world/action residual pilot.
2
+
3
+ The module deliberately contains no model-training code. It consumes residuals
4
+ materialized by a frozen Stage-1 action calibrator and a frozen tactile-free W0
5
+ world model, then fits every transform on train only.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from dataclasses import dataclass
11
+ import math
12
+
13
+ import numpy as np
14
+
15
+
16
+ LEFT_ARTICULATION = slice(6, 69)
17
+ RIGHT_ARTICULATION = slice(75, 138)
18
+ LEFT_HAND = slice(0, 69)
19
+ RIGHT_HAND = slice(69, 138)
20
+
21
+
22
+ def rectified_flow_sample(
23
+ clean: np.ndarray, noise: np.ndarray, tau: np.ndarray
24
+ ) -> tuple[np.ndarray, np.ndarray]:
25
+ """Return z_tau and the clean-to-noise velocity under the V4 convention."""
26
+
27
+ clean = np.asarray(clean)
28
+ noise = np.asarray(noise)
29
+ tau = np.asarray(tau)
30
+ if clean.shape != noise.shape:
31
+ raise ValueError("clean and noise must have identical shapes")
32
+ if np.any(~np.isfinite(tau)) or np.any((tau < 0) | (tau > 1)):
33
+ raise ValueError("tau must be finite and in [0,1]")
34
+ while tau.ndim < clean.ndim:
35
+ tau = tau[..., None]
36
+ return (1.0 - tau) * clean + tau * noise, noise - clean
37
+
38
+
39
+ def action_window_start(decision_seconds: np.ndarray, action_hz: float = 80.0) -> np.ndarray:
40
+ """Match ``build_future_action_mask`` including its float32 tolerance."""
41
+
42
+ seconds = np.asarray(decision_seconds)
43
+ if np.any(~np.isfinite(seconds)) or np.any(seconds < 0):
44
+ raise ValueError("decision_seconds must be finite and non-negative")
45
+ tolerance = 1e-6 if seconds.dtype != np.float64 else 1e-9
46
+ return np.ceil(seconds.astype(np.float64) * action_hz - tolerance).astype(np.int64)
47
+
48
+
49
+ def pool_visual_residual(residual: np.ndarray, output_size: int = 4) -> np.ndarray:
50
+ """Average-pool ``[...,C,H,W]`` residual maps and flatten non-batch axes."""
51
+
52
+ value = np.asarray(residual, dtype=np.float32)
53
+ if value.ndim < 4:
54
+ raise ValueError("visual residual must end in [C,H,W]")
55
+ height, width = value.shape[-2:]
56
+ if height % output_size or width % output_size:
57
+ raise ValueError("spatial dimensions must be divisible by output_size")
58
+ value = value.reshape(
59
+ *value.shape[:-2], output_size, height // output_size,
60
+ output_size, width // output_size,
61
+ ).mean(axis=(-3, -1))
62
+ return value.reshape(len(value), -1)
63
+
64
+
65
+ @dataclass(frozen=True)
66
+ class HandRows:
67
+ visual: np.ndarray
68
+ articulation: np.ndarray
69
+ hand_action: np.ndarray
70
+ source_row: np.ndarray
71
+ hand: np.ndarray
72
+ recording: np.ndarray
73
+ timestamp: np.ndarray
74
+ subsets: dict[str, np.ndarray]
75
+
76
+
77
+ def build_hand_rows(
78
+ visual: np.ndarray,
79
+ anchor_action: np.ndarray,
80
+ target_action: np.ndarray,
81
+ action_mask: np.ndarray,
82
+ decision_seconds: np.ndarray,
83
+ recording: np.ndarray,
84
+ *,
85
+ absolute_seconds: np.ndarray | None = None,
86
+ event_masks: dict[str, np.ndarray] | None = None,
87
+ horizon: int = 8,
88
+ action_hz: float = 80.0,
89
+ ) -> HandRows:
90
+ """Create one fully observed, time-aligned row per supervised hand.
91
+
92
+ A decision at 33.3 ms starts at action index 3, not index 0. The selected
93
+ eight-step residual therefore always covers the same physical 0.1-second
94
+ interval as the future RGB target.
95
+ """
96
+
97
+ visual = np.asarray(visual, dtype=np.float32)
98
+ anchor = np.asarray(anchor_action, dtype=np.float32)
99
+ target = np.asarray(target_action, dtype=np.float32)
100
+ mask = np.asarray(action_mask, dtype=bool)
101
+ seconds = np.asarray(decision_seconds)
102
+ timestamps = seconds if absolute_seconds is None else np.asarray(absolute_seconds)
103
+ recordings = np.asarray(recording)
104
+ n = len(visual)
105
+ if anchor.shape != target.shape or anchor.shape != mask.shape:
106
+ raise ValueError("anchor, target, and mask must have identical shapes")
107
+ if anchor.ndim != 3 or anchor.shape[1:] != (16, 192):
108
+ raise ValueError("actions must be [N,16,192]")
109
+ if any(len(value) != n for value in (anchor, seconds, timestamps, recordings)):
110
+ raise ValueError("all arrays must have the same leading dimension")
111
+ starts = action_window_start(seconds, action_hz)
112
+ specs = ((LEFT_ARTICULATION, LEFT_HAND), (RIGHT_ARTICULATION, RIGHT_HAND))
113
+ rows: list[int] = []
114
+ hands: list[int] = []
115
+ articulation: list[np.ndarray] = []
116
+ hand_action: list[np.ndarray] = []
117
+ for row, start in enumerate(starts):
118
+ stop = int(start) + horizon
119
+ if stop > anchor.shape[1]:
120
+ continue
121
+ for hand, (art_slice, hand_slice) in enumerate(specs):
122
+ # CCA and ordinary ridge require a fixed fully observed target.
123
+ if not mask[row, start:stop, art_slice].all():
124
+ continue
125
+ residual = target[row, start:stop] - anchor[row, start:stop]
126
+ rows.append(row)
127
+ hands.append(hand)
128
+ articulation.append(residual[:, art_slice].reshape(-1))
129
+ hand_action.append(residual[:, hand_slice].reshape(-1))
130
+ if not rows:
131
+ raise ValueError("no hand has a fully valid aligned action horizon")
132
+ source = np.asarray(rows, dtype=np.int64)
133
+ hand_index = np.asarray(hands, dtype=np.int8)
134
+ subsets: dict[str, np.ndarray] = {}
135
+ for name, values in (event_masks or {}).items():
136
+ value = np.asarray(values, dtype=bool)
137
+ if value.shape == (n,):
138
+ subsets[name] = value[source]
139
+ elif value.shape == (n, 2):
140
+ subsets[name] = value[source, hand_index]
141
+ else:
142
+ raise ValueError(f"event mask {name!r} must be [N] or [N,2]")
143
+ return HandRows(
144
+ visual=visual[source],
145
+ articulation=np.stack(articulation),
146
+ hand_action=np.stack(hand_action),
147
+ source_row=source,
148
+ hand=hand_index,
149
+ recording=recordings[source],
150
+ timestamp=timestamps[source],
151
+ subsets=subsets,
152
+ )
153
+
154
+
155
+ @dataclass(frozen=True)
156
+ class PCATransform:
157
+ mean: np.ndarray
158
+ scale: np.ndarray
159
+ components: np.ndarray
160
+ explained_variance: np.ndarray
161
+
162
+ def transform(self, value: np.ndarray) -> np.ndarray:
163
+ standardized = (np.asarray(value, dtype=np.float64) - self.mean) / self.scale
164
+ return standardized @ self.components
165
+
166
+
167
+ def fit_pca(
168
+ value: np.ndarray,
169
+ components: int = 64,
170
+ *,
171
+ seed: int = 20260911,
172
+ oversample: int = 12,
173
+ power_iterations: int = 2,
174
+ ) -> PCATransform:
175
+ """Fit randomized PCA without materializing a feature covariance matrix."""
176
+
177
+ x = np.asarray(value, dtype=np.float64)
178
+ if x.ndim != 2 or len(x) < 3 or not np.isfinite(x).all():
179
+ raise ValueError("PCA input must be a finite [N,D] matrix")
180
+ mean = x.mean(0)
181
+ scale = x.std(0, ddof=1)
182
+ scale[scale < 1e-8] = 1.0
183
+ z = (x - mean) / scale
184
+ rank = min(int(components), len(x) - 1, x.shape[1])
185
+ sketch = min(rank + oversample, len(x), x.shape[1])
186
+ rng = np.random.default_rng(seed)
187
+ omega = rng.standard_normal((x.shape[1], sketch))
188
+ q = z @ omega
189
+ for _ in range(power_iterations):
190
+ q = z @ (z.T @ q)
191
+ q, _ = np.linalg.qr(q, mode="reduced")
192
+ _, singular, vt = np.linalg.svd(q.T @ z, full_matrices=False)
193
+ basis = vt[:rank].T
194
+ variance = singular[:rank] ** 2 / max(len(x) - 1, 1)
195
+ return PCATransform(mean, scale, basis, variance)
196
+
197
+
198
+ def _inverse_sqrt(matrix: np.ndarray) -> np.ndarray:
199
+ values, vectors = np.linalg.eigh(matrix)
200
+ values = np.maximum(values, 1e-12)
201
+ return (vectors / np.sqrt(values)[None, :]) @ vectors.T
202
+
203
+
204
+ @dataclass(frozen=True)
205
+ class CCAModel:
206
+ x_mean: np.ndarray
207
+ y_mean: np.ndarray
208
+ x_weights: np.ndarray
209
+ y_weights: np.ndarray
210
+ singular_values: np.ndarray
211
+ ridge: float
212
+
213
+ def transform(self, x: np.ndarray, y: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
214
+ return (
215
+ (np.asarray(x, dtype=np.float64) - self.x_mean) @ self.x_weights,
216
+ (np.asarray(y, dtype=np.float64) - self.y_mean) @ self.y_weights,
217
+ )
218
+
219
+
220
+ def fit_regularized_cca(
221
+ x: np.ndarray, y: np.ndarray, *, ridge: float, components: int = 16
222
+ ) -> CCAModel:
223
+ x = np.asarray(x, dtype=np.float64)
224
+ y = np.asarray(y, dtype=np.float64)
225
+ if x.ndim != 2 or y.ndim != 2 or len(x) != len(y) or len(x) < 3:
226
+ raise ValueError("CCA inputs must be paired [N,D] matrices")
227
+ xm, ym = x.mean(0), y.mean(0)
228
+ xc, yc = x - xm, y - ym
229
+ denominator = max(len(x) - 1, 1)
230
+ cxx = xc.T @ xc / denominator + ridge * np.eye(x.shape[1])
231
+ cyy = yc.T @ yc / denominator + ridge * np.eye(y.shape[1])
232
+ cxy = xc.T @ yc / denominator
233
+ inv_x = _inverse_sqrt(cxx)
234
+ inv_y = _inverse_sqrt(cyy)
235
+ left, singular, right_t = np.linalg.svd(inv_x @ cxy @ inv_y, full_matrices=False)
236
+ rank = min(components, len(singular))
237
+ return CCAModel(
238
+ xm, ym, inv_x @ left[:, :rank], inv_y @ right_t.T[:, :rank],
239
+ singular[:rank], float(ridge),
240
+ )
241
+
242
+
243
+ def canonical_correlations(x_scores: np.ndarray, y_scores: np.ndarray) -> np.ndarray:
244
+ if x_scores.shape != y_scores.shape:
245
+ raise ValueError("canonical score matrices must match")
246
+ x = x_scores - x_scores.mean(0)
247
+ y = y_scores - y_scores.mean(0)
248
+ numerator = (x * y).sum(0)
249
+ denominator = np.sqrt(((x * x).sum(0)) * ((y * y).sum(0)))
250
+ return np.divide(numerator, denominator, out=np.zeros_like(numerator), where=denominator > 0)
251
+
252
+
253
+ @dataclass(frozen=True)
254
+ class RidgeModel:
255
+ x_mean: np.ndarray
256
+ y_mean: np.ndarray
257
+ weights: np.ndarray
258
+ alpha: float
259
+
260
+ def predict(self, x: np.ndarray) -> np.ndarray:
261
+ return (np.asarray(x, dtype=np.float64) - self.x_mean) @ self.weights + self.y_mean
262
+
263
+
264
+ def fit_ridge(x: np.ndarray, y: np.ndarray, alpha: float) -> RidgeModel:
265
+ x = np.asarray(x, dtype=np.float64)
266
+ y = np.asarray(y, dtype=np.float64)
267
+ xm, ym = x.mean(0), y.mean(0)
268
+ xc, yc = x - xm, y - ym
269
+ gram = xc.T @ xc + alpha * np.eye(x.shape[1])
270
+ weights = np.linalg.solve(gram, xc.T @ yc)
271
+ return RidgeModel(xm, ym, weights, float(alpha))
272
+
273
+
274
+ def r2_scores(target: np.ndarray, prediction: np.ndarray) -> dict[str, float | list[float]]:
275
+ target = np.asarray(target, dtype=np.float64)
276
+ prediction = np.asarray(prediction, dtype=np.float64)
277
+ if target.shape != prediction.shape or target.ndim != 2:
278
+ raise ValueError("target and prediction must be matching [N,D] matrices")
279
+ residual = ((target - prediction) ** 2).sum(0)
280
+ total = ((target - target.mean(0)) ** 2).sum(0)
281
+ per_dimension = np.divide(
282
+ residual, total, out=np.full_like(residual, np.nan), where=total > 1e-12
283
+ )
284
+ per_dimension = 1.0 - per_dimension
285
+ valid = np.isfinite(per_dimension)
286
+ uniform = float(per_dimension[valid].mean()) if valid.any() else math.nan
287
+ variance_weighted = float(1.0 - residual.sum() / total.sum()) if total.sum() else math.nan
288
+ return {
289
+ "uniform_average": uniform,
290
+ "variance_weighted": variance_weighted,
291
+ "per_dimension": per_dimension.tolist(),
292
+ }
293
+
294
+
295
+ def within_recording_donors(
296
+ recording: np.ndarray,
297
+ hand: np.ndarray,
298
+ timestamp: np.ndarray,
299
+ *,
300
+ seed: int,
301
+ minimum_separation: float = 1.0,
302
+ ) -> tuple[np.ndarray, np.ndarray]:
303
+ """Sample same-recording/same-hand donors at least one second away."""
304
+
305
+ recording = np.asarray(recording)
306
+ hand = np.asarray(hand)
307
+ timestamp = np.asarray(timestamp, dtype=np.float64)
308
+ rng = np.random.default_rng(seed)
309
+ donors = np.arange(len(recording), dtype=np.int64)
310
+ eligible = np.zeros(len(recording), dtype=bool)
311
+ groups: dict[tuple[object, int], list[int]] = {}
312
+ for row, key in enumerate(zip(recording.tolist(), hand.tolist())):
313
+ groups.setdefault(key, []).append(row)
314
+ for indices in groups.values():
315
+ group = np.asarray(indices, dtype=np.int64)
316
+ times = timestamp[group]
317
+ for local, row in enumerate(group):
318
+ candidates = group[np.abs(times - times[local]) >= minimum_separation]
319
+ if len(candidates):
320
+ donors[row] = int(rng.choice(candidates))
321
+ eligible[row] = True
322
+ return donors, eligible
323
+
324
+
325
+ def cross_recording_donors(
326
+ recording: np.ndarray, hand: np.ndarray, *, seed: int
327
+ ) -> np.ndarray:
328
+ """Sample a different-recording donor with the same hand identity."""
329
+
330
+ recording = np.asarray(recording)
331
+ hand = np.asarray(hand)
332
+ rng = np.random.default_rng(seed)
333
+ donors = np.empty(len(recording), dtype=np.int64)
334
+ by_hand = {value: np.flatnonzero(hand == value) for value in np.unique(hand)}
335
+ for row in range(len(recording)):
336
+ candidates = by_hand[hand[row]]
337
+ candidates = candidates[recording[candidates] != recording[row]]
338
+ if not len(candidates):
339
+ raise ValueError("no cross-recording donor with matching hand")
340
+ donors[row] = int(rng.choice(candidates))
341
+ return donors
342
+
343
+
344
+ def cluster_bootstrap_interval(
345
+ values: np.ndarray,
346
+ recording: np.ndarray,
347
+ statistic,
348
+ *,
349
+ samples: int = 2000,
350
+ seed: int = 20260911,
351
+ ) -> tuple[float, float]:
352
+ """Bootstrap complete recordings and return a percentile interval."""
353
+
354
+ recordings = np.unique(recording)
355
+ groups = {value: np.flatnonzero(recording == value) for value in recordings}
356
+ rng = np.random.default_rng(seed)
357
+ estimates = np.empty(samples, dtype=np.float64)
358
+ for draw in range(samples):
359
+ selected = rng.choice(recordings, size=len(recordings), replace=True)
360
+ indices = np.concatenate([groups[value] for value in selected])
361
+ estimates[draw] = float(statistic(values, indices))
362
+ return tuple(np.quantile(estimates, [0.025, 0.975]).tolist())
model/tachin_paired30_cache_20260923/code/tests/test_tachin_native_editor.py ADDED
@@ -0,0 +1,63 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+
3
+ from tactile_editor.native_tachin import (NativeResidualEditor, NativeTachinEncoder,
4
+ NativeTachinW3Editor, SENSOR_OFFSETS,
5
+ SENSOR_SHAPES)
6
+
7
+
8
+ def inputs(batch=2):
9
+ mean = torch.linspace(0, 0.2, 1760).reshape(2, 880)
10
+ scale = torch.ones(2, 880) * 0.1
11
+ pressure = mean[None, None].expand(batch, 8, -1, -1).clone()
12
+ cell_valid = torch.ones_like(pressure, dtype=torch.bool)
13
+ sample_valid = torch.ones(batch, 8, 2, dtype=torch.bool)
14
+ return mean, scale, pressure, cell_valid, sample_valid
15
+
16
+
17
+ def test_native_regions_cover_each_cell_once():
18
+ assert SENSOR_OFFSETS[0] == 0 and SENSOR_OFFSETS[-1] == 880
19
+ assert all(stop - start == height * width for (height, width), start, stop
20
+ in zip(SENSOR_SHAPES, SENSOR_OFFSETS[:-1], SENSOR_OFFSETS[1:]))
21
+
22
+
23
+ def test_reference_touch_is_exact_base_for_memory_free_editor():
24
+ mean, scale, pressure, cell_valid, sample_valid = inputs()
25
+ model = NativeResidualEditor(NativeTachinEncoder(mean, scale)).eval()
26
+ base = torch.randn(2, 16, 192)
27
+ mask = torch.ones_like(base, dtype=torch.bool)
28
+ final, delta, allowed, features, hand_valid = model(
29
+ base, mask, pressure, cell_valid, sample_valid)
30
+ assert torch.equal(final, base)
31
+ assert not delta.bool().any()
32
+ assert not features.bool().any()
33
+ assert allowed[..., :138].all() and not allowed[..., 138:].any()
34
+ assert hand_valid.all()
35
+
36
+
37
+ def test_reference_touch_stays_exact_after_coefficient_bias_is_trained():
38
+ mean, scale, pressure, cell_valid, sample_valid = inputs()
39
+ pressure = mean[None, None].expand_as(pressure).clone()
40
+ model = NativeResidualEditor(NativeTachinEncoder(mean, scale)).eval()
41
+ with torch.no_grad():
42
+ model.coefficients[0].bias.fill_(0.5)
43
+ model.action.basis_head.bias.fill_(0.5)
44
+ base = torch.randn(2, 16, 192)
45
+ final, delta, _, features, _ = model(
46
+ base, torch.ones_like(base, dtype=torch.bool), pressure,
47
+ cell_valid, sample_valid,
48
+ )
49
+ assert not features.bool().any()
50
+ assert not delta.bool().any()
51
+ assert torch.equal(final, base)
52
+
53
+
54
+ def test_native_w3_no_auxiliary_shape_and_reference_invariance():
55
+ mean, scale, pressure, cell_valid, sample_valid = inputs(batch=1)
56
+ model = NativeTachinW3Editor(NativeTachinEncoder(mean, scale), mean, scale).eval()
57
+ base = torch.randn(1, 16, 192)
58
+ output = model(base_action=base, edit_mask=torch.ones_like(base, dtype=torch.bool),
59
+ pressure=pressure, cell_valid=cell_valid,
60
+ sample_valid=sample_valid, final_memory=torch.randn(1, 8, 960),
61
+ compute_auxiliary=False)
62
+ assert torch.equal(output.final_action, base)
63
+ assert output.tactile_velocity is None and output.rgb is None
model/tachin_paired30_cache_20260923/environment.txt ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ python==3.12.3
2
+ torch==2.8.0+cu128
3
+ numpy==1.26.4
4
+ scipy==1.17.1
5
+ huggingface_hub==1.32.0
6
+ transformers==5.17.0
7
+
model/twla_tachin_common_adaptation_20260922/README.md ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # T-WLA Tachin common-target adaptation
2
+
3
+ This is the T-WLA checkpoint used to generate the Tachin paired-30 W3 cache.
4
+ No MemoryVLA checkpoint, cache, or adaptation result is included in this
5
+ release.
6
+
7
+ ## Adaptation contract
8
+
9
+ - Initialization: T-WLA ten-domain 110k checkpoint
10
+ (`v4-runs:/v4d_ten_stageA/best.pt`).
11
+ - Initial checkpoint SHA-256:
12
+ `ccd65fcf845ec01c7de46bb76e9cf0fee33674dd848771e114ff23b2535f13d9`.
13
+ - Dataset: Tachin training/validation split used by all Table 1 methods.
14
+ - Targets and action masks: the exact common Tachin implementation shared by
15
+ the baseline adaptations.
16
+ - Training: 1,000 optimizer steps, effective batch size 128, learning rate
17
+ `5e-5`, and 100 warm-up steps.
18
+ - Frozen: VLM and visual-memory modules.
19
+ - Trainable: action expert only (102,968,832 parameters).
20
+ - Checkpoint selection: validation loss only; the test split was not used.
21
+ - Selected step: 800.
22
+ - Validation loss: 0.79419 before adaptation and 0.55530 after adaptation.
23
+
24
+ `checkpoint/best.pt` is the complete, directly loadable checkpoint. It was
25
+ assembled from the original frozen tensors, the adapted action expert, and
26
+ Tachin train-only normalization statistics. Its SHA-256 is
27
+ `8df6c87fda2e5ec09d61686451744c37bdcc2d1b4d9d13b58893851be620db7d`.
28
+
29
+ `checkpoint/best_adapter.pt` contains the selected expert-only adapter. The
30
+ script `code/rebuild_twla_adapted_checkpoint.py` can merge it with the original
31
+ ten-domain checkpoint. The complete `best.pt` does not require the original
32
+ checkpoint for inference.
33
+
34
+ The verifier confirmed that all 381 VLM/memory tensors are bit-identical to the
35
+ initial checkpoint and that 155 expert tensors changed. See
36
+ `provenance/verification.json` and `SHA256SUMS`.
37
+
38
+ To retrieve the original initialization from Modal, use:
39
+
40
+ ```bash
41
+ modal volume get v4-runs /v4d_ten_stageA/best.pt
42
+ ```
43
+
44
+ The paired-30 policy cache and tactile-editor artifacts generated from this
45
+ checkpoint are published in `model/tachin_paired30_cache_20260923`.
model/twla_tachin_common_adaptation_20260922/SHA256SUMS ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 35a777f79038f0e009be3da6978b75a42202f7f3ce842cbb3b442fca5efe6411 ./README.md
2
+ 8df6c87fda2e5ec09d61686451744c37bdcc2d1b4d9d13b58893851be620db7d ./checkpoint/best.pt
3
+ caeb8b59c909728941b49af3237e1639f75a26a11a30d487dd4d1cff990b4050 ./checkpoint/best_adapter.pt
4
+ 3515348b90c53e42d2eb4f7435a6c922a5a89d097126a12eda10335ce5c7c7c5 ./code/adapt_twla_tachin_common.py
5
+ f89146ac7f47157deea98e6dd713093f5c80d25af811a8581f553e593de9e7ea ./code/cache_tachin_twla_w3.py
6
+ 0196f9a4cb9fa7b5758501529efb7910a07aafd14c80ef6831578b86273fdd05 ./code/rebuild_twla_adapted_checkpoint.py
7
+ 12912c259f733d3fedf91a567c94c18a2573a4b1a33c428fb9fe04b9dabd9602 ./code/verify_twla_tachin_adaptation.py
8
+ 6f6263f2b8be7f7fe7a55296488e1888ac16aaa2fa37125a327c2acf29686bc9 ./provenance/COMPLETE.json
9
+ bd7ea64950e76084e820f1c2cc0d2700115599214e137103f22e4bcf6f5f1c29 ./provenance/config.json
10
+ 140ec19765ec0752f9af2dd8918d1fda23e5c436392fc15ca9ba25f41fbf8d7a ./provenance/initial_config_used.json
11
+ 140ec19765ec0752f9af2dd8918d1fda23e5c436392fc15ca9ba25f41fbf8d7a ./provenance/initial_ten_domain_config.json
12
+ 4ae9e15ca2706fb1160af13ca6a858c12efc11ff64a7ee237f07cfd8d02a212b ./provenance/metrics.jsonl
13
+ dfb99ee046015d443c068e21e3f58fa0d882c22be6eba2384269b5b308714994 ./provenance/verification.json
model/twla_tachin_common_adaptation_20260922/code/adapt_twla_tachin_common.py ADDED
@@ -0,0 +1,374 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Adapt T-WLA-110k to Tachin using the paper's exact common action targets.
2
+
3
+ The VLM and recurrent memory remain frozen. Only the action expert is adapted, which
4
+ matches the expert-only baseline protocol and keeps the frozen RGB world head compatible
5
+ with the 110k memory representation. Inputs use the same Tachin episode split and anchor
6
+ rows as the common Table-1 cache; labels and masks are read directly from that cache.
7
+ """
8
+ from __future__ import annotations
9
+
10
+ import argparse
11
+ import hashlib
12
+ import json
13
+ import math
14
+ import os
15
+ import random
16
+ import shutil
17
+ import sys
18
+ import time
19
+ from pathlib import Path
20
+
21
+ import numpy as np
22
+ import torch
23
+ from torch.utils.data import DataLoader, Dataset
24
+
25
+
26
+ ROOT = Path(__file__).resolve().parents[1]
27
+ V4 = ROOT / "ref/yuzhench_distill_editor/V4"
28
+ sys.path[:0] = [str(ROOT / "src"), str(V4), str(V4 / "vla_hand_pose"),
29
+ str(ROOT / "scripts")]
30
+
31
+
32
+ def digest(path: Path) -> str:
33
+ value = hashlib.sha256()
34
+ with path.open("rb") as stream:
35
+ for block in iter(lambda: stream.read(8 << 20), b""):
36
+ value.update(block)
37
+ return value.hexdigest()
38
+
39
+
40
+ def atomic_torch(payload, path: Path) -> None:
41
+ temporary = path.with_suffix(path.suffix + ".tmp")
42
+ torch.save(payload, temporary)
43
+ os.replace(temporary, path)
44
+
45
+
46
+ def atomic_json(payload, path: Path) -> None:
47
+ temporary = path.with_suffix(path.suffix + ".tmp")
48
+ temporary.write_text(json.dumps(payload, indent=2, default=str) + "\n")
49
+ os.replace(temporary, path)
50
+
51
+
52
+ class IndexedPlan(Dataset):
53
+ def __init__(self, base, indices):
54
+ self.base = base
55
+ self.indices = np.asarray(indices, dtype=np.int64)
56
+
57
+ def __len__(self):
58
+ return len(self.indices)
59
+
60
+ def __getitem__(self, index):
61
+ item = self.base[index]
62
+ item["query_index"] = int(self.indices[index])
63
+ return item
64
+
65
+
66
+ def planned_dataset(split: str, args, config, tokenizer, source_cls, dataset_cls):
67
+ rows = json.loads((args.adaptation_data / split / "rows.json").read_text())
68
+ identity = dict(np.load(args.common / "shared" / split / "identity.npz",
69
+ allow_pickle=False))
70
+ row_ids = np.asarray([row["sample_id"] for row in rows]).astype(str)
71
+ if not np.array_equal(row_ids, identity["sample_id"].astype(str)):
72
+ raise ValueError(f"{split} prepared rows do not match the common cache")
73
+ source = source_cls(
74
+ name="tachin", video_root="/unused",
75
+ annotation_file=str(args.annotations / "episode_frame_index.npz"),
76
+ label_folder=str(args.annotations / "episodic_annotations"),
77
+ jpeg_root=str(args.jpeg_root),
78
+ )
79
+ episodes = {str(row["episode_id"]) for row in rows}
80
+ dataset = dataset_cls([source], cfg=config,
81
+ video_whitelist={"tachin": episodes}, tokenizer=tokenizer,
82
+ seed=args.seed, load_images=True, training=False,
83
+ deterministic=True)
84
+ units = {(unit.episode, unit.phase): index
85
+ for index, unit in enumerate(dataset.units)}
86
+ plans = []
87
+ for common_index, row in enumerate(rows):
88
+ anchor = int(row["anchor_source_row"])
89
+ phase, relative = anchor % 3, anchor // 3
90
+ length = min(relative + 1, 9)
91
+ key = (str(row["episode_id"]), phase)
92
+ if key not in units:
93
+ raise KeyError(f"Missing T-WLA sequence unit {key}")
94
+ plans.append((units[key], relative - length + 1, length, common_index))
95
+ dataset.set_eval_plan(plans)
96
+ return IndexedPlan(dataset, np.arange(len(rows))), dataset
97
+
98
+
99
+ def set_train_mode(model) -> None:
100
+ model.eval()
101
+ model.expert.train()
102
+
103
+
104
+ def common_batch(batch, split: str, targets, masks, device, observation_cls,
105
+ to_observations):
106
+ steps, step_valid, _, _ = to_observations(
107
+ batch, device, observation_cls, dtype=torch.bfloat16)
108
+ indices = batch["query_index"].numpy()
109
+ actions = torch.from_numpy(np.asarray(targets[split][indices]).copy()).to(
110
+ device, dtype=torch.float32)
111
+ action_mask = torch.from_numpy(np.asarray(masks[split][indices]).copy()).to(
112
+ device, dtype=torch.bool)
113
+ return steps, step_valid, actions, action_mask
114
+
115
+
116
+ @torch.no_grad()
117
+ def validate(model, loader, split, targets, masks, device, observation_cls,
118
+ to_observations, seed):
119
+ model.eval()
120
+ total = 0.0
121
+ count = 0
122
+ generator = torch.Generator(device=device).manual_seed(seed)
123
+ for batch in loader:
124
+ values = common_batch(batch, split, targets, masks, device, observation_cls,
125
+ to_observations)
126
+ with torch.autocast("cuda", dtype=torch.bfloat16):
127
+ loss, _ = model.loss(*values, generator=generator)
128
+ if not torch.isfinite(loss):
129
+ raise ValueError("Nonfinite T-WLA validation loss")
130
+ size = len(batch["query_index"])
131
+ total += float(loss) * size
132
+ count += size
133
+ return total / max(count, 1)
134
+
135
+
136
+ def main() -> None:
137
+ parser = argparse.ArgumentParser()
138
+ parser.add_argument("--checkpoint", type=Path, required=True)
139
+ parser.add_argument("--checkpoint-config", type=Path, required=True)
140
+ parser.add_argument("--annotations", type=Path, required=True)
141
+ parser.add_argument("--jpeg-root", type=Path, required=True)
142
+ parser.add_argument("--adaptation-data", type=Path, required=True)
143
+ parser.add_argument("--common", type=Path, required=True)
144
+ parser.add_argument("--output", type=Path, required=True)
145
+ parser.add_argument("--steps", type=int, default=1000)
146
+ parser.add_argument("--batch-size", type=int, default=32)
147
+ parser.add_argument("--accumulation", type=int, default=4)
148
+ parser.add_argument("--lr", type=float, default=5e-5)
149
+ parser.add_argument("--warmup", type=int, default=100)
150
+ parser.add_argument("--val-every", type=int, default=100)
151
+ parser.add_argument("--workers", type=int, default=2)
152
+ parser.add_argument("--seed", type=int, default=20260922)
153
+ parser.add_argument("--smoke", action="store_true")
154
+ args = parser.parse_args()
155
+ if min(args.steps, args.batch_size, args.accumulation, args.val_every) < 1:
156
+ raise ValueError("steps, batch-size, accumulation, and val-every must be positive")
157
+ if args.lr <= 0 or args.warmup < 0 or args.workers < 0:
158
+ raise ValueError("invalid optimizer or loader setting")
159
+ if args.output.exists():
160
+ raise FileExistsError(args.output)
161
+ if args.smoke:
162
+ args.steps = 2
163
+ args.batch_size = 2
164
+ args.accumulation = 1
165
+ args.val_every = 2
166
+ args.warmup = 0
167
+ args.output.mkdir(parents=True)
168
+
169
+ os.environ.update(
170
+ METHOD_D_CHECKPOINT=str(args.checkpoint.resolve()),
171
+ METHOD_D_RUN_CONFIG=str(args.checkpoint_config.resolve()),
172
+ METHOD_D_SOURCE=str(V4 / "vla_hand_pose"),
173
+ )
174
+ from transformers import AutoTokenizer
175
+ from modal_build_full_editor_cache import load_backbone
176
+ from vla_hand_pose.dataloader import (SequenceDataset, Source, make_collate,
177
+ to_observations)
178
+ from vla_hand_pose.v4_d_fm_0 import Observation
179
+
180
+ random.seed(args.seed)
181
+ np.random.seed(args.seed)
182
+ torch.manual_seed(args.seed)
183
+ torch.cuda.manual_seed_all(args.seed)
184
+ torch.set_num_threads(4)
185
+ torch.set_float32_matmul_precision("high")
186
+ device = torch.device("cuda:0")
187
+ torch.cuda.set_device(device)
188
+
189
+ model, config = load_backbone(device)
190
+ statistics = json.loads((args.adaptation_data / "statistics.json").read_text())[
191
+ "keypoints"]
192
+ model.set_normalisation(**{name: statistics[name] for name in
193
+ ("action_mean", "action_std", "state_mean", "state_std")})
194
+ model.requires_grad_(False)
195
+ model.expert.requires_grad_(True)
196
+ trainable = [parameter for parameter in model.expert.parameters()
197
+ if parameter.requires_grad]
198
+ tokenizer = AutoTokenizer.from_pretrained(config.vlm.model_id)
199
+
200
+ targets = {}
201
+ masks = {}
202
+ loaders = {}
203
+ split_rows = {}
204
+ for split in ("train", "val"):
205
+ targets[split] = np.load(args.common / "shared" / split /
206
+ "target_action.npy", mmap_mode="r")
207
+ masks[split] = np.load(args.common / "shared" / split /
208
+ "action_mask.npy", mmap_mode="r")
209
+ wrapped, base = planned_dataset(split, args, config, tokenizer, Source,
210
+ SequenceDataset)
211
+ base_collate = make_collate(config, tokenizer)
212
+
213
+ def collate(items, base_collate=base_collate):
214
+ batch = base_collate(items)
215
+ batch["query_index"] = torch.as_tensor(
216
+ [item["query_index"] for item in items], dtype=torch.int64)
217
+ return batch
218
+
219
+ loaders[split] = DataLoader(
220
+ wrapped, batch_size=args.batch_size,
221
+ shuffle=split == "train",
222
+ generator=torch.Generator().manual_seed(args.seed),
223
+ num_workers=args.workers, collate_fn=collate, pin_memory=True,
224
+ persistent_workers=args.workers > 0,
225
+ )
226
+ split_rows[split] = len(wrapped)
227
+
228
+ groups = model.param_groups()
229
+ if {id(parameter) for group in groups for parameter in group["params"]} != {
230
+ id(parameter) for parameter in trainable}:
231
+ raise ValueError("optimizer groups do not exactly cover the action expert")
232
+ for group in groups:
233
+ group["lr"] = args.lr
234
+ group["base_lr"] = args.lr
235
+ group["weight_decay"] = 0.01 if group["name"] == "decay" else 0.0
236
+ optimizer = torch.optim.AdamW(groups, lr=args.lr, betas=(0.9, 0.95), eps=1e-8)
237
+
238
+ run_config = {
239
+ **{name: str(value) if isinstance(value, Path) else value
240
+ for name, value in vars(args).items()},
241
+ "protocol": "Tachin-only expert adaptation from T-WLA-110k",
242
+ "targets": "exact common cache target_action/action_mask",
243
+ "normalization": "exact common Tachin train-only keypoint statistics",
244
+ "history": "up to eight prior 10-Hz observations plus current",
245
+ "frozen": ["vlm", "memory"],
246
+ "trainable": ["expert"],
247
+ "train_rows": split_rows["train"],
248
+ "val_rows": split_rows["val"],
249
+ "initial_sha256": digest(args.checkpoint),
250
+ "common_complete_sha256": digest(args.common / "complete.json"),
251
+ }
252
+ atomic_json(run_config, args.output / "config.json")
253
+ shutil.copy2(args.checkpoint_config, args.output / "initial_config_used.json")
254
+
255
+ generator = torch.Generator(device=device).manual_seed(args.seed + 1)
256
+ best = validate(model, loaders["val"], "val", targets, masks, device,
257
+ Observation, to_observations, args.seed + 20000)
258
+ best_step = 0
259
+ atomic_torch({"expert": model.expert.state_dict(), "step": 0,
260
+ "val_loss": best}, args.output / "best_adapter.pt")
261
+ started = time.time()
262
+ samples_seen = 0
263
+ iterator = iter(loaders["train"])
264
+ with (args.output / "metrics.jsonl").open("a") as stream:
265
+ stream.write(json.dumps({"step": 0, "val_loss": best, "seconds": 0.0}) + "\n")
266
+ print(json.dumps({"step": 0, "val_loss": best}), flush=True)
267
+
268
+ for step in range(1, args.steps + 1):
269
+ set_train_mode(model)
270
+ optimizer.zero_grad(set_to_none=True)
271
+ train_loss = 0.0
272
+ step_samples = 0
273
+ if args.warmup and step <= args.warmup:
274
+ factor = step / args.warmup
275
+ else:
276
+ factor = 0.1 + 0.45 * (1 + math.cos(
277
+ math.pi * (step - args.warmup) / max(1, args.steps - args.warmup)))
278
+ for group in optimizer.param_groups:
279
+ group["lr"] = group["base_lr"] * factor
280
+ for _ in range(args.accumulation):
281
+ try:
282
+ batch = next(iterator)
283
+ except StopIteration:
284
+ iterator = iter(loaders["train"])
285
+ batch = next(iterator)
286
+ values = common_batch(batch, "train", targets, masks, device,
287
+ Observation, to_observations)
288
+ with torch.autocast("cuda", dtype=torch.bfloat16):
289
+ loss, _ = model.loss(*values, generator=generator)
290
+ scaled = loss / args.accumulation
291
+ if not torch.isfinite(scaled):
292
+ raise ValueError("Nonfinite T-WLA adaptation loss")
293
+ scaled.backward()
294
+ train_loss += float(scaled.detach())
295
+ step_samples += len(batch["query_index"])
296
+ gradient = torch.nn.utils.clip_grad_norm_(trainable, 1.0,
297
+ error_if_nonfinite=True)
298
+ optimizer.step()
299
+ samples_seen += step_samples
300
+ record = {
301
+ "step": step, "train_loss": train_loss,
302
+ "grad_norm": float(gradient), "samples_seen": samples_seen,
303
+ "equivalent_epochs": samples_seen / split_rows["train"],
304
+ "learning_rates": [group["lr"] for group in optimizer.param_groups],
305
+ "seconds": time.time() - started,
306
+ "peak_gpu_gb": torch.cuda.max_memory_allocated() / 1e9,
307
+ }
308
+ if step % args.val_every == 0 or step == args.steps:
309
+ value = validate(model, loaders["val"], "val", targets, masks, device,
310
+ Observation, to_observations, args.seed + 20000)
311
+ record["val_loss"] = value
312
+ if value < best:
313
+ best = value
314
+ best_step = step
315
+ atomic_torch({"expert": model.expert.state_dict(), "step": step,
316
+ "val_loss": value}, args.output / "best_adapter.pt")
317
+ atomic_torch({
318
+ "expert": model.expert.state_dict(), "optimizer": optimizer.state_dict(),
319
+ "step": step, "best": best, "best_step": best_step,
320
+ "generator_state": generator.get_state(),
321
+ }, args.output / "last_adapter.pt")
322
+ with (args.output / "metrics.jsonl").open("a") as stream:
323
+ stream.write(json.dumps(record) + "\n")
324
+ if step == 1 or step % 10 == 0 or "val_loss" in record:
325
+ print(json.dumps(record), flush=True)
326
+
327
+ selected = torch.load(args.output / "best_adapter.pt", map_location="cpu",
328
+ weights_only=False)
329
+ # Do not serialize the complete live model here. ``load_backbone`` casts it to
330
+ # bfloat16, so doing that would numerically change every frozen FP32 tensor even
331
+ # though only the expert was optimized. Assemble from the source checkpoint and
332
+ # replace only the expert and Tachin normalization buffers.
333
+ source = torch.load(args.checkpoint, map_location="cpu", mmap=True,
334
+ weights_only=False)
335
+ source_state = dict(source["model"])
336
+ for short_name, value in selected["expert"].items():
337
+ name = f"expert.{short_name}"
338
+ if name not in source_state:
339
+ raise KeyError(f"Adapted expert tensor is absent from source checkpoint: {name}")
340
+ source_state[name] = value.to(dtype=source_state[name].dtype)
341
+ for name in ("action_mean", "action_std", "state_mean", "state_std"):
342
+ source_state[name] = torch.as_tensor(
343
+ np.asarray(statistics[name]), dtype=source_state[name].dtype)
344
+ source_state["stats_loaded"] = torch.ones_like(
345
+ source_state["stats_loaded"], dtype=torch.bool)
346
+ full = {
347
+ "model": source_state, "step": int(selected["step"]), "epoch": -1,
348
+ "best_val": float(selected["val_loss"]),
349
+ "adaptation": {
350
+ **run_config,
351
+ "assembly": "original frozen state + adapted expert + Tachin normalization",
352
+ "frozen_tensor_contract": "vlm.* and memory.* bit-identical to initial",
353
+ },
354
+ }
355
+ atomic_torch(full, args.output / "best.pt")
356
+ atomic_json({
357
+ "schema": "twla.tachin_common_adaptation.v1",
358
+ "selected_step": int(selected["step"]),
359
+ "best_validation_loss": float(selected["val_loss"]),
360
+ "initial_checkpoint": str(args.checkpoint.resolve()),
361
+ "initial_sha256": run_config["initial_sha256"],
362
+ "checkpoint_sha256": digest(args.output / "best.pt"),
363
+ "test_used": False,
364
+ "trainable_parameters": sum(parameter.numel() for parameter in trainable),
365
+ "frozen_parameters": sum(parameter.numel() for parameter in model.parameters()
366
+ if not parameter.requires_grad),
367
+ "seconds": time.time() - started,
368
+ }, args.output / "COMPLETE.json")
369
+ print(json.dumps({"status": "complete", "selected_step": int(selected["step"]),
370
+ "best_validation_loss": float(selected["val_loss"])}), flush=True)
371
+
372
+
373
+ if __name__ == "__main__":
374
+ main()
model/twla_tachin_common_adaptation_20260922/code/cache_tachin_twla_w3.py ADDED
@@ -0,0 +1,299 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Cache T-WLA-110k actions, recurrent memory, and W3 RGB-world inputs."""
2
+ from __future__ import annotations
3
+
4
+ import argparse
5
+ import copy
6
+ import hashlib
7
+ import json
8
+ import os
9
+ import sys
10
+ import time
11
+ from pathlib import Path
12
+
13
+ import numpy as np
14
+ import torch
15
+
16
+
17
+ ROOT = Path(__file__).resolve().parents[1]
18
+ V4 = ROOT / "ref/yuzhench_distill_editor/V4"
19
+ sys.path[:0] = [str(ROOT / "src"), str(ROOT / "ref/Tactile"), str(V4), str(V4 / "vla_hand_pose"), str(ROOT / "scripts")]
20
+
21
+
22
+ def sha256(path: Path) -> str:
23
+ h = hashlib.sha256()
24
+ with path.open("rb") as stream:
25
+ for block in iter(lambda: stream.read(8 << 20), b""):
26
+ h.update(block)
27
+ return h.hexdigest()
28
+
29
+
30
+ def array(root: Path, name: str, shape, dtype):
31
+ root.mkdir(parents=True, exist_ok=True)
32
+ path = root / f"{name}.npy"
33
+ if path.exists():
34
+ value = np.load(path, mmap_mode="r+")
35
+ if value.shape != tuple(shape) or value.dtype != np.dtype(dtype):
36
+ raise ValueError(f"Existing cache schema mismatch: {path}")
37
+ return value
38
+ return np.lib.format.open_memmap(path, mode="w+", shape=shape, dtype=dtype)
39
+
40
+
41
+ def atomic_json(path: Path, payload) -> None:
42
+ temporary = path.with_suffix(path.suffix + ".tmp")
43
+ temporary.write_text(json.dumps(payload, indent=2, default=str) + "\n")
44
+ os.replace(temporary, path)
45
+
46
+
47
+ def checkpoint_provenance(path: Path) -> dict:
48
+ """Read cache-facing provenance without assuming the 110k initialization."""
49
+ payload = torch.load(path, map_location="cpu", mmap=True, weights_only=False)
50
+ step = payload.get("optimizer_step", payload.get("step"))
51
+ if step is None:
52
+ raise ValueError(f"Checkpoint does not declare a training/adaptation step: {path}")
53
+ adaptation = payload.get("adaptation")
54
+ result = {
55
+ "checkpoint_step": int(step),
56
+ "checkpoint_is_tachin_adapted": isinstance(adaptation, dict),
57
+ }
58
+ if isinstance(adaptation, dict):
59
+ result.update(
60
+ adaptation_protocol=adaptation.get("protocol"),
61
+ adaptation_initial_checkpoint=adaptation.get("checkpoint"),
62
+ adaptation_initial_sha256=adaptation.get("initial_sha256"),
63
+ adaptation_common_complete_sha256=adaptation.get("common_complete_sha256"),
64
+ )
65
+ del payload
66
+ return result
67
+
68
+
69
+ class PlannedDataset:
70
+ def __init__(self, base, common_indices, anchors):
71
+ self.base = base
72
+ self.common_indices = np.asarray(common_indices)
73
+ self.anchors = np.asarray(anchors)
74
+
75
+ def __len__(self):
76
+ return len(self.common_indices)
77
+
78
+ def __getitem__(self, index):
79
+ from tactile_editor.rgb_world_data import project_future_hand_roi
80
+ item = self.base[index]
81
+ unit = self.base.units[self.base._eval_plan[index][0]]
82
+ annotation, _, _ = self.base.cores[unit.source]._load_or_cache_episode(unit.episode)
83
+ anchor = int(self.anchors[index])
84
+ roi = np.zeros((2, 16, 16), bool)
85
+ roi_valid = np.zeros(2, bool)
86
+ hand = annotation.get(str(annotation["anno_type"]))
87
+ if isinstance(hand, dict):
88
+ for future_index, offset in enumerate((3, 6)):
89
+ if anchor + offset < len(annotation["video_decode_frame"]):
90
+ projected = project_future_hand_roi(
91
+ np.asarray(hand["joints_camspace"])[anchor + offset],
92
+ annotation["intrinsics"], source_hw=(1080, 1920),
93
+ )
94
+ roi[future_index] = projected.roi
95
+ roi_valid[future_index] = projected.valid
96
+ item.update(query_index=int(self.common_indices[index]), future_roi=roi,
97
+ future_roi_valid=roi_valid)
98
+ return item
99
+
100
+
101
+ def main() -> None:
102
+ parser = argparse.ArgumentParser()
103
+ parser.add_argument("--checkpoint", type=Path, required=True)
104
+ parser.add_argument("--checkpoint-config", type=Path, required=True)
105
+ parser.add_argument("--world-checkpoint", type=Path, required=True)
106
+ parser.add_argument("--world-config", type=Path, required=True)
107
+ parser.add_argument("--annotations", type=Path, required=True)
108
+ parser.add_argument("--jpeg-root", type=Path, required=True)
109
+ parser.add_argument("--adaptation-data", type=Path, required=True)
110
+ parser.add_argument("--common", type=Path, required=True)
111
+ parser.add_argument("--output", type=Path, required=True)
112
+ parser.add_argument("--shard", type=int, required=True)
113
+ parser.add_argument("--shards", type=int, default=4)
114
+ parser.add_argument("--batch-size", type=int, default=24)
115
+ parser.add_argument("--workers", type=int, default=4)
116
+ parser.add_argument("--seed", type=int, default=20260922)
117
+ parser.add_argument("--limit", type=int, default=0)
118
+ args = parser.parse_args()
119
+ if not 0 <= args.shard < args.shards:
120
+ raise ValueError("shard must be in [0, shards)")
121
+
122
+ checkpoint_metadata = checkpoint_provenance(args.checkpoint)
123
+
124
+ os.environ.update(METHOD_D_CHECKPOINT=str(args.checkpoint.resolve()),
125
+ METHOD_D_RUN_CONFIG=str(args.checkpoint_config.resolve()),
126
+ METHOD_D_SOURCE=str(V4 / "vla_hand_pose"))
127
+ from transformers import AutoTokenizer
128
+ from modal_build_full_editor_cache import load_backbone, apply_saved_config
129
+ from vla_hand_pose.v4_d_fm_0 import V4DConfig, MemoryBank, Observation, flow
130
+ from vla_hand_pose.dataloader import SequenceDataset, Source, make_collate, to_observations
131
+ from vla_hand_pose.world_model import dc_ae
132
+ from tactile_editor import EditorConfig, FusionConfig, TactileMethodD, Variant
133
+
134
+ torch.set_num_threads(3)
135
+ torch.set_float32_matmul_precision("high")
136
+ torch.manual_seed(args.seed)
137
+ device = torch.device("cuda:0")
138
+ backbone, config = load_backbone(device)
139
+
140
+ world_payload = torch.load(args.world_checkpoint, map_location="cpu", mmap=True, weights_only=False)
141
+ world_metadata = json.loads(args.world_config.read_text())
142
+ saved = world_payload.get("effective_config", world_metadata.get("effective_config", world_metadata.get("config")))
143
+ world_config = V4DConfig()
144
+ apply_saved_config(world_config, saved)
145
+ world_config.validate()
146
+ del world_payload
147
+ dataset_config = copy.deepcopy(config)
148
+ dataset_config.world = world_config.world
149
+ autoencoder = dc_ae.build(world_config.world.dcae, dtype=torch.float32).to(device)
150
+ fusion = FusionConfig(hidden_dim=config.expert.hidden_size, num_heads=config.expert.num_heads)
151
+ sampler = TactileMethodD(backbone, EditorConfig(variant=Variant.V0_BASE, fusion=fusion)).eval()
152
+ tokenizer = AutoTokenizer.from_pretrained(config.vlm.model_id)
153
+ source = Source(name="tachin", video_root="/unused",
154
+ annotation_file=str(args.annotations / "episode_frame_index.npz"),
155
+ label_folder=str(args.annotations / "episodic_annotations"),
156
+ jpeg_root=str(args.jpeg_root))
157
+
158
+ for split in ("train", "val", "test"):
159
+ rows = json.loads((args.adaptation_data / split / "rows.json").read_text())
160
+ identity = dict(np.load(args.common / "shared" / split / "identity.npz", allow_pickle=False))
161
+ if not np.array_equal(np.asarray([row["sample_id"] for row in rows]).astype(str),
162
+ identity["sample_id"].astype(str)):
163
+ raise ValueError(f"T-WLA {split} row identity mismatch")
164
+ all_indices = np.arange(args.shard, len(rows), args.shards, dtype=np.int64)
165
+ if args.limit:
166
+ all_indices = all_indices[:args.limit]
167
+ output = args.output / "bases/twla" / split / f"shard_{args.shard:02d}"
168
+ completion = output / ("smoke_complete.json" if args.limit else "complete.json")
169
+ if completion.exists():
170
+ print(json.dumps({"split": split, "shard": args.shard, "status": "already complete"}), flush=True)
171
+ continue
172
+
173
+ episodes = {str(rows[index]["episode_id"]) for index in all_indices}
174
+ dataset = SequenceDataset([source], cfg=dataset_config,
175
+ video_whitelist={"tachin": episodes}, tokenizer=tokenizer,
176
+ seed=0, load_images=True, training=False, deterministic=True)
177
+ units = {(unit.episode, unit.phase): index for index, unit in enumerate(dataset.units)}
178
+ plans = []
179
+ anchors = []
180
+ for common_index in all_indices:
181
+ row = rows[int(common_index)]
182
+ anchor = int(row["anchor_source_row"])
183
+ phase, relative = anchor % 3, anchor // 3
184
+ length = min(relative + 1, 9)
185
+ plans.append((units[(str(row["episode_id"]), phase)], relative - length + 1,
186
+ length, int(common_index)))
187
+ anchors.append(anchor)
188
+ dataset.set_eval_plan(plans)
189
+ wrapped = PlannedDataset(dataset, all_indices, anchors)
190
+ base_collate = make_collate(dataset_config, tokenizer)
191
+ def collate(items):
192
+ batch = base_collate(items)
193
+ for name in ("query_index", "future_roi", "future_roi_valid"):
194
+ batch[name] = torch.from_numpy(np.asarray([item[name] for item in items]))
195
+ return batch
196
+
197
+ count = len(all_indices)
198
+ fields = {
199
+ "base_action": ((count, 16, 192), "float32"),
200
+ "final_memory": ((count, 8, 960), "float16"),
201
+ "image_tokens": ((count, 64, 960), "float16"),
202
+ "clean_latent": ((count, 2, 32, 16, 16), "float16"),
203
+ "rgb_valid": ((count, 2), "bool"),
204
+ "future_roi": ((count, 2, 16, 16), "bool"),
205
+ "query_index": ((count,), "int64"),
206
+ }
207
+ values = {name: array(output, ("smoke_" if args.limit else "") + name, shape, dtype)
208
+ for name, (shape, dtype) in fields.items()}
209
+ marker = array(output, ("smoke_" if args.limit else "") + "row_complete", (count,), "bool")
210
+ remaining = np.flatnonzero(~marker)
211
+ loader = torch.utils.data.DataLoader(torch.utils.data.Subset(wrapped, remaining.tolist()),
212
+ batch_size=args.batch_size, shuffle=False, num_workers=args.workers,
213
+ collate_fn=collate, pin_memory=True, persistent_workers=args.workers > 0)
214
+ started = time.time()
215
+ cursor = 0
216
+ max_target_difference = 0.0
217
+ mask_mismatches = 0
218
+ target = np.load(args.common / "shared" / split / "target_action.npy", mmap_mode="r")
219
+ target_mask = np.load(args.common / "shared" / split / "action_mask.npy", mmap_mode="r")
220
+ with torch.inference_mode():
221
+ for batch_index, batch in enumerate(loader):
222
+ steps, step_valid, actions, action_mask = to_observations(batch, device, Observation, dtype=torch.bfloat16)
223
+ size = len(actions)
224
+ bank = MemoryBank(size, config.memory.bank_capacity, config.memory.num_slots,
225
+ config.memory.dim, device=device, dtype=backbone.vlm.dtype)
226
+ for time_index, observation in enumerate(steps[:-1]):
227
+ active = torch.nonzero(step_valid[:, time_index], as_tuple=True)[0]
228
+ if not active.numel():
229
+ continue
230
+ subset = observation if active.numel() == size else observation.index_select(active)
231
+ bank.append_rows(active, backbone.encode(subset, bank, rows=active))
232
+ final_memory, image_tokens = backbone.encode(steps[-1], bank, return_image_tokens=True)
233
+ generation_mask = flow.component_action_mask(size, 16, 192, config.flow, device)
234
+ query_indices = batch["query_index"].numpy()
235
+ noise = torch.stack([
236
+ torch.randn((16, 192), device=device, dtype=final_memory.dtype,
237
+ generator=torch.Generator(device=device).manual_seed(args.seed + int(index)))
238
+ for index in query_indices
239
+ ])
240
+ normalized = sampler.sample_from_memory(
241
+ final_memory, generation_mask,
242
+ state_with_mask=(backbone.state_with_mask(steps[-1].state, steps[-1].state_mask)
243
+ if config.action.direct_state_to_expert else None),
244
+ noise=noise, num_steps=config.flow.num_inference_steps,
245
+ return_normalized=True,
246
+ ).float()
247
+ physical = normalized * backbone.action_std.float() + backbone.action_mean.float()
248
+ future_images = batch["future_images"].to(device).permute(0, 1, 4, 2, 3).float().div_(127.5).sub_(1)
249
+ clean_latent = dc_ae.encode_targets(autoencoder, future_images, world_config.world.dcae)
250
+ positions = remaining[cursor:cursor + size]
251
+ difference = np.abs(actions.float().cpu().numpy() - target[query_indices])
252
+ common_mask = target_mask[query_indices]
253
+ max_target_difference = max(max_target_difference,
254
+ float(difference[common_mask].max(initial=0)))
255
+ mask_mismatches += int(np.count_nonzero(action_mask.cpu().numpy() != common_mask))
256
+ payload = {
257
+ "base_action": physical,
258
+ "final_memory": final_memory,
259
+ "image_tokens": image_tokens,
260
+ "clean_latent": clean_latent,
261
+ "rgb_valid": batch["future_valid"].bool() & batch["future_roi_valid"].bool(),
262
+ "future_roi": batch["future_roi"],
263
+ "query_index": batch["query_index"],
264
+ }
265
+ for name, value in payload.items():
266
+ if value.dtype == torch.bfloat16:
267
+ value = value.float()
268
+ result = value.detach().cpu().numpy()
269
+ if not np.isfinite(result).all():
270
+ raise ValueError(f"Nonfinite T-WLA cache field {name}")
271
+ values[name][positions] = result
272
+ values[name].flush()
273
+ marker[positions] = True
274
+ marker.flush()
275
+ cursor += size
276
+ if batch_index % 25 == 0:
277
+ print(json.dumps({"split": split, "shard": args.shard,
278
+ "done": int(marker.sum()), "total": count,
279
+ "rows_per_second": cursor / max(time.time() - started, 1e-6)}), flush=True)
280
+ if not marker.all():
281
+ raise RuntimeError(f"Incomplete T-WLA shard: {output}")
282
+ atomic_json(completion, {
283
+ "schema": "tachin.table1.twla_w3_cache.v1",
284
+ "split": split, "shard": args.shard, "shards": args.shards,
285
+ "rows": count, "checkpoint": str(args.checkpoint.resolve()),
286
+ "checkpoint_sha256": sha256(args.checkpoint), **checkpoint_metadata,
287
+ "memory_protocol": "up to eight available 10-Hz historical observations plus current",
288
+ "noise": f"per-row torch seed {args.seed} + zero-based common row index",
289
+ "world_checkpoint": str(args.world_checkpoint.resolve()),
290
+ "world_checkpoint_sha256": sha256(args.world_checkpoint),
291
+ "max_common_target_difference": max_target_difference,
292
+ "common_mask_mismatches": mask_mismatches,
293
+ "seconds": time.time() - started,
294
+ })
295
+ print(json.dumps({"split": split, "shard": args.shard, "status": "complete"}), flush=True)
296
+
297
+
298
+ if __name__ == "__main__":
299
+ main()
model/twla_tachin_common_adaptation_20260922/code/rebuild_twla_adapted_checkpoint.py ADDED
@@ -0,0 +1,96 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Build a strict expert-only T-WLA checkpoint from an adaptation adapter.
2
+
3
+ The backbone loader casts the live model to bfloat16. Saving its complete state dict
4
+ therefore changes frozen FP32 tensors even when they received no optimizer updates.
5
+ This utility starts from the original checkpoint and replaces only the action expert
6
+ and the four Tachin normalization buffers. Frozen VLM/memory tensors consequently
7
+ remain bit-identical to the released 110k checkpoint.
8
+ """
9
+ from __future__ import annotations
10
+
11
+ import argparse
12
+ import hashlib
13
+ import json
14
+ import os
15
+ from pathlib import Path
16
+
17
+ import numpy as np
18
+ import torch
19
+
20
+
21
+ def sha256(path: Path) -> str:
22
+ digest = hashlib.sha256()
23
+ with path.open("rb") as stream:
24
+ for block in iter(lambda: stream.read(8 << 20), b""):
25
+ digest.update(block)
26
+ return digest.hexdigest()
27
+
28
+
29
+ def main() -> None:
30
+ parser = argparse.ArgumentParser()
31
+ parser.add_argument("--initial", type=Path, required=True)
32
+ parser.add_argument("--adapter", type=Path, required=True)
33
+ parser.add_argument("--adaptation-data", type=Path, required=True)
34
+ parser.add_argument("--run-config", type=Path, required=True)
35
+ parser.add_argument("--output", type=Path, required=True)
36
+ args = parser.parse_args()
37
+ if args.output.exists():
38
+ raise FileExistsError(args.output)
39
+
40
+ initial = torch.load(args.initial, map_location="cpu", mmap=True, weights_only=False)
41
+ adapter = torch.load(args.adapter, map_location="cpu", mmap=True, weights_only=False)
42
+ run_config = json.loads(args.run_config.read_text())
43
+ state = dict(initial["model"])
44
+ expert = adapter.get("expert")
45
+ if not isinstance(expert, dict):
46
+ raise ValueError("adapter checkpoint has no expert state dict")
47
+ expected = {name.removeprefix("expert.") for name in state if name.startswith("expert.")}
48
+ if set(expert) != expected:
49
+ raise ValueError(
50
+ f"expert schema mismatch: missing={sorted(expected - set(expert))[:10]}, "
51
+ f"extra={sorted(set(expert) - expected)[:10]}"
52
+ )
53
+ changed = 0
54
+ for short_name, value in expert.items():
55
+ name = f"expert.{short_name}"
56
+ replacement = value.to(dtype=state[name].dtype)
57
+ changed += int(not torch.equal(state[name], replacement))
58
+ state[name] = replacement
59
+ if changed == 0:
60
+ raise ValueError("adapted action expert is identical to the initial expert")
61
+
62
+ statistics = json.loads((args.adaptation_data / "statistics.json").read_text())["keypoints"]
63
+ for name in ("action_mean", "action_std", "state_mean", "state_std"):
64
+ value = torch.as_tensor(np.asarray(statistics[name]), dtype=state[name].dtype)
65
+ if tuple(value.shape) != tuple(state[name].shape):
66
+ raise ValueError(f"normalization shape mismatch for {name}")
67
+ state[name] = value
68
+ state["stats_loaded"] = torch.ones_like(state["stats_loaded"], dtype=torch.bool)
69
+
70
+ payload = {
71
+ "model": state,
72
+ "step": int(adapter["step"]),
73
+ "epoch": -1,
74
+ "best_val": float(adapter["val_loss"]),
75
+ "adaptation": {
76
+ **run_config,
77
+ "assembly": "original frozen state + adapted expert + Tachin normalization",
78
+ "initial_sha256": sha256(args.initial),
79
+ "adapter_sha256": sha256(args.adapter),
80
+ "frozen_tensor_contract": "vlm.* and memory.* bit-identical to initial",
81
+ },
82
+ }
83
+ temporary = args.output.with_suffix(args.output.suffix + ".tmp")
84
+ torch.save(payload, temporary)
85
+ os.replace(temporary, args.output)
86
+ print(json.dumps({
87
+ "output": str(args.output),
88
+ "step": payload["step"],
89
+ "best_val": payload["best_val"],
90
+ "changed_expert_tensors": changed,
91
+ "sha256": sha256(args.output),
92
+ }, indent=2))
93
+
94
+
95
+ if __name__ == "__main__":
96
+ main()
model/twla_tachin_common_adaptation_20260922/code/verify_twla_tachin_adaptation.py ADDED
@@ -0,0 +1,128 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Audit the expert-only T-WLA Tachin adaptation before cache generation."""
2
+ from __future__ import annotations
3
+
4
+ import argparse
5
+ import hashlib
6
+ import json
7
+ import math
8
+ import os
9
+ from pathlib import Path
10
+
11
+ import numpy as np
12
+ import torch
13
+
14
+
15
+ def sha256(path: Path) -> str:
16
+ value = hashlib.sha256()
17
+ with path.open("rb") as stream:
18
+ for block in iter(lambda: stream.read(8 << 20), b""):
19
+ value.update(block)
20
+ return value.hexdigest()
21
+
22
+
23
+ def atomic_json(path: Path, payload) -> None:
24
+ temporary = path.with_suffix(path.suffix + ".tmp")
25
+ temporary.write_text(json.dumps(payload, indent=2, default=str) + "\n")
26
+ os.replace(temporary, path)
27
+
28
+
29
+ def main() -> None:
30
+ parser = argparse.ArgumentParser()
31
+ parser.add_argument("--run", type=Path, required=True)
32
+ parser.add_argument("--initial", type=Path, required=True)
33
+ parser.add_argument("--adaptation-data", type=Path, required=True)
34
+ parser.add_argument("--common", type=Path, required=True)
35
+ args = parser.parse_args()
36
+
37
+ required = ("COMPLETE.json", "config.json", "metrics.jsonl", "best.pt")
38
+ missing = [name for name in required if not (args.run / name).is_file()]
39
+ if missing:
40
+ raise FileNotFoundError(f"Incomplete adaptation run; missing {missing}")
41
+
42
+ completion = json.loads((args.run / "COMPLETE.json").read_text())
43
+ config = json.loads((args.run / "config.json").read_text())
44
+ metrics = [json.loads(line) for line in (args.run / "metrics.jsonl").read_text().splitlines()
45
+ if line.strip()]
46
+ if not metrics or int(metrics[0].get("step", -1)) != 0 or "val_loss" not in metrics[0]:
47
+ raise ValueError("metrics.jsonl does not begin with the step-0 validation")
48
+ initial_validation = float(metrics[0]["val_loss"])
49
+ selected_step = int(completion["selected_step"])
50
+ best_validation = float(completion["best_validation_loss"])
51
+ if selected_step <= 0:
52
+ raise ValueError("Tachin adaptation did not beat the unadapted step-0 checkpoint")
53
+ if not (math.isfinite(best_validation) and best_validation < initial_validation):
54
+ raise ValueError(
55
+ f"Selected validation {best_validation} did not improve step 0 {initial_validation}")
56
+ if completion.get("test_used") is not False:
57
+ raise ValueError("Adaptation completion does not certify test_used=false")
58
+
59
+ initial_sha = sha256(args.initial)
60
+ common_sha = sha256(args.common / "complete.json")
61
+ if completion.get("initial_sha256") != initial_sha:
62
+ raise ValueError("COMPLETE.json initial checkpoint hash mismatch")
63
+ if config.get("initial_sha256") != initial_sha:
64
+ raise ValueError("config.json initial checkpoint hash mismatch")
65
+ if config.get("common_complete_sha256") != common_sha:
66
+ raise ValueError("config.json common-cache hash mismatch")
67
+
68
+ adapted = torch.load(args.run / "best.pt", map_location="cpu", mmap=True,
69
+ weights_only=False)
70
+ initial = torch.load(args.initial, map_location="cpu", mmap=True, weights_only=False)
71
+ if int(adapted.get("step", -1)) != selected_step:
72
+ raise ValueError("best.pt selected step disagrees with COMPLETE.json")
73
+ if not isinstance(adapted.get("adaptation"), dict):
74
+ raise ValueError("best.pt has no adaptation provenance")
75
+ adapted_state = adapted["model"]
76
+ initial_state = initial["model"]
77
+ if set(adapted_state) != set(initial_state):
78
+ missing_keys = sorted(set(initial_state) - set(adapted_state))[:10]
79
+ extra_keys = sorted(set(adapted_state) - set(initial_state))[:10]
80
+ raise ValueError(f"State schema changed; missing={missing_keys}, extra={extra_keys}")
81
+
82
+ frozen_prefixes = ("vlm.", "memory.")
83
+ frozen_tensors = 0
84
+ for name in initial_state:
85
+ if not name.startswith(frozen_prefixes):
86
+ continue
87
+ if not torch.equal(initial_state[name], adapted_state[name]):
88
+ delta = float((initial_state[name].float() - adapted_state[name].float()).abs().max())
89
+ raise ValueError(f"Frozen tensor changed: {name}, max delta {delta}")
90
+ frozen_tensors += 1
91
+
92
+ expert_changed = []
93
+ for name in initial_state:
94
+ if name.startswith("expert.") and not torch.equal(initial_state[name], adapted_state[name]):
95
+ expert_changed.append(name)
96
+ if not expert_changed:
97
+ raise ValueError("No action-expert tensor changed during adaptation")
98
+
99
+ stats = json.loads((args.adaptation_data / "statistics.json").read_text())["keypoints"]
100
+ for name in ("action_mean", "action_std", "state_mean", "state_std"):
101
+ actual = adapted_state[name].float().numpy()
102
+ expected = np.asarray(stats[name], dtype=np.float32)
103
+ if not np.allclose(actual, expected, rtol=1e-6, atol=1e-7):
104
+ raise ValueError(f"Adapted checkpoint {name} does not match Tachin statistics")
105
+ if not bool(adapted_state["stats_loaded"]):
106
+ raise ValueError("Adapted checkpoint statistics are not marked loaded")
107
+
108
+ report = {
109
+ "status": "passed",
110
+ "run": str(args.run.resolve()),
111
+ "selected_step": selected_step,
112
+ "initial_validation_loss": initial_validation,
113
+ "best_validation_loss": best_validation,
114
+ "validation_reduction": initial_validation - best_validation,
115
+ "test_used": False,
116
+ "initial_checkpoint_sha256": initial_sha,
117
+ "adapted_checkpoint_sha256": sha256(args.run / "best.pt"),
118
+ "common_complete_sha256": common_sha,
119
+ "frozen_vlm_memory_tensors_verified": frozen_tensors,
120
+ "changed_expert_tensors": len(expert_changed),
121
+ "normalization": "Tachin train-only keypoints statistics",
122
+ }
123
+ atomic_json(args.run / "verification.json", report)
124
+ print(json.dumps(report, indent=2))
125
+
126
+
127
+ if __name__ == "__main__":
128
+ main()
model/twla_tachin_common_adaptation_20260922/provenance/COMPLETE.json ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "schema": "twla.tachin_common_adaptation.v1",
3
+ "selected_step": 800,
4
+ "best_validation_loss": 0.5552965396678234,
5
+ "initial_checkpoint": "/scratch/hannahyao24/VLA-tactile_paper/checkpoints/v4d_ten_stageA/best.pt",
6
+ "initial_sha256": "ccd65fcf845ec01c7de46bb76e9cf0fee33674dd848771e114ff23b2535f13d9",
7
+ "checkpoint_sha256": "eb6ef381882dfbf4188d2bce13f3ea470214f037b0d3340e6aadb6064cf90ba5",
8
+ "test_used": false,
9
+ "trainable_parameters": 102968832,
10
+ "frozen_parameters": 325414726,
11
+ "seconds": 1778.7159943580627
12
+ }
model/twla_tachin_common_adaptation_20260922/provenance/config.json ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint": "checkpoints/v4d_ten_stageA/best.pt",
3
+ "checkpoint_config": "checkpoints/v4d_ten_stageA/config_used.json",
4
+ "annotations": "artifacts/modal/v4-data/tachin_ours",
5
+ "jpeg_root": "artifacts/modal/v4-data/tachin_jpeg",
6
+ "adaptation_data": "artifacts/datasets/tachin_adaptation_20260922",
7
+ "common": "artifacts/tachin_table1_20260922",
8
+ "output": "runs/twla_tachin_common_adaptation_20260922",
9
+ "steps": 1000,
10
+ "batch_size": 32,
11
+ "accumulation": 4,
12
+ "lr": 5e-05,
13
+ "warmup": 100,
14
+ "val_every": 100,
15
+ "workers": 8,
16
+ "seed": 20260922,
17
+ "smoke": false,
18
+ "protocol": "Tachin-only expert adaptation from T-WLA-110k",
19
+ "targets": "exact common cache target_action/action_mask",
20
+ "normalization": "exact common Tachin train-only keypoint statistics",
21
+ "history": "up to eight prior 10-Hz observations plus current",
22
+ "frozen": [
23
+ "vlm",
24
+ "memory"
25
+ ],
26
+ "trainable": [
27
+ "expert"
28
+ ],
29
+ "train_rows": 18208,
30
+ "val_rows": 1006,
31
+ "initial_sha256": "ccd65fcf845ec01c7de46bb76e9cf0fee33674dd848771e114ff23b2535f13d9",
32
+ "common_complete_sha256": "1cffe026d12e556cc2f66fbda0b34a65a985f50d7aeb0a5adb50571e8437e740"
33
+ }
model/twla_tachin_common_adaptation_20260922/provenance/initial_config_used.json ADDED
@@ -0,0 +1,115 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_comment": "Variant D on all ten domains. Validation does NOT run in this process: `val_every` is set so it never fires, and `snapshot_for_validator` makes each save also drop `snapshots/snap_<step>.pt` for `validate_loop` to score on its own GPU. Measured 2026-09-08: an in-loop pass cost 25 min against 73 min of training, 34% of the wall clock, because the validation frames go cold between passes. `best.pt` is written by the validator, not here.\\n\\nThe per-domain `weight` is NOT the share -- it is the share divided by the domain's natural size, because MixtureSampler and compute_stats both consume `weight * units`. Shares were set by the user on 2026-09-07 as relative parts against epic30 (30/30/15/15/5/5/5/3/2/2); see final/README.md. CHANGING A SHARE MEANS RECOMPUTING THE WEIGHT.",
3
+ "variant": "d",
4
+ "seed": 0,
5
+ "batch_size": 64,
6
+ "num_workers": 12,
7
+ "prefetch_factor": 2,
8
+ "max_steps": 110000,
9
+ "log_every": 50,
10
+ "val_every": 110000,
11
+ "save_every": 2000,
12
+ "output_dir": "/runs/v4d_ten_stageA",
13
+ "data": {
14
+ "statistics": "/data/splits/ten_v3/stats_80hz.json",
15
+ "sources": [
16
+ {
17
+ "name": "epic30",
18
+ "video_root": "/data/epic_vitra/Video/Epic-Kitchen_root",
19
+ "annotation_file": "/data/epic30_ours_v2/episode_frame_index.npz",
20
+ "label_folder": "/data/epic30_ours_v2",
21
+ "jpeg_root": "/data/preprocessed/epic30_ours_v2_jpeg",
22
+ "split_dir": "/data/splits/epic30_v5",
23
+ "weight": 0.2654
24
+ },
25
+ {
26
+ "name": "egotouch",
27
+ "video_root": "/data/et_video",
28
+ "annotation_file": "/data/egotouch_ours/episode_frame_index.npz",
29
+ "label_folder": "/data/egotouch_ours/episodic_annotations",
30
+ "jpeg_root": "/data/preprocessed/egotouch_jpeg",
31
+ "split_dir": "/data/splits/egotouch_v2",
32
+ "weight": 0.3694
33
+ },
34
+ {
35
+ "name": "gigahands",
36
+ "video_root": "/data/gigahands_vitra_mano_strict_v1__CHONGHE/Video/GigaHands_root",
37
+ "annotation_file": "/data/gigahands_ours/episode_frame_index.npz",
38
+ "label_folder": "/data/gigahands_ours/episodic_annotations",
39
+ "jpeg_root": "/data/preprocessed/gigahands_jpeg",
40
+ "split_dir": "/data/splits/gigahands_v1",
41
+ "weight": 0.2811
42
+ },
43
+ {
44
+ "name": "egoexo4d",
45
+ "video_root": "/data/egoexo4d_OFFICIAL/undistorted",
46
+ "annotation_file": "/data/vitra1m_egoexo4d/egoexo4d/episode_frame_index.npz",
47
+ "label_folder": "/data/vitra1m_egoexo4d/egoexo4d/episodic_annotations",
48
+ "jpeg_root": "/data/preprocessed/egoexo4d_jpeg",
49
+ "split_dir": "/data/splits/egoexo4d_v1",
50
+ "weight": 0.2962
51
+ },
52
+ {
53
+ "name": "oakink2",
54
+ "video_root": "/data/oakink2_vitra_streaming_v1__CHONGHE/Video/OakInk2_root",
55
+ "annotation_file": "/data/oakink2_ours/episode_frame_index.npz",
56
+ "label_folder": "/data/oakink2_ours/episodic_annotations",
57
+ "jpeg_root": "/data/preprocessed/oakink2_jpeg",
58
+ "split_dir": "/data/splits/oakink2_v2",
59
+ "weight": 0.2275
60
+ },
61
+ {
62
+ "name": "taco",
63
+ "video_root": "/data/taco_raw/v1/Egocentric_RGB_Videos_x/Egocentric_RGB_Videos",
64
+ "annotation_file": "/data/taco_ours/episode_frame_index.npz",
65
+ "label_folder": "/data/taco_ours/episodic_annotations",
66
+ "jpeg_root": "/data/preprocessed/taco_jpeg",
67
+ "split_dir": "/data/splits/taco_v1",
68
+ "weight": 0.278
69
+ },
70
+ {
71
+ "name": "hot3d",
72
+ "video_root": "/data/hot3d_vitra_atomic_v1__CHONGHE/Video/HOT3D_root",
73
+ "annotation_file": "/data/hot3d_ours/episode_frame_index.npz",
74
+ "label_folder": "/data/hot3d_ours/episodic_annotations",
75
+ "jpeg_root": "/data/preprocessed/hot3d_jpeg",
76
+ "split_dir": "/data/splits/hot3d_v2",
77
+ "weight": 0.3513
78
+ },
79
+ {
80
+ "name": "arctic",
81
+ "video_root": "/data/arctic_vitra/Video/arctic_root",
82
+ "annotation_file": "/data/arctic_vitra/arctic/episode_frame_index.npz",
83
+ "label_folder": "/data/arctic_vitra/arctic/episodic_annotations",
84
+ "jpeg_root": "/data/preprocessed/arctic_jpeg",
85
+ "split_dir": "/data/splits/arctic_v1",
86
+ "weight": 0.3142
87
+ },
88
+ {
89
+ "name": "h2o",
90
+ "video_root": "/data/h2o_vitra/Video/h2o_root",
91
+ "annotation_file": "/data/h2o_ours/episode_frame_index.npz",
92
+ "label_folder": "/data/h2o_ours/episodic_annotations",
93
+ "jpeg_root": "/data/preprocessed/h2o_jpeg",
94
+ "split_dir": "/data/splits/h2o_v2",
95
+ "weight": 0.4535
96
+ },
97
+ {
98
+ "name": "tachin",
99
+ "video_root": "/data/tachin_precut_v1__CHONGHE",
100
+ "annotation_file": "/data/tachin_ours/episode_frame_index.npz",
101
+ "label_folder": "/data/tachin_ours/episodic_annotations",
102
+ "jpeg_root": "/data/preprocessed/tachin_jpeg",
103
+ "split_dir": "/data/splits/tachin_v2",
104
+ "weight": 1.0
105
+ }
106
+ ]
107
+ },
108
+ "config": {
109
+ "optim": {
110
+ "warmup_steps": 1000,
111
+ "decay_steps": 110000
112
+ }
113
+ },
114
+ "snapshot_for_validator": true
115
+ }
model/twla_tachin_common_adaptation_20260922/provenance/initial_ten_domain_config.json ADDED
@@ -0,0 +1,115 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_comment": "Variant D on all ten domains. Validation does NOT run in this process: `val_every` is set so it never fires, and `snapshot_for_validator` makes each save also drop `snapshots/snap_<step>.pt` for `validate_loop` to score on its own GPU. Measured 2026-09-08: an in-loop pass cost 25 min against 73 min of training, 34% of the wall clock, because the validation frames go cold between passes. `best.pt` is written by the validator, not here.\\n\\nThe per-domain `weight` is NOT the share -- it is the share divided by the domain's natural size, because MixtureSampler and compute_stats both consume `weight * units`. Shares were set by the user on 2026-09-07 as relative parts against epic30 (30/30/15/15/5/5/5/3/2/2); see final/README.md. CHANGING A SHARE MEANS RECOMPUTING THE WEIGHT.",
3
+ "variant": "d",
4
+ "seed": 0,
5
+ "batch_size": 64,
6
+ "num_workers": 12,
7
+ "prefetch_factor": 2,
8
+ "max_steps": 110000,
9
+ "log_every": 50,
10
+ "val_every": 110000,
11
+ "save_every": 2000,
12
+ "output_dir": "/runs/v4d_ten_stageA",
13
+ "data": {
14
+ "statistics": "/data/splits/ten_v3/stats_80hz.json",
15
+ "sources": [
16
+ {
17
+ "name": "epic30",
18
+ "video_root": "/data/epic_vitra/Video/Epic-Kitchen_root",
19
+ "annotation_file": "/data/epic30_ours_v2/episode_frame_index.npz",
20
+ "label_folder": "/data/epic30_ours_v2",
21
+ "jpeg_root": "/data/preprocessed/epic30_ours_v2_jpeg",
22
+ "split_dir": "/data/splits/epic30_v5",
23
+ "weight": 0.2654
24
+ },
25
+ {
26
+ "name": "egotouch",
27
+ "video_root": "/data/et_video",
28
+ "annotation_file": "/data/egotouch_ours/episode_frame_index.npz",
29
+ "label_folder": "/data/egotouch_ours/episodic_annotations",
30
+ "jpeg_root": "/data/preprocessed/egotouch_jpeg",
31
+ "split_dir": "/data/splits/egotouch_v2",
32
+ "weight": 0.3694
33
+ },
34
+ {
35
+ "name": "gigahands",
36
+ "video_root": "/data/gigahands_vitra_mano_strict_v1__CHONGHE/Video/GigaHands_root",
37
+ "annotation_file": "/data/gigahands_ours/episode_frame_index.npz",
38
+ "label_folder": "/data/gigahands_ours/episodic_annotations",
39
+ "jpeg_root": "/data/preprocessed/gigahands_jpeg",
40
+ "split_dir": "/data/splits/gigahands_v1",
41
+ "weight": 0.2811
42
+ },
43
+ {
44
+ "name": "egoexo4d",
45
+ "video_root": "/data/egoexo4d_OFFICIAL/undistorted",
46
+ "annotation_file": "/data/vitra1m_egoexo4d/egoexo4d/episode_frame_index.npz",
47
+ "label_folder": "/data/vitra1m_egoexo4d/egoexo4d/episodic_annotations",
48
+ "jpeg_root": "/data/preprocessed/egoexo4d_jpeg",
49
+ "split_dir": "/data/splits/egoexo4d_v1",
50
+ "weight": 0.2962
51
+ },
52
+ {
53
+ "name": "oakink2",
54
+ "video_root": "/data/oakink2_vitra_streaming_v1__CHONGHE/Video/OakInk2_root",
55
+ "annotation_file": "/data/oakink2_ours/episode_frame_index.npz",
56
+ "label_folder": "/data/oakink2_ours/episodic_annotations",
57
+ "jpeg_root": "/data/preprocessed/oakink2_jpeg",
58
+ "split_dir": "/data/splits/oakink2_v2",
59
+ "weight": 0.2275
60
+ },
61
+ {
62
+ "name": "taco",
63
+ "video_root": "/data/taco_raw/v1/Egocentric_RGB_Videos_x/Egocentric_RGB_Videos",
64
+ "annotation_file": "/data/taco_ours/episode_frame_index.npz",
65
+ "label_folder": "/data/taco_ours/episodic_annotations",
66
+ "jpeg_root": "/data/preprocessed/taco_jpeg",
67
+ "split_dir": "/data/splits/taco_v1",
68
+ "weight": 0.278
69
+ },
70
+ {
71
+ "name": "hot3d",
72
+ "video_root": "/data/hot3d_vitra_atomic_v1__CHONGHE/Video/HOT3D_root",
73
+ "annotation_file": "/data/hot3d_ours/episode_frame_index.npz",
74
+ "label_folder": "/data/hot3d_ours/episodic_annotations",
75
+ "jpeg_root": "/data/preprocessed/hot3d_jpeg",
76
+ "split_dir": "/data/splits/hot3d_v2",
77
+ "weight": 0.3513
78
+ },
79
+ {
80
+ "name": "arctic",
81
+ "video_root": "/data/arctic_vitra/Video/arctic_root",
82
+ "annotation_file": "/data/arctic_vitra/arctic/episode_frame_index.npz",
83
+ "label_folder": "/data/arctic_vitra/arctic/episodic_annotations",
84
+ "jpeg_root": "/data/preprocessed/arctic_jpeg",
85
+ "split_dir": "/data/splits/arctic_v1",
86
+ "weight": 0.3142
87
+ },
88
+ {
89
+ "name": "h2o",
90
+ "video_root": "/data/h2o_vitra/Video/h2o_root",
91
+ "annotation_file": "/data/h2o_ours/episode_frame_index.npz",
92
+ "label_folder": "/data/h2o_ours/episodic_annotations",
93
+ "jpeg_root": "/data/preprocessed/h2o_jpeg",
94
+ "split_dir": "/data/splits/h2o_v2",
95
+ "weight": 0.4535
96
+ },
97
+ {
98
+ "name": "tachin",
99
+ "video_root": "/data/tachin_precut_v1__CHONGHE",
100
+ "annotation_file": "/data/tachin_ours/episode_frame_index.npz",
101
+ "label_folder": "/data/tachin_ours/episodic_annotations",
102
+ "jpeg_root": "/data/preprocessed/tachin_jpeg",
103
+ "split_dir": "/data/splits/tachin_v2",
104
+ "weight": 1.0
105
+ }
106
+ ]
107
+ },
108
+ "config": {
109
+ "optim": {
110
+ "warmup_steps": 1000,
111
+ "decay_steps": 110000
112
+ }
113
+ },
114
+ "snapshot_for_validator": true
115
+ }
model/twla_tachin_common_adaptation_20260922/provenance/metrics.jsonl ADDED
The diff for this file is too large to render. See raw diff
 
model/twla_tachin_common_adaptation_20260922/provenance/verification.json ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "status": "passed",
3
+ "run": "/scratch/hannahyao24/VLA-tactile_paper/runs/twla_tachin_common_adaptation_20260922",
4
+ "selected_step": 800,
5
+ "initial_validation_loss": 0.7941877830099633,
6
+ "best_validation_loss": 0.5552965396678234,
7
+ "validation_reduction": 0.23889124334213985,
8
+ "test_used": false,
9
+ "initial_checkpoint_sha256": "ccd65fcf845ec01c7de46bb76e9cf0fee33674dd848771e114ff23b2535f13d9",
10
+ "adapted_checkpoint_sha256": "8df6c87fda2e5ec09d61686451744c37bdcc2d1b4d9d13b58893851be620db7d",
11
+ "common_complete_sha256": "1cffe026d12e556cc2f66fbda0b34a65a985f50d7aeb0a5adb50571e8437e740",
12
+ "frozen_vlm_memory_tensors_verified": 381,
13
+ "changed_expert_tensors": 155,
14
+ "normalization": "Tachin train-only keypoints statistics"
15
+ }