yzzhao commited on
Commit
cdb65a5
·
verified ·
1 Parent(s): ed270d6

Update native Kinex Hard results: 94 completed valid episodes

Browse files
data-source.json CHANGED
@@ -1,8 +1,8 @@
1
  {
2
  "schema": "programvla/dataset-source/1",
3
  "dataset": "RLE-Bench/programmable-vla",
4
- "revision": "cb4127679d3fc24111dde621fbc98fb460229668",
5
- "evidence_root": "https://huggingface.co/datasets/RLE-Bench/programmable-vla/resolve/cb4127679d3fc24111dde621fbc98fb460229668/",
6
  "files": {
7
  "data.json": {
8
  "url": "./results/data.json"
 
1
  {
2
  "schema": "programvla/dataset-source/1",
3
  "dataset": "RLE-Bench/programmable-vla",
4
+ "revision": "1544dfe655c88efbab6d454629449ca11a67e5ef",
5
+ "evidence_root": "https://huggingface.co/datasets/RLE-Bench/programmable-vla/resolve/1544dfe655c88efbab6d454629449ca11a67e5ef/",
6
  "files": {
7
  "data.json": {
8
  "url": "./results/data.json"
results/cost-data.json CHANGED
@@ -13029,6 +13029,64 @@
13029
  "long_context_responses": 0,
13030
  "cache_write_assumed": false,
13031
  "evidence_url": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-45-20261113/session.jsonl"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
13032
  }
13033
  },
13034
  "sources": [],
 
13029
  "long_context_responses": 0,
13030
  "cache_write_assumed": false,
13031
  "evidence_url": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-45-20261113/session.jsonl"
13032
+ },
13033
+ "native-kinex-vlact50k-41-20261113": {
13034
+ "signature": [
13035
+ "native-kinex-vlact50k-41-20261113",
13036
+ "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-41-20261113/usage.json",
13037
+ 1601489,
13038
+ 8290,
13039
+ 1535104,
13040
+ 0,
13041
+ null
13042
+ ],
13043
+ "lower": 2.613454,
13044
+ "upper": 2.613454,
13045
+ "actual_billed_usd": null,
13046
+ "basis": "native_pi_session_api_equivalent",
13047
+ "buckets": [
13048
+ {
13049
+ "category": "fresh_input",
13050
+ "tier": "Recorded session",
13051
+ "tokens": 66385,
13052
+ "rate": 10.0,
13053
+ "rate_upper": 10.0,
13054
+ "lower": 0.66385,
13055
+ "upper": 0.66385
13056
+ },
13057
+ {
13058
+ "category": "cache_read",
13059
+ "tier": "Recorded session",
13060
+ "tokens": 1535104,
13061
+ "rate": 1.0,
13062
+ "rate_upper": 1.0,
13063
+ "lower": 1.535104,
13064
+ "upper": 1.535104
13065
+ },
13066
+ {
13067
+ "category": "cache_write",
13068
+ "tier": "Recorded session",
13069
+ "tokens": 0,
13070
+ "rate": null,
13071
+ "rate_upper": null,
13072
+ "lower": 0,
13073
+ "upper": 0
13074
+ },
13075
+ {
13076
+ "category": "output",
13077
+ "tier": "Recorded session",
13078
+ "tokens": 8290,
13079
+ "rate": 50.0,
13080
+ "rate_upper": 50.0,
13081
+ "lower": 0.41450000000000004,
13082
+ "upper": 0.41450000000000004
13083
+ }
13084
+ ],
13085
+ "cache_lower": 1535104,
13086
+ "cache_upper": 1535104,
13087
+ "long_context_responses": 0,
13088
+ "cache_write_assumed": false,
13089
+ "evidence_url": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-41-20261113/session.jsonl"
13090
  }
13091
  },
13092
  "sources": [],
results/data.json CHANGED
@@ -143,7 +143,7 @@
143
  "provider": "OpenAI Codex + StarVLA",
144
  "instruction_policy": "original_native",
145
  "instruction_source": "native",
146
- "evaluation_status": "43 / 50 completed valid; 6 running; 0 queued",
147
  "evaluation_date": "2026-11-13",
148
  "native_agent_campaign": true,
149
  "vla_model": "VLAct Qwen3-VL-OFT",
@@ -153,7 +153,7 @@
153
  "input_contract": "images-and-task-instruction",
154
  "checkpoint": "research/kinex-hard-native-20261113/checkpoint.json",
155
  "model_card": "research/kinex-hard-native-20261113/PROTOCOL.md",
156
- "running_episodes": 6,
157
  "pending_episodes": 0,
158
  "evaluation_phase": "running"
159
  }
@@ -10224,7 +10224,7 @@
10224
  }
10225
  ],
10226
  "title": "RoboTwin 2.0 Hard · Native instruction",
10227
- "updated_at": "2026-11-14T00:22:14.775008+00:00",
10228
  "benchmark_complete": false,
10229
  "complete": false,
10230
  "episodes": [
@@ -78422,6 +78422,651 @@
78422
  },
78423
  "runtime_revision": "native-model-device-20261113"
78424
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
78425
  {
78426
  "id": "native-kinex-vlact50k-42-20261113",
78427
  "variant": "kinex-vlact50k-native",
 
143
  "provider": "OpenAI Codex + StarVLA",
144
  "instruction_policy": "original_native",
145
  "instruction_source": "native",
146
+ "evaluation_status": "44 / 50 completed valid; 5 running; 0 queued",
147
  "evaluation_date": "2026-11-13",
148
  "native_agent_campaign": true,
149
  "vla_model": "VLAct Qwen3-VL-OFT",
 
153
  "input_contract": "images-and-task-instruction",
154
  "checkpoint": "research/kinex-hard-native-20261113/checkpoint.json",
155
  "model_card": "research/kinex-hard-native-20261113/PROTOCOL.md",
156
+ "running_episodes": 5,
157
  "pending_episodes": 0,
158
  "evaluation_phase": "running"
159
  }
 
10224
  }
10225
  ],
10226
  "title": "RoboTwin 2.0 Hard · Native instruction",
10227
+ "updated_at": "2026-11-14T00:24:55.856971+00:00",
10228
  "benchmark_complete": false,
10229
  "complete": false,
10230
  "episodes": [
 
78422
  },
78423
  "runtime_revision": "native-model-device-20261113"
78424
  },
78425
+ {
78426
+ "id": "native-kinex-vlact50k-41-20261113",
78427
+ "variant": "kinex-vlact50k-native",
78428
+ "task_key": "task03/41",
78429
+ "family": "task03",
78430
+ "source_family": "task13",
78431
+ "provider": "rosbridge",
78432
+ "slot": "41",
78433
+ "seed": 0,
78434
+ "source_episode_id": "episode_01",
78435
+ "status": "completed",
78436
+ "success": false,
78437
+ "valid": true,
78438
+ "protocol_valid": true,
78439
+ "execution": {
78440
+ "status": "finished",
78441
+ "reason": null
78442
+ },
78443
+ "verdict": {
78444
+ "episode": 1,
78445
+ "initialization": {
78446
+ "expert_plan_success": false,
78447
+ "requested_seed": 0
78448
+ },
78449
+ "native_reward": 0.0,
78450
+ "planned_episodes": 1,
78451
+ "reason": "step_limit",
78452
+ "simulation_time_s": 16.0,
78453
+ "step_limit": 400,
78454
+ "steps": 400,
78455
+ "success": false,
78456
+ "valid": true
78457
+ },
78458
+ "steps": 400,
78459
+ "simulation_time_s": 16.0,
78460
+ "wall_time_s": 2022.783411,
78461
+ "timing": {
78462
+ "simulation_time_s": 16.0,
78463
+ "wall_time_s": 2022.783411,
78464
+ "wall_time_source": "Harbor agent_execution interval; initialization excluded"
78465
+ },
78466
+ "model": "GPT-6 Astra",
78467
+ "effort": "high",
78468
+ "harness": "kinex",
78469
+ "instruction": "Set the phone with shiny black back on the L-shaped smooth brown phonestand.",
78470
+ "native_instruction": "Set the phone with shiny black back on the L-shaped smooth brown phonestand.",
78471
+ "instruction_policy": "original_native",
78472
+ "usage": {
78473
+ "accounting": "reported-responses",
78474
+ "audit_complete": true,
78475
+ "audit_source": "Provider terminal response usage",
78476
+ "cache_hit_rate": 0.9585479513128096,
78477
+ "cache_read_input_tokens": 1535104,
78478
+ "cache_reported_input_tokens": 1601489,
78479
+ "cache_write_basis": "provider-reported",
78480
+ "cache_write_input_tokens": 0,
78481
+ "cache_write_reported_input_tokens": 1601489,
78482
+ "cached_input_tokens": 1535104,
78483
+ "cost_usd": null,
78484
+ "fresh_input_tokens": 66385,
78485
+ "harbor_mismatches": {},
78486
+ "input_breakdown_complete": true,
78487
+ "input_tokens": 1601489,
78488
+ "known_cache_write_input_tokens": 0,
78489
+ "known_cached_input_tokens": 1535104,
78490
+ "known_fresh_input_tokens": 66385,
78491
+ "known_input_tokens": 1601489,
78492
+ "known_output_tokens": 8290,
78493
+ "known_reasoning_output_tokens": 2026,
78494
+ "output_tokens": 8290,
78495
+ "parser": "harbor/0.22.0",
78496
+ "reasoning_output_tokens": 2026,
78497
+ "reasoning_reported_output_tokens": 8290,
78498
+ "rejected_attempts": 0,
78499
+ "reported_responses": {
78500
+ "cache_reported_input_tokens": 51,
78501
+ "cache_write_input_tokens": 51,
78502
+ "cache_write_reported_input_tokens": 51,
78503
+ "cached_input_tokens": 51,
78504
+ "fresh_input_tokens": 51,
78505
+ "input_tokens": 51,
78506
+ "output_tokens": 51,
78507
+ "reasoning_output_tokens": 51,
78508
+ "reasoning_reported_output_tokens": 51
78509
+ },
78510
+ "request_attempts": 51,
78511
+ "response_count": 51,
78512
+ "response_ids_complete": true,
78513
+ "schema": "rlebench/token-usage/1",
78514
+ "source": "Harbor session metrics",
78515
+ "total_tokens": 1609779,
78516
+ "uncached_input_tokens": 66385,
78517
+ "unresolved_attempts": [],
78518
+ "usage_records": [
78519
+ {
78520
+ "cache_write_input_tokens": 0,
78521
+ "cached_input_tokens": 9088,
78522
+ "input_tokens": 11820,
78523
+ "output_tokens": 19,
78524
+ "reasoning_output_tokens": 0
78525
+ },
78526
+ {
78527
+ "cache_write_input_tokens": 0,
78528
+ "cached_input_tokens": 11648,
78529
+ "input_tokens": 11860,
78530
+ "output_tokens": 29,
78531
+ "reasoning_output_tokens": 0
78532
+ },
78533
+ {
78534
+ "cache_write_input_tokens": 0,
78535
+ "cached_input_tokens": 9088,
78536
+ "input_tokens": 11993,
78537
+ "output_tokens": 20,
78538
+ "reasoning_output_tokens": 0
78539
+ },
78540
+ {
78541
+ "cache_write_input_tokens": 0,
78542
+ "cached_input_tokens": 11776,
78543
+ "input_tokens": 12041,
78544
+ "output_tokens": 28,
78545
+ "reasoning_output_tokens": 0
78546
+ },
78547
+ {
78548
+ "cache_write_input_tokens": 0,
78549
+ "cached_input_tokens": 11904,
78550
+ "input_tokens": 14120,
78551
+ "output_tokens": 49,
78552
+ "reasoning_output_tokens": 0
78553
+ },
78554
+ {
78555
+ "cache_write_input_tokens": 0,
78556
+ "cached_input_tokens": 14080,
78557
+ "input_tokens": 14409,
78558
+ "output_tokens": 193,
78559
+ "reasoning_output_tokens": 0
78560
+ },
78561
+ {
78562
+ "cache_write_input_tokens": 0,
78563
+ "cached_input_tokens": 14464,
78564
+ "input_tokens": 18012,
78565
+ "output_tokens": 191,
78566
+ "reasoning_output_tokens": 16
78567
+ },
78568
+ {
78569
+ "cache_write_input_tokens": 0,
78570
+ "cached_input_tokens": 18048,
78571
+ "input_tokens": 21846,
78572
+ "output_tokens": 108,
78573
+ "reasoning_output_tokens": 0
78574
+ },
78575
+ {
78576
+ "cache_write_input_tokens": 0,
78577
+ "cached_input_tokens": 21760,
78578
+ "input_tokens": 22172,
78579
+ "output_tokens": 73,
78580
+ "reasoning_output_tokens": 0
78581
+ },
78582
+ {
78583
+ "cache_write_input_tokens": 0,
78584
+ "cached_input_tokens": 22144,
78585
+ "input_tokens": 23589,
78586
+ "output_tokens": 23,
78587
+ "reasoning_output_tokens": 0
78588
+ },
78589
+ {
78590
+ "cache_write_input_tokens": 0,
78591
+ "cached_input_tokens": 23424,
78592
+ "input_tokens": 23779,
78593
+ "output_tokens": 39,
78594
+ "reasoning_output_tokens": 0
78595
+ },
78596
+ {
78597
+ "cache_write_input_tokens": 0,
78598
+ "cached_input_tokens": 23680,
78599
+ "input_tokens": 24187,
78600
+ "output_tokens": 661,
78601
+ "reasoning_output_tokens": 32
78602
+ },
78603
+ {
78604
+ "cache_write_input_tokens": 0,
78605
+ "cached_input_tokens": 24704,
78606
+ "input_tokens": 24869,
78607
+ "output_tokens": 62,
78608
+ "reasoning_output_tokens": 0
78609
+ },
78610
+ {
78611
+ "cache_write_input_tokens": 0,
78612
+ "cached_input_tokens": 24832,
78613
+ "input_tokens": 25076,
78614
+ "output_tokens": 45,
78615
+ "reasoning_output_tokens": 15
78616
+ },
78617
+ {
78618
+ "cache_write_input_tokens": 0,
78619
+ "cached_input_tokens": 24960,
78620
+ "input_tokens": 25183,
78621
+ "output_tokens": 51,
78622
+ "reasoning_output_tokens": 14
78623
+ },
78624
+ {
78625
+ "cache_write_input_tokens": 0,
78626
+ "cached_input_tokens": 25088,
78627
+ "input_tokens": 25266,
78628
+ "output_tokens": 39,
78629
+ "reasoning_output_tokens": 0
78630
+ },
78631
+ {
78632
+ "cache_write_input_tokens": 0,
78633
+ "cached_input_tokens": 11904,
78634
+ "input_tokens": 26219,
78635
+ "output_tokens": 126,
78636
+ "reasoning_output_tokens": 0
78637
+ },
78638
+ {
78639
+ "cache_write_input_tokens": 0,
78640
+ "cached_input_tokens": 25088,
78641
+ "input_tokens": 26372,
78642
+ "output_tokens": 62,
78643
+ "reasoning_output_tokens": 0
78644
+ },
78645
+ {
78646
+ "cache_write_input_tokens": 0,
78647
+ "cached_input_tokens": 26240,
78648
+ "input_tokens": 27248,
78649
+ "output_tokens": 30,
78650
+ "reasoning_output_tokens": 0
78651
+ },
78652
+ {
78653
+ "cache_write_input_tokens": 0,
78654
+ "cached_input_tokens": 27136,
78655
+ "input_tokens": 27340,
78656
+ "output_tokens": 64,
78657
+ "reasoning_output_tokens": 26
78658
+ },
78659
+ {
78660
+ "cache_write_input_tokens": 0,
78661
+ "cached_input_tokens": 26240,
78662
+ "input_tokens": 27606,
78663
+ "output_tokens": 71,
78664
+ "reasoning_output_tokens": 30
78665
+ },
78666
+ {
78667
+ "cache_write_input_tokens": 0,
78668
+ "cached_input_tokens": 27520,
78669
+ "input_tokens": 28473,
78670
+ "output_tokens": 654,
78671
+ "reasoning_output_tokens": 13
78672
+ },
78673
+ {
78674
+ "cache_write_input_tokens": 0,
78675
+ "cached_input_tokens": 28928,
78676
+ "input_tokens": 29148,
78677
+ "output_tokens": 59,
78678
+ "reasoning_output_tokens": 0
78679
+ },
78680
+ {
78681
+ "cache_write_input_tokens": 0,
78682
+ "cached_input_tokens": 29056,
78683
+ "input_tokens": 30230,
78684
+ "output_tokens": 61,
78685
+ "reasoning_output_tokens": 0
78686
+ },
78687
+ {
78688
+ "cache_write_input_tokens": 0,
78689
+ "cached_input_tokens": 30208,
78690
+ "input_tokens": 31322,
78691
+ "output_tokens": 72,
78692
+ "reasoning_output_tokens": 8
78693
+ },
78694
+ {
78695
+ "cache_write_input_tokens": 0,
78696
+ "cached_input_tokens": 31232,
78697
+ "input_tokens": 31454,
78698
+ "output_tokens": 32,
78699
+ "reasoning_output_tokens": 0
78700
+ },
78701
+ {
78702
+ "cache_write_input_tokens": 0,
78703
+ "cached_input_tokens": 31360,
78704
+ "input_tokens": 31548,
78705
+ "output_tokens": 48,
78706
+ "reasoning_output_tokens": 0
78707
+ },
78708
+ {
78709
+ "cache_write_input_tokens": 0,
78710
+ "cached_input_tokens": 31488,
78711
+ "input_tokens": 31856,
78712
+ "output_tokens": 84,
78713
+ "reasoning_output_tokens": 21
78714
+ },
78715
+ {
78716
+ "cache_write_input_tokens": 0,
78717
+ "cached_input_tokens": 31744,
78718
+ "input_tokens": 32982,
78719
+ "output_tokens": 310,
78720
+ "reasoning_output_tokens": 59
78721
+ },
78722
+ {
78723
+ "cache_write_input_tokens": 0,
78724
+ "cached_input_tokens": 33152,
78725
+ "input_tokens": 33778,
78726
+ "output_tokens": 106,
78727
+ "reasoning_output_tokens": 44
78728
+ },
78729
+ {
78730
+ "cache_write_input_tokens": 0,
78731
+ "cached_input_tokens": 33792,
78732
+ "input_tokens": 34761,
78733
+ "output_tokens": 110,
78734
+ "reasoning_output_tokens": 17
78735
+ },
78736
+ {
78737
+ "cache_write_input_tokens": 0,
78738
+ "cached_input_tokens": 34688,
78739
+ "input_tokens": 35545,
78740
+ "output_tokens": 345,
78741
+ "reasoning_output_tokens": 137
78742
+ },
78743
+ {
78744
+ "cache_write_input_tokens": 0,
78745
+ "cached_input_tokens": 35712,
78746
+ "input_tokens": 36177,
78747
+ "output_tokens": 741,
78748
+ "reasoning_output_tokens": 91
78749
+ },
78750
+ {
78751
+ "cache_write_input_tokens": 0,
78752
+ "cached_input_tokens": 36736,
78753
+ "input_tokens": 36939,
78754
+ "output_tokens": 95,
78755
+ "reasoning_output_tokens": 0
78756
+ },
78757
+ {
78758
+ "cache_write_input_tokens": 0,
78759
+ "cached_input_tokens": 36864,
78760
+ "input_tokens": 37415,
78761
+ "output_tokens": 223,
78762
+ "reasoning_output_tokens": 17
78763
+ },
78764
+ {
78765
+ "cache_write_input_tokens": 0,
78766
+ "cached_input_tokens": 37504,
78767
+ "input_tokens": 37911,
78768
+ "output_tokens": 97,
78769
+ "reasoning_output_tokens": 26
78770
+ },
78771
+ {
78772
+ "cache_write_input_tokens": 0,
78773
+ "cached_input_tokens": 27264,
78774
+ "input_tokens": 38394,
78775
+ "output_tokens": 97,
78776
+ "reasoning_output_tokens": 33
78777
+ },
78778
+ {
78779
+ "cache_write_input_tokens": 0,
78780
+ "cached_input_tokens": 37888,
78781
+ "input_tokens": 39362,
78782
+ "output_tokens": 73,
78783
+ "reasoning_output_tokens": 11
78784
+ },
78785
+ {
78786
+ "cache_write_input_tokens": 0,
78787
+ "cached_input_tokens": 39296,
78788
+ "input_tokens": 40482,
78789
+ "output_tokens": 70,
78790
+ "reasoning_output_tokens": 8
78791
+ },
78792
+ {
78793
+ "cache_write_input_tokens": 0,
78794
+ "cached_input_tokens": 40448,
78795
+ "input_tokens": 41597,
78796
+ "output_tokens": 324,
78797
+ "reasoning_output_tokens": 79
78798
+ },
78799
+ {
78800
+ "cache_write_input_tokens": 0,
78801
+ "cached_input_tokens": 41728,
78802
+ "input_tokens": 42288,
78803
+ "output_tokens": 764,
78804
+ "reasoning_output_tokens": 556
78805
+ },
78806
+ {
78807
+ "cache_write_input_tokens": 0,
78808
+ "cached_input_tokens": 42880,
78809
+ "input_tokens": 43339,
78810
+ "output_tokens": 440,
78811
+ "reasoning_output_tokens": 285
78812
+ },
78813
+ {
78814
+ "cache_write_input_tokens": 0,
78815
+ "cached_input_tokens": 43648,
78816
+ "input_tokens": 43934,
78817
+ "output_tokens": 347,
78818
+ "reasoning_output_tokens": 240
78819
+ },
78820
+ {
78821
+ "cache_write_input_tokens": 0,
78822
+ "cached_input_tokens": 44032,
78823
+ "input_tokens": 44665,
78824
+ "output_tokens": 183,
78825
+ "reasoning_output_tokens": 77
78826
+ },
78827
+ {
78828
+ "cache_write_input_tokens": 0,
78829
+ "cached_input_tokens": 44800,
78830
+ "input_tokens": 45233,
78831
+ "output_tokens": 51,
78832
+ "reasoning_output_tokens": 13
78833
+ },
78834
+ {
78835
+ "cache_write_input_tokens": 0,
78836
+ "cached_input_tokens": 45184,
78837
+ "input_tokens": 45306,
78838
+ "output_tokens": 158,
78839
+ "reasoning_output_tokens": 28
78840
+ },
78841
+ {
78842
+ "cache_write_input_tokens": 0,
78843
+ "cached_input_tokens": 45312,
78844
+ "input_tokens": 45655,
78845
+ "output_tokens": 69,
78846
+ "reasoning_output_tokens": 0
78847
+ },
78848
+ {
78849
+ "cache_write_input_tokens": 0,
78850
+ "cached_input_tokens": 45568,
78851
+ "input_tokens": 46112,
78852
+ "output_tokens": 102,
78853
+ "reasoning_output_tokens": 51
78854
+ },
78855
+ {
78856
+ "cache_write_input_tokens": 0,
78857
+ "cached_input_tokens": 46080,
78858
+ "input_tokens": 46596,
78859
+ "output_tokens": 98,
78860
+ "reasoning_output_tokens": 65
78861
+ },
78862
+ {
78863
+ "cache_write_input_tokens": 0,
78864
+ "cached_input_tokens": 46592,
78865
+ "input_tokens": 46747,
78866
+ "output_tokens": 463,
78867
+ "reasoning_output_tokens": 14
78868
+ },
78869
+ {
78870
+ "cache_write_input_tokens": 0,
78871
+ "cached_input_tokens": 47104,
78872
+ "input_tokens": 47233,
78873
+ "output_tokens": 31,
78874
+ "reasoning_output_tokens": 0
78875
+ }
78876
+ ],
78877
+ "usage_source": "provider-raw"
78878
+ },
78879
+ "call_activity": {
78880
+ "schema": "rlebench/call-activity/1",
78881
+ "source": "Kinex native trace",
78882
+ "model_tool_calls": 60,
78883
+ "model_tool_calls_by_name": {
78884
+ "ls": 2,
78885
+ "read": 17,
78886
+ "find": 1,
78887
+ "bash": 1,
78888
+ "devices": 4,
78889
+ "exec": 23,
78890
+ "device": 2,
78891
+ "write": 4,
78892
+ "device_stop": 4,
78893
+ "grep": 1,
78894
+ "edit": 1
78895
+ },
78896
+ "python_instrumented_model_tool_calls": null,
78897
+ "python_tool_invocations": null,
78898
+ "nested_python_tool_invocations": null,
78899
+ "python_tool_invocations_by_origin": null,
78900
+ "python_device_rpc_attempts": null,
78901
+ "python_device_rpc_attempts_by_action": null,
78902
+ "python_device_rpc_errors": null,
78903
+ "python_scope": "Recorded instrumented Python invocations only; other paths and automatic fatal stops are not counted."
78904
+ },
78905
+ "analysis": {
78906
+ "observed": "Native unsuccessful outcome after 400 rosbridge control frames; 70 optional VLA calls.",
78907
+ "instruction_alignment": "Verbatim native RoboTwin episode instruction.",
78908
+ "confidence": "One authored seed per task in the full 50-task Hard suite."
78909
+ },
78910
+ "links": {
78911
+ "video": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-41-20261113/presentation/video.mp4",
78912
+ "poster": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-41-20261113/presentation/poster.jpg",
78913
+ "terminal_frame": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-41-20261113/presentation/terminal.jpg",
78914
+ "transcript": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-41-20261113/transcript.json",
78915
+ "usage": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-41-20261113/usage.json",
78916
+ "analysis": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-41-20261113/analysis.json",
78917
+ "protocol": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-41-20261113/protocol.json",
78918
+ "native_goal": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-41-20261113/instruction.json",
78919
+ "verdict": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-41-20261113/episode.json",
78920
+ "media_check": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-41-20261113/presentation/media-validation.json",
78921
+ "recording_manifest": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-41-20261113/presentation/media-validation.json",
78922
+ "tool_calls": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-41-20261113/tool-calls.json",
78923
+ "artifact_index": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-41-20261113/artifact-index.json",
78924
+ "vla_io": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-41-20261113/vla-io.json",
78925
+ "evidence_archive": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-41-20261113/evidence.zip",
78926
+ "timing": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-41-20261113/timing.json",
78927
+ "native_session": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-41-20261113/session.jsonl",
78928
+ "trace": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-41-20261113/trace.json",
78929
+ "provider_audit": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-41-20261113/provider-requests.jsonl",
78930
+ "provenance": "research/kinex-hard-native-20261113/source-identities.json",
78931
+ "workspace": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-41-20261113/workspace.zip",
78932
+ "workspace_changes": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-41-20261113/workspace.json"
78933
+ },
78934
+ "resources": [
78935
+ {
78936
+ "name": ".kinex/memos/phone-stand.md",
78937
+ "file": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-41-20261113/workspace/.kinex/memos/phone-stand.md",
78938
+ "kind": "Agent-authored episode resource"
78939
+ },
78940
+ {
78941
+ "name": ".kinex/tools/cartesian_step.py",
78942
+ "file": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-41-20261113/workspace/.kinex/tools/cartesian_step.py",
78943
+ "kind": "Agent-authored episode resource"
78944
+ },
78945
+ {
78946
+ "name": ".kinex/tools/policy_step.py",
78947
+ "file": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-41-20261113/workspace/.kinex/tools/policy_step.py",
78948
+ "kind": "Agent-authored episode resource"
78949
+ }
78950
+ ],
78951
+ "session_counts": {
78952
+ "visible_events": 114,
78953
+ "observed_images": 33
78954
+ },
78955
+ "measured_protocol": {
78956
+ "seed": 0,
78957
+ "episodes": 1,
78958
+ "max_steps": 400,
78959
+ "max_control_steps": 400,
78960
+ "task_config": "demo_randomized",
78961
+ "instruction_source": "native",
78962
+ "instruction_policy": "original_native",
78963
+ "provider": "rosbridge",
78964
+ "control_frequency_hz": 25,
78965
+ "mode": "stepped",
78966
+ "admission": "Authored task13 seed; no substitution, expert-success filter, or valid-outcome retry.",
78967
+ "runtime_revision": "native-model-device-20261113",
78968
+ "agent_protocol": "optional-model-device",
78969
+ "input_contract": "images-and-task-instruction",
78970
+ "source_action_period_s": 0.06,
78971
+ "action_chunk_size": 32,
78972
+ "ticks": 4,
78973
+ "tick_s": 0.04
78974
+ },
78975
+ "vla_calls": 70,
78976
+ "kinex_version": "1.1.0",
78977
+ "media": {
78978
+ "speed": 4,
78979
+ "source_fps": 10.0,
78980
+ "output_fps": 40.0,
78981
+ "width": 2880,
78982
+ "height": 720,
78983
+ "native_frames": 161,
78984
+ "published_frames": 161,
78985
+ "decode_ok": true,
78986
+ "duration_s": 4.025,
78987
+ "playback_kind": "simulation_time",
78988
+ "terminal_frame_retained": true,
78989
+ "camera_order": [
78990
+ "third_person",
78991
+ "left_wrist",
78992
+ "right_wrist"
78993
+ ],
78994
+ "sha256": "8ca3fe7667ba55b5591c09713df15642c451afc0b2788283eefe96fb5766f50d",
78995
+ "source_sha256": {
78996
+ "third_person": "a2a65b339b6e7d0d73abfb1a8cf4e462617370fd15f441ef77dd1ec649ab507a",
78997
+ "left_wrist": "69bc290ed91d76ef4ceca031ada539719c4e04dccb1bdaa2aad31fd04f67220b",
78998
+ "right_wrist": "9d9f8e3b539e87141b22797228127a0c18305f139e0699b3ecc1e237b9cf7d36"
78999
+ },
79000
+ "recording": {
79001
+ "schema": "rosbridge/recording/1",
79002
+ "run_id": "cd389a2713d8",
79003
+ "task": "robotwin/place-phone-stand-aloha-agilex-hard",
79004
+ "state": "complete",
79005
+ "views": {
79006
+ "third_person": {
79007
+ "received": 162,
79008
+ "dropped": 0,
79009
+ "segments": [
79010
+ {
79011
+ "file": "third_person.mp4",
79012
+ "frames": 161,
79013
+ "width": 960,
79014
+ "height": 720,
79015
+ "fps": 10.0,
79016
+ "first_stamp_ns": 0,
79017
+ "last_stamp_ns": 16000000000
79018
+ }
79019
+ ],
79020
+ "error": null
79021
+ },
79022
+ "left_wrist": {
79023
+ "received": 162,
79024
+ "dropped": 0,
79025
+ "segments": [
79026
+ {
79027
+ "file": "left_wrist.mp4",
79028
+ "frames": 161,
79029
+ "width": 960,
79030
+ "height": 720,
79031
+ "fps": 10.0,
79032
+ "first_stamp_ns": 0,
79033
+ "last_stamp_ns": 16000000000
79034
+ }
79035
+ ],
79036
+ "error": null
79037
+ },
79038
+ "right_wrist": {
79039
+ "received": 162,
79040
+ "dropped": 0,
79041
+ "segments": [
79042
+ {
79043
+ "file": "right_wrist.mp4",
79044
+ "frames": 161,
79045
+ "width": 960,
79046
+ "height": 720,
79047
+ "fps": 10.0,
79048
+ "first_stamp_ns": 0,
79049
+ "last_stamp_ns": 16000000000
79050
+ }
79051
+ ],
79052
+ "error": null
79053
+ }
79054
+ },
79055
+ "type": "mp4",
79056
+ "spectator_views": [
79057
+ "third_person",
79058
+ "left_wrist",
79059
+ "right_wrist"
79060
+ ],
79061
+ "fps": 10.0,
79062
+ "topics": [],
79063
+ "error": null,
79064
+ "bag_messages": 0
79065
+ },
79066
+ "scope": "All recorded camera frames, composed side by side; 4x playback. No simulator replay."
79067
+ },
79068
+ "runtime_revision": "native-model-device-20261113"
79069
+ },
79070
  {
79071
  "id": "native-kinex-vlact50k-42-20261113",
79072
  "variant": "kinex-vlact50k-native",
results/measured-instructions.json CHANGED
@@ -2909,6 +2909,17 @@
2909
  "instructions_url": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-45-20261113/instruction.json",
2910
  "transcript_url": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-45-20261113/transcript.json"
2911
  }
 
 
 
 
 
 
 
 
 
 
 
2912
  }
2913
  }
2914
  }
 
2909
  "instructions_url": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-45-20261113/instruction.json",
2910
  "transcript_url": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-45-20261113/transcript.json"
2911
  }
2912
+ },
2913
+ "native-kinex-vlact50k-41-20261113": {
2914
+ "task_key": "task03/41",
2915
+ "initial_goal": "Set the phone with shiny black back on the L-shaped smooth brown phonestand.",
2916
+ "text": "Set the phone with shiny black back on the L-shaped smooth brown phonestand.",
2917
+ "kind": "task_instruction",
2918
+ "observed_native": "Set the phone with shiny black back on the L-shaped smooth brown phonestand.",
2919
+ "source": {
2920
+ "instructions_url": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-41-20261113/instruction.json",
2921
+ "transcript_url": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-41-20261113/transcript.json"
2922
+ }
2923
  }
2924
  }
2925
  }