yzzhao commited on
Commit
fbb44cf
·
verified ·
1 Parent(s): 524a4d6

Update native Kinex Hard results: 82 completed valid episodes

Browse files
data-source.json CHANGED
@@ -1,8 +1,8 @@
1
  {
2
  "schema": "programvla/dataset-source/1",
3
  "dataset": "RLE-Bench/programmable-vla",
4
- "revision": "ec304d3072a3f719de7c49ae06952050e8e2e3c6",
5
- "evidence_root": "https://huggingface.co/datasets/RLE-Bench/programmable-vla/resolve/ec304d3072a3f719de7c49ae06952050e8e2e3c6/",
6
  "files": {
7
  "data.json": {
8
  "url": "./results/data.json"
 
1
  {
2
  "schema": "programvla/dataset-source/1",
3
  "dataset": "RLE-Bench/programmable-vla",
4
+ "revision": "471b89eab26307071173c05c5aa3b08ac0214f0b",
5
+ "evidence_root": "https://huggingface.co/datasets/RLE-Bench/programmable-vla/resolve/471b89eab26307071173c05c5aa3b08ac0214f0b/",
6
  "files": {
7
  "data.json": {
8
  "url": "./results/data.json"
results/cost-data.json CHANGED
@@ -12333,6 +12333,64 @@
12333
  "long_context_responses": 0,
12334
  "cache_write_assumed": false,
12335
  "evidence_url": "experiments/kinex-hard-native-20261113/native-kinex-38-20261113/session.jsonl"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
12336
  }
12337
  },
12338
  "sources": [],
 
12333
  "long_context_responses": 0,
12334
  "cache_write_assumed": false,
12335
  "evidence_url": "experiments/kinex-hard-native-20261113/native-kinex-38-20261113/session.jsonl"
12336
+ },
12337
+ "native-kinex-vlact50k-32-20261113": {
12338
+ "signature": [
12339
+ "native-kinex-vlact50k-32-20261113",
12340
+ "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-32-20261113/usage.json",
12341
+ 1892670,
12342
+ 9487,
12343
+ 1826688,
12344
+ 0,
12345
+ null
12346
+ ],
12347
+ "lower": 2.960858,
12348
+ "upper": 2.960858,
12349
+ "actual_billed_usd": null,
12350
+ "basis": "native_pi_session_api_equivalent",
12351
+ "buckets": [
12352
+ {
12353
+ "category": "fresh_input",
12354
+ "tier": "Recorded session",
12355
+ "tokens": 65982,
12356
+ "rate": 10.0,
12357
+ "rate_upper": 10.0,
12358
+ "lower": 0.6598200000000001,
12359
+ "upper": 0.6598200000000001
12360
+ },
12361
+ {
12362
+ "category": "cache_read",
12363
+ "tier": "Recorded session",
12364
+ "tokens": 1826688,
12365
+ "rate": 1.0,
12366
+ "rate_upper": 1.0,
12367
+ "lower": 1.8266879999999999,
12368
+ "upper": 1.8266879999999999
12369
+ },
12370
+ {
12371
+ "category": "cache_write",
12372
+ "tier": "Recorded session",
12373
+ "tokens": 0,
12374
+ "rate": null,
12375
+ "rate_upper": null,
12376
+ "lower": 0,
12377
+ "upper": 0
12378
+ },
12379
+ {
12380
+ "category": "output",
12381
+ "tier": "Recorded session",
12382
+ "tokens": 9487,
12383
+ "rate": 50.0,
12384
+ "rate_upper": 50.0,
12385
+ "lower": 0.47435000000000005,
12386
+ "upper": 0.47435000000000005
12387
+ }
12388
+ ],
12389
+ "cache_lower": 1826688,
12390
+ "cache_upper": 1826688,
12391
+ "long_context_responses": 0,
12392
+ "cache_write_assumed": false,
12393
+ "evidence_url": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-32-20261113/session.jsonl"
12394
  }
12395
  },
12396
  "sources": [],
results/data.json CHANGED
@@ -127,10 +127,10 @@
127
  "provider": "OpenAI Codex",
128
  "instruction_policy": "original_native",
129
  "instruction_source": "native",
130
- "evaluation_status": "46 / 50 completed valid; 4 running; 0 queued",
131
  "evaluation_date": "2026-11-13",
132
  "native_agent_campaign": true,
133
- "running_episodes": 4,
134
  "pending_episodes": 0,
135
  "evaluation_phase": "running"
136
  },
@@ -143,7 +143,7 @@
143
  "provider": "OpenAI Codex + StarVLA",
144
  "instruction_policy": "original_native",
145
  "instruction_source": "native",
146
- "evaluation_status": "35 / 50 completed valid; 14 running; 0 queued",
147
  "evaluation_date": "2026-11-13",
148
  "native_agent_campaign": true,
149
  "vla_model": "VLAct Qwen3-VL-OFT",
@@ -153,7 +153,7 @@
153
  "input_contract": "images-and-task-instruction",
154
  "checkpoint": "research/kinex-hard-native-20261113/checkpoint.json",
155
  "model_card": "research/kinex-hard-native-20261113/PROTOCOL.md",
156
- "running_episodes": 14,
157
  "pending_episodes": 0,
158
  "evaluation_phase": "running"
159
  }
@@ -10224,7 +10224,7 @@
10224
  }
10225
  ],
10226
  "title": "RoboTwin 2.0 Hard · Native instruction",
10227
- "updated_at": "2026-11-14T00:09:51.709148+00:00",
10228
  "benchmark_complete": false,
10229
  "complete": false,
10230
  "episodes": [
@@ -70409,6 +70409,675 @@
70409
  },
70410
  "runtime_revision": "native-model-device-20261113"
70411
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
70412
  {
70413
  "id": "native-kinex-vlact50k-34-20261113",
70414
  "variant": "kinex-vlact50k-native",
 
127
  "provider": "OpenAI Codex",
128
  "instruction_policy": "original_native",
129
  "instruction_source": "native",
130
+ "evaluation_status": "46 / 50 completed valid; 3 running; 0 queued",
131
  "evaluation_date": "2026-11-13",
132
  "native_agent_campaign": true,
133
+ "running_episodes": 3,
134
  "pending_episodes": 0,
135
  "evaluation_phase": "running"
136
  },
 
143
  "provider": "OpenAI Codex + StarVLA",
144
  "instruction_policy": "original_native",
145
  "instruction_source": "native",
146
+ "evaluation_status": "36 / 50 completed valid; 13 running; 0 queued",
147
  "evaluation_date": "2026-11-13",
148
  "native_agent_campaign": true,
149
  "vla_model": "VLAct Qwen3-VL-OFT",
 
153
  "input_contract": "images-and-task-instruction",
154
  "checkpoint": "research/kinex-hard-native-20261113/checkpoint.json",
155
  "model_card": "research/kinex-hard-native-20261113/PROTOCOL.md",
156
+ "running_episodes": 13,
157
  "pending_episodes": 0,
158
  "evaluation_phase": "running"
159
  }
 
10224
  }
10225
  ],
10226
  "title": "RoboTwin 2.0 Hard · Native instruction",
10227
+ "updated_at": "2026-11-14T00:10:41.619158+00:00",
10228
  "benchmark_complete": false,
10229
  "complete": false,
10230
  "episodes": [
 
70409
  },
70410
  "runtime_revision": "native-model-device-20261113"
70411
  },
70412
+ {
70413
+ "id": "native-kinex-vlact50k-32-20261113",
70414
+ "variant": "kinex-vlact50k-native",
70415
+ "task_key": "task03/32",
70416
+ "family": "task03",
70417
+ "source_family": "task13",
70418
+ "provider": "rosbridge",
70419
+ "slot": "32",
70420
+ "seed": 1,
70421
+ "source_episode_id": "episode_01",
70422
+ "status": "completed",
70423
+ "success": true,
70424
+ "valid": true,
70425
+ "protocol_valid": true,
70426
+ "execution": {
70427
+ "status": "finished",
70428
+ "reason": null
70429
+ },
70430
+ "verdict": {
70431
+ "episode": 1,
70432
+ "initialization": {
70433
+ "expert_plan_success": false,
70434
+ "requested_seed": 1
70435
+ },
70436
+ "native_reward": 1.0,
70437
+ "planned_episodes": 1,
70438
+ "reason": "success",
70439
+ "simulation_time_s": 20.84,
70440
+ "step_limit": 700,
70441
+ "steps": 521,
70442
+ "success": true,
70443
+ "valid": true
70444
+ },
70445
+ "steps": 521,
70446
+ "simulation_time_s": 20.84,
70447
+ "wall_time_s": 2140.804326,
70448
+ "timing": {
70449
+ "simulation_time_s": 20.84,
70450
+ "wall_time_s": 2140.804326,
70451
+ "wall_time_source": "Harbor agent_execution interval; initialization excluded"
70452
+ },
70453
+ "model": "GPT-6 Astra",
70454
+ "effort": "high",
70455
+ "harness": "kinex",
70456
+ "instruction": "Use the left arm to put the red can in the medium basket with smooth texture; the other arm lifts the medium basket with smooth texture.",
70457
+ "native_instruction": "Use the left arm to put the red can in the medium basket with smooth texture; the other arm lifts the medium basket with smooth texture.",
70458
+ "instruction_policy": "original_native",
70459
+ "usage": {
70460
+ "accounting": "reported-responses",
70461
+ "audit_complete": true,
70462
+ "audit_source": "Provider terminal response usage",
70463
+ "cache_hit_rate": 0.9651381381857376,
70464
+ "cache_read_input_tokens": 1826688,
70465
+ "cache_reported_input_tokens": 1892670,
70466
+ "cache_write_basis": "provider-reported",
70467
+ "cache_write_input_tokens": 0,
70468
+ "cache_write_reported_input_tokens": 1892670,
70469
+ "cached_input_tokens": 1826688,
70470
+ "cost_usd": null,
70471
+ "fresh_input_tokens": 65982,
70472
+ "harbor_mismatches": {},
70473
+ "input_breakdown_complete": true,
70474
+ "input_tokens": 1892670,
70475
+ "known_cache_write_input_tokens": 0,
70476
+ "known_cached_input_tokens": 1826688,
70477
+ "known_fresh_input_tokens": 65982,
70478
+ "known_input_tokens": 1892670,
70479
+ "known_output_tokens": 9487,
70480
+ "known_reasoning_output_tokens": 1612,
70481
+ "output_tokens": 9487,
70482
+ "parser": "harbor/0.22.0",
70483
+ "reasoning_output_tokens": 1612,
70484
+ "reasoning_reported_output_tokens": 9487,
70485
+ "rejected_attempts": 0,
70486
+ "reported_responses": {
70487
+ "cache_reported_input_tokens": 54,
70488
+ "cache_write_input_tokens": 54,
70489
+ "cache_write_reported_input_tokens": 54,
70490
+ "cached_input_tokens": 54,
70491
+ "fresh_input_tokens": 54,
70492
+ "input_tokens": 54,
70493
+ "output_tokens": 54,
70494
+ "reasoning_output_tokens": 54,
70495
+ "reasoning_reported_output_tokens": 54
70496
+ },
70497
+ "request_attempts": 55,
70498
+ "response_count": 54,
70499
+ "response_ids_complete": true,
70500
+ "schema": "rlebench/token-usage/1",
70501
+ "source": "Harbor session metrics",
70502
+ "total_tokens": 1902157,
70503
+ "uncached_input_tokens": 65982,
70504
+ "unresolved_attempts": [],
70505
+ "usage_records": [
70506
+ {
70507
+ "cache_write_input_tokens": 0,
70508
+ "cached_input_tokens": 9088,
70509
+ "input_tokens": 11830,
70510
+ "output_tokens": 19,
70511
+ "reasoning_output_tokens": 0
70512
+ },
70513
+ {
70514
+ "cache_write_input_tokens": 0,
70515
+ "cached_input_tokens": 11648,
70516
+ "input_tokens": 11870,
70517
+ "output_tokens": 29,
70518
+ "reasoning_output_tokens": 0
70519
+ },
70520
+ {
70521
+ "cache_write_input_tokens": 0,
70522
+ "cached_input_tokens": 11776,
70523
+ "input_tokens": 12003,
70524
+ "output_tokens": 35,
70525
+ "reasoning_output_tokens": 0
70526
+ },
70527
+ {
70528
+ "cache_write_input_tokens": 0,
70529
+ "cached_input_tokens": 11904,
70530
+ "input_tokens": 12343,
70531
+ "output_tokens": 252,
70532
+ "reasoning_output_tokens": 0
70533
+ },
70534
+ {
70535
+ "cache_write_input_tokens": 0,
70536
+ "cached_input_tokens": 12416,
70537
+ "input_tokens": 17579,
70538
+ "output_tokens": 168,
70539
+ "reasoning_output_tokens": 23
70540
+ },
70541
+ {
70542
+ "cache_write_input_tokens": 0,
70543
+ "cached_input_tokens": 0,
70544
+ "input_tokens": 19861,
70545
+ "output_tokens": 75,
70546
+ "reasoning_output_tokens": 0
70547
+ },
70548
+ {
70549
+ "cache_write_input_tokens": 0,
70550
+ "cached_input_tokens": 19840,
70551
+ "input_tokens": 22696,
70552
+ "output_tokens": 68,
70553
+ "reasoning_output_tokens": 0
70554
+ },
70555
+ {
70556
+ "cache_write_input_tokens": 0,
70557
+ "cached_input_tokens": 17664,
70558
+ "input_tokens": 23860,
70559
+ "output_tokens": 108,
70560
+ "reasoning_output_tokens": 0
70561
+ },
70562
+ {
70563
+ "cache_write_input_tokens": 0,
70564
+ "cached_input_tokens": 22656,
70565
+ "input_tokens": 24186,
70566
+ "output_tokens": 23,
70567
+ "reasoning_output_tokens": 0
70568
+ },
70569
+ {
70570
+ "cache_write_input_tokens": 0,
70571
+ "cached_input_tokens": 24064,
70572
+ "input_tokens": 24378,
70573
+ "output_tokens": 39,
70574
+ "reasoning_output_tokens": 0
70575
+ },
70576
+ {
70577
+ "cache_write_input_tokens": 0,
70578
+ "cached_input_tokens": 24320,
70579
+ "input_tokens": 24786,
70580
+ "output_tokens": 854,
70581
+ "reasoning_output_tokens": 51
70582
+ },
70583
+ {
70584
+ "cache_write_input_tokens": 0,
70585
+ "cached_input_tokens": 25472,
70586
+ "input_tokens": 25661,
70587
+ "output_tokens": 74,
70588
+ "reasoning_output_tokens": 0
70589
+ },
70590
+ {
70591
+ "cache_write_input_tokens": 0,
70592
+ "cached_input_tokens": 25600,
70593
+ "input_tokens": 25880,
70594
+ "output_tokens": 125,
70595
+ "reasoning_output_tokens": 10
70596
+ },
70597
+ {
70598
+ "cache_write_input_tokens": 0,
70599
+ "cached_input_tokens": 25856,
70600
+ "input_tokens": 26032,
70601
+ "output_tokens": 74,
70602
+ "reasoning_output_tokens": 0
70603
+ },
70604
+ {
70605
+ "cache_write_input_tokens": 0,
70606
+ "cached_input_tokens": 23808,
70607
+ "input_tokens": 26921,
70608
+ "output_tokens": 27,
70609
+ "reasoning_output_tokens": 0
70610
+ },
70611
+ {
70612
+ "cache_write_input_tokens": 0,
70613
+ "cached_input_tokens": 26752,
70614
+ "input_tokens": 27010,
70615
+ "output_tokens": 73,
70616
+ "reasoning_output_tokens": 13
70617
+ },
70618
+ {
70619
+ "cache_write_input_tokens": 0,
70620
+ "cached_input_tokens": 26880,
70621
+ "input_tokens": 27622,
70622
+ "output_tokens": 741,
70623
+ "reasoning_output_tokens": 38
70624
+ },
70625
+ {
70626
+ "cache_write_input_tokens": 0,
70627
+ "cached_input_tokens": 28160,
70628
+ "input_tokens": 28390,
70629
+ "output_tokens": 74,
70630
+ "reasoning_output_tokens": 0
70631
+ },
70632
+ {
70633
+ "cache_write_input_tokens": 0,
70634
+ "cached_input_tokens": 28288,
70635
+ "input_tokens": 29545,
70636
+ "output_tokens": 75,
70637
+ "reasoning_output_tokens": 0
70638
+ },
70639
+ {
70640
+ "cache_write_input_tokens": 0,
70641
+ "cached_input_tokens": 29568,
70642
+ "input_tokens": 30709,
70643
+ "output_tokens": 155,
70644
+ "reasoning_output_tokens": 28
70645
+ },
70646
+ {
70647
+ "cache_write_input_tokens": 0,
70648
+ "cached_input_tokens": 30720,
70649
+ "input_tokens": 32231,
70650
+ "output_tokens": 312,
70651
+ "reasoning_output_tokens": 26
70652
+ },
70653
+ {
70654
+ "cache_write_input_tokens": 0,
70655
+ "cached_input_tokens": 32384,
70656
+ "input_tokens": 33341,
70657
+ "output_tokens": 290,
70658
+ "reasoning_output_tokens": 171
70659
+ },
70660
+ {
70661
+ "cache_write_input_tokens": 0,
70662
+ "cached_input_tokens": 33536,
70663
+ "input_tokens": 33776,
70664
+ "output_tokens": 1102,
70665
+ "reasoning_output_tokens": 138
70666
+ },
70667
+ {
70668
+ "cache_write_input_tokens": 0,
70669
+ "cached_input_tokens": 34688,
70670
+ "input_tokens": 34899,
70671
+ "output_tokens": 98,
70672
+ "reasoning_output_tokens": 0
70673
+ },
70674
+ {
70675
+ "cache_write_input_tokens": 0,
70676
+ "cached_input_tokens": 34816,
70677
+ "input_tokens": 35627,
70678
+ "output_tokens": 75,
70679
+ "reasoning_output_tokens": 7
70680
+ },
70681
+ {
70682
+ "cache_write_input_tokens": 0,
70683
+ "cached_input_tokens": 35584,
70684
+ "input_tokens": 35729,
70685
+ "output_tokens": 98,
70686
+ "reasoning_output_tokens": 0
70687
+ },
70688
+ {
70689
+ "cache_write_input_tokens": 0,
70690
+ "cached_input_tokens": 35712,
70691
+ "input_tokens": 36352,
70692
+ "output_tokens": 270,
70693
+ "reasoning_output_tokens": 14
70694
+ },
70695
+ {
70696
+ "cache_write_input_tokens": 0,
70697
+ "cached_input_tokens": 36480,
70698
+ "input_tokens": 36649,
70699
+ "output_tokens": 98,
70700
+ "reasoning_output_tokens": 0
70701
+ },
70702
+ {
70703
+ "cache_write_input_tokens": 0,
70704
+ "cached_input_tokens": 36608,
70705
+ "input_tokens": 37248,
70706
+ "output_tokens": 55,
70707
+ "reasoning_output_tokens": 0
70708
+ },
70709
+ {
70710
+ "cache_write_input_tokens": 0,
70711
+ "cached_input_tokens": 37120,
70712
+ "input_tokens": 37803,
70713
+ "output_tokens": 75,
70714
+ "reasoning_output_tokens": 0
70715
+ },
70716
+ {
70717
+ "cache_write_input_tokens": 0,
70718
+ "cached_input_tokens": 37760,
70719
+ "input_tokens": 38579,
70720
+ "output_tokens": 56,
70721
+ "reasoning_output_tokens": 12
70722
+ },
70723
+ {
70724
+ "cache_write_input_tokens": 0,
70725
+ "cached_input_tokens": 38400,
70726
+ "input_tokens": 40595,
70727
+ "output_tokens": 117,
70728
+ "reasoning_output_tokens": 15
70729
+ },
70730
+ {
70731
+ "cache_write_input_tokens": 0,
70732
+ "cached_input_tokens": 40576,
70733
+ "input_tokens": 40739,
70734
+ "output_tokens": 75,
70735
+ "reasoning_output_tokens": 0
70736
+ },
70737
+ {
70738
+ "cache_write_input_tokens": 0,
70739
+ "cached_input_tokens": 40576,
70740
+ "input_tokens": 41560,
70741
+ "output_tokens": 143,
70742
+ "reasoning_output_tokens": 66
70743
+ },
70744
+ {
70745
+ "cache_write_input_tokens": 0,
70746
+ "cached_input_tokens": 41472,
70747
+ "input_tokens": 42202,
70748
+ "output_tokens": 221,
70749
+ "reasoning_output_tokens": 105
70750
+ },
70751
+ {
70752
+ "cache_write_input_tokens": 0,
70753
+ "cached_input_tokens": 42240,
70754
+ "input_tokens": 42483,
70755
+ "output_tokens": 27,
70756
+ "reasoning_output_tokens": 0
70757
+ },
70758
+ {
70759
+ "cache_write_input_tokens": 0,
70760
+ "cached_input_tokens": 42368,
70761
+ "input_tokens": 42572,
70762
+ "output_tokens": 109,
70763
+ "reasoning_output_tokens": 60
70764
+ },
70765
+ {
70766
+ "cache_write_input_tokens": 0,
70767
+ "cached_input_tokens": 42496,
70768
+ "input_tokens": 43052,
70769
+ "output_tokens": 163,
70770
+ "reasoning_output_tokens": 32
70771
+ },
70772
+ {
70773
+ "cache_write_input_tokens": 0,
70774
+ "cached_input_tokens": 43136,
70775
+ "input_tokens": 43242,
70776
+ "output_tokens": 103,
70777
+ "reasoning_output_tokens": 0
70778
+ },
70779
+ {
70780
+ "cache_write_input_tokens": 0,
70781
+ "cached_input_tokens": 43264,
70782
+ "input_tokens": 43844,
70783
+ "output_tokens": 125,
70784
+ "reasoning_output_tokens": 94
70785
+ },
70786
+ {
70787
+ "cache_write_input_tokens": 0,
70788
+ "cached_input_tokens": 43904,
70789
+ "input_tokens": 44031,
70790
+ "output_tokens": 83,
70791
+ "reasoning_output_tokens": 44
70792
+ },
70793
+ {
70794
+ "cache_write_input_tokens": 0,
70795
+ "cached_input_tokens": 44032,
70796
+ "input_tokens": 44437,
70797
+ "output_tokens": 71,
70798
+ "reasoning_output_tokens": 16
70799
+ },
70800
+ {
70801
+ "cache_write_input_tokens": 0,
70802
+ "cached_input_tokens": 44416,
70803
+ "input_tokens": 45003,
70804
+ "output_tokens": 432,
70805
+ "reasoning_output_tokens": 188
70806
+ },
70807
+ {
70808
+ "cache_write_input_tokens": 0,
70809
+ "cached_input_tokens": 45312,
70810
+ "input_tokens": 45456,
70811
+ "output_tokens": 96,
70812
+ "reasoning_output_tokens": 0
70813
+ },
70814
+ {
70815
+ "cache_write_input_tokens": 0,
70816
+ "cached_input_tokens": 45312,
70817
+ "input_tokens": 46051,
70818
+ "output_tokens": 131,
70819
+ "reasoning_output_tokens": 47
70820
+ },
70821
+ {
70822
+ "cache_write_input_tokens": 0,
70823
+ "cached_input_tokens": 46080,
70824
+ "input_tokens": 46610,
70825
+ "output_tokens": 340,
70826
+ "reasoning_output_tokens": 168
70827
+ },
70828
+ {
70829
+ "cache_write_input_tokens": 0,
70830
+ "cached_input_tokens": 46848,
70831
+ "input_tokens": 47697,
70832
+ "output_tokens": 172,
70833
+ "reasoning_output_tokens": 30
70834
+ },
70835
+ {
70836
+ "cache_write_input_tokens": 0,
70837
+ "cached_input_tokens": 47744,
70838
+ "input_tokens": 48375,
70839
+ "output_tokens": 112,
70840
+ "reasoning_output_tokens": 57
70841
+ },
70842
+ {
70843
+ "cache_write_input_tokens": 0,
70844
+ "cached_input_tokens": 48256,
70845
+ "input_tokens": 48985,
70846
+ "output_tokens": 73,
70847
+ "reasoning_output_tokens": 0
70848
+ },
70849
+ {
70850
+ "cache_write_input_tokens": 0,
70851
+ "cached_input_tokens": 48896,
70852
+ "input_tokens": 49200,
70853
+ "output_tokens": 80,
70854
+ "reasoning_output_tokens": 19
70855
+ },
70856
+ {
70857
+ "cache_write_input_tokens": 0,
70858
+ "cached_input_tokens": 49152,
70859
+ "input_tokens": 49707,
70860
+ "output_tokens": 168,
70861
+ "reasoning_output_tokens": 77
70862
+ },
70863
+ {
70864
+ "cache_write_input_tokens": 0,
70865
+ "cached_input_tokens": 49792,
70866
+ "input_tokens": 50010,
70867
+ "output_tokens": 336,
70868
+ "reasoning_output_tokens": 63
70869
+ },
70870
+ {
70871
+ "cache_write_input_tokens": 0,
70872
+ "cached_input_tokens": 50304,
70873
+ "input_tokens": 50373,
70874
+ "output_tokens": 651,
70875
+ "reasoning_output_tokens": 0
70876
+ },
70877
+ {
70878
+ "cache_write_input_tokens": 0,
70879
+ "cached_input_tokens": 50944,
70880
+ "input_tokens": 51050,
70881
+ "output_tokens": 42,
70882
+ "reasoning_output_tokens": 0
70883
+ }
70884
+ ],
70885
+ "usage_source": "provider-raw"
70886
+ },
70887
+ "call_activity": {
70888
+ "schema": "rlebench/call-activity/1",
70889
+ "source": "Kinex native trace",
70890
+ "model_tool_calls": 65,
70891
+ "model_tool_calls_by_name": {
70892
+ "ls": 1,
70893
+ "read": 18,
70894
+ "bash": 1,
70895
+ "exec": 25,
70896
+ "devices": 5,
70897
+ "device": 1,
70898
+ "write": 4,
70899
+ "edit": 7,
70900
+ "device_stop": 3
70901
+ },
70902
+ "python_instrumented_model_tool_calls": null,
70903
+ "python_tool_invocations": null,
70904
+ "nested_python_tool_invocations": null,
70905
+ "python_tool_invocations_by_origin": null,
70906
+ "python_device_rpc_attempts": null,
70907
+ "python_device_rpc_attempts_by_action": null,
70908
+ "python_device_rpc_errors": null,
70909
+ "python_scope": "Recorded instrumented Python invocations only; other paths and automatic fatal stops are not counted."
70910
+ },
70911
+ "analysis": {
70912
+ "observed": "Native success after 521 rosbridge control frames; 42 optional VLA calls.",
70913
+ "instruction_alignment": "Verbatim native RoboTwin episode instruction.",
70914
+ "confidence": "One authored seed per task in the full 50-task Hard suite."
70915
+ },
70916
+ "links": {
70917
+ "video": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-32-20261113/presentation/video.mp4",
70918
+ "poster": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-32-20261113/presentation/poster.jpg",
70919
+ "terminal_frame": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-32-20261113/presentation/terminal.jpg",
70920
+ "transcript": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-32-20261113/transcript.json",
70921
+ "usage": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-32-20261113/usage.json",
70922
+ "analysis": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-32-20261113/analysis.json",
70923
+ "protocol": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-32-20261113/protocol.json",
70924
+ "native_goal": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-32-20261113/instruction.json",
70925
+ "verdict": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-32-20261113/episode.json",
70926
+ "media_check": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-32-20261113/presentation/media-validation.json",
70927
+ "recording_manifest": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-32-20261113/presentation/media-validation.json",
70928
+ "tool_calls": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-32-20261113/tool-calls.json",
70929
+ "artifact_index": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-32-20261113/artifact-index.json",
70930
+ "vla_io": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-32-20261113/vla-io.json",
70931
+ "evidence_archive": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-32-20261113/evidence.zip",
70932
+ "timing": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-32-20261113/timing.json",
70933
+ "native_session": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-32-20261113/session.jsonl",
70934
+ "trace": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-32-20261113/trace.json",
70935
+ "provider_audit": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-32-20261113/provider-requests.jsonl",
70936
+ "provenance": "research/kinex-hard-native-20261113/source-identities.json",
70937
+ "workspace": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-32-20261113/workspace.zip",
70938
+ "workspace_changes": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-32-20261113/workspace.json"
70939
+ },
70940
+ "resources": [
70941
+ {
70942
+ "name": ".kinex/memos/robotwin-place-can-basket.md",
70943
+ "file": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-32-20261113/workspace/.kinex/memos/robotwin-place-can-basket.md",
70944
+ "kind": "Agent-authored episode resource"
70945
+ },
70946
+ {
70947
+ "name": ".kinex/tools/cartesian_move.py",
70948
+ "file": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-32-20261113/workspace/.kinex/tools/cartesian_move.py",
70949
+ "kind": "Agent-authored episode resource"
70950
+ },
70951
+ {
70952
+ "name": ".kinex/tools/probe_pixels.py",
70953
+ "file": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-32-20261113/workspace/.kinex/tools/probe_pixels.py",
70954
+ "kind": "Agent-authored episode resource"
70955
+ },
70956
+ {
70957
+ "name": ".kinex/tools/run_visual_policy.py",
70958
+ "file": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-32-20261113/workspace/.kinex/tools/run_visual_policy.py",
70959
+ "kind": "Agent-authored episode resource"
70960
+ }
70961
+ ],
70962
+ "session_counts": {
70963
+ "visible_events": 123,
70964
+ "observed_images": 20
70965
+ },
70966
+ "measured_protocol": {
70967
+ "seed": 1,
70968
+ "episodes": 1,
70969
+ "max_steps": 700,
70970
+ "max_control_steps": 700,
70971
+ "task_config": "demo_randomized",
70972
+ "instruction_source": "native",
70973
+ "instruction_policy": "original_native",
70974
+ "provider": "rosbridge",
70975
+ "control_frequency_hz": 25,
70976
+ "mode": "stepped",
70977
+ "admission": "Authored task13 seed; no substitution, expert-success filter, or valid-outcome retry.",
70978
+ "runtime_revision": "native-model-device-20261113",
70979
+ "agent_protocol": "optional-model-device",
70980
+ "input_contract": "images-and-task-instruction",
70981
+ "source_action_period_s": 0.06,
70982
+ "action_chunk_size": 32,
70983
+ "ticks": 4,
70984
+ "tick_s": 0.04
70985
+ },
70986
+ "vla_calls": 42,
70987
+ "kinex_version": "1.1.0",
70988
+ "media": {
70989
+ "speed": 4,
70990
+ "source_fps": 10.0,
70991
+ "output_fps": 40.0,
70992
+ "width": 2880,
70993
+ "height": 720,
70994
+ "native_frames": 210,
70995
+ "published_frames": 210,
70996
+ "decode_ok": true,
70997
+ "duration_s": 5.25,
70998
+ "playback_kind": "simulation_time",
70999
+ "terminal_frame_retained": true,
71000
+ "camera_order": [
71001
+ "third_person",
71002
+ "left_wrist",
71003
+ "right_wrist"
71004
+ ],
71005
+ "sha256": "3edcde6e090708746cc561cb16fd3d208297ccf7b319162c7b327e86ae9fbeab",
71006
+ "source_sha256": {
71007
+ "third_person": "d761838cad9631a2bd7cb60cddfde450f833c2c1b56e01987c8c461683cf7fe2",
71008
+ "left_wrist": "607167b83cab8476721c5ee1bf8a7e4def44ab50bb2dcf6f00a5e1d41806d2ed",
71009
+ "right_wrist": "5c467e099cb2a20de8c6bb616f57981d0a4f381cb906541b8c5455359ccdcfdb"
71010
+ },
71011
+ "recording": {
71012
+ "schema": "rosbridge/recording/1",
71013
+ "run_id": "264d1f662a56",
71014
+ "task": "robotwin/place-can-basket-aloha-agilex-hard",
71015
+ "state": "complete",
71016
+ "views": {
71017
+ "third_person": {
71018
+ "received": 210,
71019
+ "dropped": 0,
71020
+ "segments": [
71021
+ {
71022
+ "file": "third_person.mp4",
71023
+ "frames": 210,
71024
+ "width": 960,
71025
+ "height": 720,
71026
+ "fps": 10.0,
71027
+ "first_stamp_ns": 0,
71028
+ "last_stamp_ns": 20840000000
71029
+ }
71030
+ ],
71031
+ "error": null
71032
+ },
71033
+ "left_wrist": {
71034
+ "received": 210,
71035
+ "dropped": 0,
71036
+ "segments": [
71037
+ {
71038
+ "file": "left_wrist.mp4",
71039
+ "frames": 210,
71040
+ "width": 960,
71041
+ "height": 720,
71042
+ "fps": 10.0,
71043
+ "first_stamp_ns": 0,
71044
+ "last_stamp_ns": 20840000000
71045
+ }
71046
+ ],
71047
+ "error": null
71048
+ },
71049
+ "right_wrist": {
71050
+ "received": 210,
71051
+ "dropped": 0,
71052
+ "segments": [
71053
+ {
71054
+ "file": "right_wrist.mp4",
71055
+ "frames": 210,
71056
+ "width": 960,
71057
+ "height": 720,
71058
+ "fps": 10.0,
71059
+ "first_stamp_ns": 0,
71060
+ "last_stamp_ns": 20840000000
71061
+ }
71062
+ ],
71063
+ "error": null
71064
+ }
71065
+ },
71066
+ "type": "mp4",
71067
+ "spectator_views": [
71068
+ "third_person",
71069
+ "left_wrist",
71070
+ "right_wrist"
71071
+ ],
71072
+ "fps": 10.0,
71073
+ "topics": [],
71074
+ "error": null,
71075
+ "bag_messages": 0
71076
+ },
71077
+ "scope": "All recorded camera frames, composed side by side; 4x playback. No simulator replay."
71078
+ },
71079
+ "runtime_revision": "native-model-device-20261113"
71080
+ },
71081
  {
71082
  "id": "native-kinex-vlact50k-34-20261113",
71083
  "variant": "kinex-vlact50k-native",
results/measured-instructions.json CHANGED
@@ -2777,6 +2777,17 @@
2777
  "instructions_url": "experiments/kinex-hard-native-20261113/native-kinex-38-20261113/instruction.json",
2778
  "transcript_url": "experiments/kinex-hard-native-20261113/native-kinex-38-20261113/transcript.json"
2779
  }
 
 
 
 
 
 
 
 
 
 
 
2780
  }
2781
  }
2782
  }
 
2777
  "instructions_url": "experiments/kinex-hard-native-20261113/native-kinex-38-20261113/instruction.json",
2778
  "transcript_url": "experiments/kinex-hard-native-20261113/native-kinex-38-20261113/transcript.json"
2779
  }
2780
+ },
2781
+ "native-kinex-vlact50k-32-20261113": {
2782
+ "task_key": "task03/32",
2783
+ "initial_goal": "Use the left arm to put the red can in the medium basket with smooth texture; the other arm lifts the medium basket with smooth texture.",
2784
+ "text": "Use the left arm to put the red can in the medium basket with smooth texture; the other arm lifts the medium basket with smooth texture.",
2785
+ "kind": "task_instruction",
2786
+ "observed_native": "Use the left arm to put the red can in the medium basket with smooth texture; the other arm lifts the medium basket with smooth texture.",
2787
+ "source": {
2788
+ "instructions_url": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-32-20261113/instruction.json",
2789
+ "transcript_url": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-32-20261113/transcript.json"
2790
+ }
2791
  }
2792
  }
2793
  }