Publish asterix qwengr00t H8 evaluation artifacts

latency-aware/asterix/vla/starvla-qwengr00t-h1/asterix-qwengr00t-hist8-profile-s42-v1/README.md

@@ -4,4 +4,5 @@ Current RGB and eight binary decision-history windows; action horizon one. Initi
 
 - Training: 5000 steps, seed 42, global batch 128
 - W&B: https://wandb.ai/saberrr-zju/starvla_tasks/runs/946c608a
-- Formal profile evaluation: pending.
+- Evaluation: 100 episodes, seeds 1000000–1000099, GPU simulator, parallel 32, capacity 1, 60/15 Hz, 7200 raw-frame cap, profile latency
+- Mean raw return: 1440.0

latency-aware/asterix/vla/starvla-qwengr00t-h1/asterix-qwengr00t-hist8-profile-s42-v1/eval_profile.json

@@ -0,0 +1,217 @@
+{
+  "checkpoint_path": "/home/ubuntu/codex/atari-hist8-20260924/outputs/tasks/atari_hist8_v1/bundles/asterix-qwengr00t-hist8-profile-s42-v1/checkpoints/model.pt",
+  "experiment_name": "asterix-qwengr00t-hist8-profile-s42-v1-step5000-history01-profile-par32",
+  "latency": "profile_sample",
+  "latency_type": "profile_sample",
+  "lengths": [
+    1797,
+    3534,
+    1787,
+    1940,
+    1745,
+    1221,
+    3143,
+    2165,
+    3764,
+    2680,
+    3073,
+    2185,
+    3880,
+    1421,
+    3063,
+    2716,
+    2619,
+    2873,
+    1682,
+    1905,
+    5738,
+    1248,
+    1393,
+    1736,
+    1236,
+    2030,
+    879,
+    1488,
+    3805,
+    1691,
+    1658,
+    2749,
+    2177,
+    1246,
+    1254,
+    2064,
+    2189,
+    1506,
+    3512,
+    2011,
+    1240,
+    1464,
+    2134,
+    1432,
+    1848,
+    1812,
+    3767,
+    3698,
+    2468,
+    2438,
+    2736,
+    2748,
+    2585,
+    1590,
+    2521,
+    2279,
+    1905,
+    2431,
+    2015,
+    2923,
+    2786,
+    2356,
+    1681,
+    3525,
+    3086,
+    1221,
+    2647,
+    1450,
+    1922,
+    1415,
+    2128,
+    5399,
+    2345,
+    2612,
+    1603,
+    4164,
+    1758,
+    1711,
+    2426,
+    2117,
+    2146,
+    2855,
+    1657,
+    2845,
+    4279,
+    1927,
+    2719,
+    1433,
+    2712,
+    2436,
+    1844,
+    1921,
+    2436,
+    2211,
+    2026,
+    1901,
+    3391,
+    2115,
+    2466,
+    2738
+  ],
+  "mean_length": 2332.47,
+  "mean_return": 1440.0,
+  "returns": [
+    1150.0,
+    2400.0,
+    850.0,
+    1300.0,
+    1050.0,
+    650.0,
+    1700.0,
+    1200.0,
+    3000.0,
+    1550.0,
+    2000.0,
+    1500.0,
+    3100.0,
+    750.0,
+    1700.0,
+    1650.0,
+    1350.0,
+    1900.0,
+    1050.0,
+    1350.0,
+    4300.0,
+    750.0,
+    800.0,
+    750.0,
+    600.0,
+    1200.0,
+    350.0,
+    800.0,
+    1750.0,
+    850.0,
+    900.0,
+    1500.0,
+    1550.0,
+    750.0,
+    800.0,
+    1500.0,
+    1450.0,
+    650.0,
+    2050.0,
+    1400.0,
+    750.0,
+    850.0,
+    1450.0,
+    850.0,
+    1050.0,
+    1200.0,
+    2500.0,
+    1950.0,
+    1700.0,
+    1200.0,
+    1700.0,
+    1850.0,
+    1750.0,
+    950.0,
+    1850.0,
+    1000.0,
+    1150.0,
+    1450.0,
+    1200.0,
+    1700.0,
+    2100.0,
+    1400.0,
+    900.0,
+    2600.0,
+    2000.0,
+    550.0,
+    1500.0,
+    900.0,
+    950.0,
+    750.0,
+    1150.0,
+    4000.0,
+    1300.0,
+    1450.0,
+    1150.0,
+    2800.0,
+    1050.0,
+    1200.0,
+    1200.0,
+    1250.0,
+    1350.0,
+    1850.0,
+    950.0,
+    1600.0,
+    3400.0,
+    1150.0,
+    1800.0,
+    950.0,
+    1550.0,
+    1350.0,
+    900.0,
+    1050.0,
+    1500.0,
+    1300.0,
+    1200.0,
+    1100.0,
+    2450.0,
+    1200.0,
+    1250.0,
+    1900.0
+  ],
+  "seed": 1000000,
+  "source_profile_path": "/home/ubuntu/codex/atari-hist8-20260924/outputs/tasks/atari_hist8_v1/teacher_configs/asterix-qwengr00t/profile/profile.json",
+  "std_return": 686.6221668428715,
+  "suite_name": "profile_sample",
+  "timestamp_utc": "2026-09-24T19:36:00.626346+00:00"
+}

latency-aware/asterix/vla/starvla-qwengr00t-h1/asterix-qwengr00t-hist8-profile-s42-v1/evaluation_profile/action_audit.json

@@ -0,0 +1,153 @@
+{
+  "eval_protocol_version": "hist8-history01-sf-render-v4",
+  "history_encoding": "binary_0_1",
+  "episodes": 100,
+  "episode_seeds": [
+    1000000,
+    1000001,
+    1000002,
+    1000003,
+    1000004,
+    1000005,
+    1000006,
+    1000007,
+    1000008,
+    1000009,
+    1000010,
+    1000011,
+    1000012,
+    1000013,
+    1000014,
+    1000015,
+    1000016,
+    1000017,
+    1000018,
+    1000019,
+    1000020,
+    1000021,
+    1000022,
+    1000023,
+    1000024,
+    1000025,
+    1000026,
+    1000027,
+    1000028,
+    1000029,
+    1000030,
+    1000031,
+    1000032,
+    1000033,
+    1000034,
+    1000035,
+    1000036,
+    1000037,
+    1000038,
+    1000039,
+    1000040,
+    1000041,
+    1000042,
+    1000043,
+    1000044,
+    1000045,
+    1000046,
+    1000047,
+    1000048,
+    1000049,
+    1000050,
+    1000051,
+    1000052,
+    1000053,
+    1000054,
+    1000055,
+    1000056,
+    1000057,
+    1000058,
+    1000059,
+    1000060,
+    1000061,
+    1000062,
+    1000063,
+    1000064,
+    1000065,
+    1000066,
+    1000067,
+    1000068,
+    1000069,
+    1000070,
+    1000071,
+    1000072,
+    1000073,
+    1000074,
+    1000075,
+    1000076,
+    1000077,
+    1000078,
+    1000079,
+    1000080,
+    1000081,
+    1000082,
+    1000083,
+    1000084,
+    1000085,
+    1000086,
+    1000087,
+    1000088,
+    1000089,
+    1000090,
+    1000091,
+    1000092,
+    1000093,
+    1000094,
+    1000095,
+    1000096,
+    1000097,
+    1000098,
+    1000099
+  ],
+  "issued_actions": 14615,
+  "action_counts": {
+    "7": 1807,
+    "6": 1173,
+    "5": 9122,
+    "2": 772,
+    "8": 1720,
+    "3": 11,
+    "4": 5,
+    "0": 2,
+    "1": 3
+  },
+  "largest_action_fraction": 0.6241532671912419,
+  "raw_score_min": [
+    -0.1035124659538269,
+    -0.12095272541046143,
+    -0.06751096248626709,
+    -0.0891040563583374,
+    -0.09031903743743896,
+    -0.13926947116851807,
+    -0.07776820659637451,
+    -0.06621575355529785,
+    -0.1278035044670105
+  ],
+  "raw_score_max": [
+    1.0027379989624023,
+    1.0079847574234009,
+    1.0860569477081299,
+    0.9871726632118225,
+    0.972202718257904,
+    1.0391428470611572,
+    1.1037511825561523,
+    1.3245031833648682,
+    1.0730587244033813
+  ],
+  "raw_score_mean": [
+    -0.00413584264603533,
+    0.0007604830563476222,
+    0.05498602419388756,
+    0.0050188393725717605,
+    0.0030477551167803424,
+    0.6137112245914245,
+    0.08402741889386803,
+    0.12560979057526522,
+    0.11776857277662967
+  ]
+}

latency-aware/asterix/vla/starvla-qwengr00t-h1/asterix-qwengr00t-hist8-profile-s42-v1/evaluation_profile/eval_config.yaml

@@ -0,0 +1,165 @@
+experiment:
+  name: asterix-qwengr00t-hist8-profile-s42-v1-step5000-history01-profile-par32
+  seed: 1000000
+backend:
+  type: sample_factory
+  algo: APPO
+  device: cuda
+  train_dir: results/sample_factory
+  restart_behavior: resume
+  run_mode: eval
+  extra_args:
+  - --encoder_conv_architecture
+  - convnet_simple
+executor:
+  mode: simulated
+  simulated_worker_capacity: 1
+  simulated_inference_pool: true
+  inference_devices:
+  - cuda:0
+  - cuda:1
+  inference_batch_size: 16
+env:
+  name: gymnasium
+  task_name: asterix
+  env_id: LatencyBench/Asterix-v0
+  registration_imports:
+  - latency_bench.envs.gymnasium_asterix
+  make_kwargs:
+    base_env_id: ALE/Asterix-v5
+    render_mode: rgb_array
+    screen_size: 84
+    noop_max: 30
+    base_make_kwargs:
+      obs_type: rgb
+      frameskip: 1
+      repeat_action_probability: 0.0
+      full_action_space: false
+      mode: 0
+      difficulty: 0
+      max_num_frames_per_episode: 108000
+  env_fps: 60
+  obs_fps: 15
+  frame_stack: 1
+  simulator: gpu
+  terminal_on_life_loss: false
+  clip_reward: false
+  noop_action: noop
+  action_map:
+    noop: 0
+    up: 1
+    right: 2
+    left: 3
+    down: 4
+    upright: 5
+    upleft: 6
+    downright: 7
+    downleft: 8
+  action_order:
+  - noop
+  - up
+  - right
+  - left
+  - down
+  - upright
+  - upleft
+  - downright
+  - downleft
+  base_prompt: 'Collect useful objects and avoid lyres. Choose exactly one action
+    from: noop, up, right, left, down, upright, upleft, downright, downleft.'
+  obs_resize:
+  - 224
+  - 224
+  gpu_env_device: auto
+  noop_max: 30
+  action_history_decisions: 8
+latency:
+  method: temporal
+  profile_path: /home/ubuntu/codex/atari-hist8-20260924/outputs/tasks/atari_hist8_v1/teacher_configs/asterix-qwengr00t/profile/profile.json
+  profile_worker_slot: 0
+  seed: 271828
+  add_latency_info: false
+scheduler:
+  hold_policy: hold
+  ordering_policy: issue_order_fifo
+policy:
+  type: starvla
+  actions:
+  - noop
+  - up
+  - right
+  - left
+  - down
+  - upright
+  - upleft
+  - downright
+  - downleft
+  checkpoint_path: /home/ubuntu/codex/atari-hist8-20260924/outputs/tasks/atari_hist8_v1/bundles/asterix-qwengr00t-hist8-profile-s42-v1/checkpoints/model.pt
+  model_config_path: /home/ubuntu/codex/atari-hist8-20260924/outputs/tasks/atari_hist8_v1/bundles/asterix-qwengr00t-hist8-profile-s42-v1/config.yaml
+  task_manifest_path: /home/ubuntu/codex/atari-hist8-20260924/outputs/tasks/atari_hist8_v1/bundles/asterix-qwengr00t-hist8-profile-s42-v1/manifest.json
+  device: cuda:0
+  unnorm_key: new_embodiment
+  prompt_mode: latency_neutral
+  action_layout: discrete_9
+  backbone_path: /home/ubuntu/.cache/huggingface/hub/models--Qwen--Qwen3-VL-4B-Instruct/snapshots/ebb281ec70b05090aa6165b016eac8ec08e71b17
+  worker_python_executable: /home/ubuntu/miniconda3/envs/starvla_rl_games_openvla/bin/python3.10
+  state_source: transport
+training:
+  train_for_env_steps: 1000000
+  num_workers: 4
+  num_envs_per_worker: 16
+  worker_num_splits: 1
+  num_policies: 1
+  batch_size: 1024
+  rollout: 64
+  recurrence: 32
+  num_epochs: 2
+  num_batches_per_epoch: 8
+  max_policy_lag: 300
+  learning_rate: 0.000303
+  lr_schedule: constant
+  nonlinearity: elu
+  gamma: 0.99
+  gae_lambda: 0.95
+  ppo_clip_ratio: 0.1
+  ppo_clip_value: 0.2
+  exploration_loss: entropy
+  exploration_loss_coeff: 0.003
+  value_loss_coeff: 0.5
+  max_grad_norm: 0.0
+  adam_eps: 1.0e-06
+  obs_scale: 1.0
+  adaptive_stddev: false
+  with_vtrace: false
+  async_rl: true
+  use_rnn: false
+  normalize_input: true
+  normalize_returns: true
+  stats_avg: 100
+  experiment_summaries_interval: 1
+  save_every_sec: 600
+  keep_checkpoints: 5
+evaluation:
+  eval_interval_steps: null
+  eval_episodes: 100
+  eval_parallel_envs: 32
+  eval_max_steps: 7200
+  eval_deterministic: true
+  eval_suites:
+    fixed: []
+    normal: []
+    uniform: []
+  latency_bench_env_backend: asterix_gpu_batched
+  eval_raw_reward: true
+logging:
+  output_dir: /home/ubuntu/codex/atari-hist8-20260924/outputs/tasks/atari_hist8_v1/evals/asterix-qwengr00t-hist8-profile-s42-v1/profile_to_profile
+  video:
+    enabled: false
+    num_bins: 1
+  save_step_records: true
+  save_action_records: true
+  save_latency_records: true
+  wandb_project: null
+  wandb_group: null
+  wandb_job_type: null
+  wandb_tags: ''

latency-aware/asterix/vla/starvla-qwengr00t-h1/asterix-qwengr00t-hist8-profile-s42-v1/evaluation_profile/provenance.json

@@ -0,0 +1,46 @@
+{
+  "collection": {
+    "teacher_repo": "latency-sensitive-bench/benchmark-models",
+    "teacher_revision": "b2fe384570e74126afe1b9f75ca523747809bd36",
+    "teacher_path": "latency-aware/asterix/small-policy/sample-factory-qwengr00t-hist8-cpu15m-s0-v1",
+    "teacher_checkpoint": "best_000027296_13975552_reward_1865.000.pth",
+    "teacher_sha256": "8fc78a650079a3fc14de7667d3a4750edfa4d7481762719888be571559cfa8e0",
+    "profile_path": "/home/ubuntu/codex/atari-hist8-20260924/results/profiling/profiles/qwengr00t/1x-rtx3090/asterix/instance_d907cdaf680040e0/profile.json",
+    "profile_sha256": "5e104064d718ac51ce56aefac5318a155fd099f5c714ff8f0bc4985509639fd1",
+    "profile_repo": "latency-sensitive-bench/profiles",
+    "profile_revision": "15001405a402e30b7706476bc92aea45a61bb193",
+    "rollout_overrides": {
+      "latency_profile_path": "/home/ubuntu/codex/atari-hist8-20260924/results/profiling/profiles/qwengr00t/1x-rtx3090/asterix/instance_d907cdaf680040e0/profile.json",
+      "seed": 42,
+      "simulator": "gpu",
+      "probe_envs": 256,
+      "replay_envs": 256,
+      "policy_inference_batch_size": 256,
+      "max_episode_raw_frames": 7200
+    },
+    "episodes": 550,
+    "filter_preset": "none",
+    "history": "decision_v1"
+  },
+  "training_run_id": "asterix-qwengr00t-hist8-profile-s42-v1",
+  "training_step": 5000,
+  "global_batch": 128,
+  "base_model": "Qwen/Qwen3-VL-4B-Instruct",
+  "wandb_run_id": "946c608a",
+  "checkpoint_sha256": "a192db0f4f9707271e453150467a4d833c29479f3e6f6f8031039e84dab8afcf",
+  "eval_protocol_version": "hist8-history01-sf-render-v4",
+  "history_encoding": "binary_0_1",
+  "episodes": 100,
+  "env_condition": "profile",
+  "parallel_envs": 32,
+  "max_episode_raw_frames": 7200,
+  "inference_devices": [
+    "cuda:0",
+    "cuda:1"
+  ],
+  "episode_seeds": [
+    1000000,
+    1000099
+  ],
+  "profile_sha256": "5e104064d718ac51ce56aefac5318a155fd099f5c714ff8f0bc4985509639fd1"
+}

latency-aware/asterix/vla/starvla-qwengr00t-h1/asterix-qwengr00t-hist8-profile-s42-v1/profile_publication.json

@@ -0,0 +1,29 @@
+{
+  "model": "qwengr00t",
+  "model_publication": {
+    "checkpoint_sha256": "7b3541140bd2d00d39eca98c8788b773e594884a623ae20d7b9e4dd7b45337f5",
+    "episodes": 100,
+    "mean_return": 5714.0,
+    "model": "qwengr00t",
+    "path": "zero-latency/asterix/vla/starvla-qwengr00t-h1/asterix-qwengr00t-hist8-zero-s42-v1",
+    "repo_id": "latency-sensitive-bench/benchmark-models",
+    "revision": "d445334970ab7e110c3cacb5c8b790d3df20e1ec",
+    "run_id": "asterix-qwengr00t-hist8-zero-s42-v1",
+    "task": "asterix",
+    "training_status": "complete",
+    "zero_evaluation_status": "complete",
+    "zero_mean_return": 5714.0
+  },
+  "profile": {
+    "GPU_CLASS": "1x-rtx3090",
+    "INSTANCE_ID": "instance_d907cdaf680040e0",
+    "PROFILE_PATH": "results/profiling/profiles/qwengr00t/1x-rtx3090/asterix/instance_d907cdaf680040e0/profile.json",
+    "PROFILE_REF": "qwengr00t/1x-rtx3090/asterix/instance_d907cdaf680040e0",
+    "RUN_ID": "20260924T122426657466Z"
+  },
+  "profile_path": "profiles/qwengr00t/1x-rtx3090/asterix/instance_d907cdaf680040e0/profile.json",
+  "profile_repo": "latency-sensitive-bench/profiles",
+  "profile_revision": "15001405a402e30b7706476bc92aea45a61bb193",
+  "profile_sha256": "5e104064d718ac51ce56aefac5318a155fd099f5c714ff8f0bc4985509639fd1",
+  "task": "asterix"
+}

latency-aware/asterix/vla/starvla-qwengr00t-h1/asterix-qwengr00t-hist8-profile-s42-v1/publication_status.json

@@ -1,4 +1,7 @@
 {
-  "profile_evaluation_status": "pending",
+  "episodes": 100,
+  "mean_return": 1440.0,
+  "profile_evaluation_status": "complete",
+  "profile_mean_return": 1440.0,
   "training_status": "complete"
 }