Upload combined SO-101 FFT checkpoint

config.json

@@ -0,0 +1,71 @@
+{
+  "architectures": [
+    "RBM"
+  ],
+  "dtype": "bfloat16",
+  "image_token_id": 151655,
+  "model_name": "unsloth/Qwen3-VL-4B-Instruct",
+  "model_type": "qwen3_vl",
+  "pad_token_id": 151654,
+  "text_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bos_token_id": 151643,
+    "dtype": "bfloat16",
+    "eos_token_id": 151645,
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "hidden_size": 2560,
+    "initializer_range": 0.02,
+    "intermediate_size": 9728,
+    "max_position_embeddings": 262144,
+    "model_type": "qwen3_vl_text",
+    "num_attention_heads": 32,
+    "num_hidden_layers": 36,
+    "num_key_value_heads": 8,
+    "pad_token_id": null,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "mrope_interleaved": true,
+      "mrope_section": [
+        24,
+        20,
+        20
+      ],
+      "rope_theta": 5000000,
+      "rope_type": "default"
+    },
+    "tie_word_embeddings": true,
+    "use_cache": true,
+    "vocab_size": 151674
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.3.0",
+  "unsloth_fixed": true,
+  "unsloth_version": "2026.3.11",
+  "use_cache": false,
+  "video_token_id": 151656,
+  "vision_config": {
+    "deepstack_visual_indexes": [
+      5,
+      11,
+      17
+    ],
+    "depth": 24,
+    "dtype": "bfloat16",
+    "hidden_act": "gelu_pytorch_tanh",
+    "hidden_size": 1024,
+    "in_channels": 3,
+    "initializer_range": 0.02,
+    "intermediate_size": 4096,
+    "model_type": "qwen3_vl",
+    "num_heads": 16,
+    "num_position_embeddings": 2304,
+    "out_hidden_size": 2560,
+    "patch_size": 16,
+    "spatial_merge_size": 2,
+    "temporal_patch_size": 2
+  },
+  "vision_end_token_id": 151653,
+  "vision_start_token_id": 151652
+}

config.yaml

@@ -0,0 +1,185 @@
+custom_eval:
+  comparisons_per_task: 5
+  confusion_matrix:
+  - rbm-1m-ood
+  confusion_matrix_n_trajectories_per_source: 5
+  custom_eval_random_seed: 42
+  eval_types:
+  - reward_alignment
+  - policy_ranking
+  max_comparisons: null
+  num_examples_per_quality_pr: 5
+  num_partial_successes: 5
+  pad_frames: true
+  policy_ranking:
+  - villekuosmanen_armnetbench_robometer_v01_so101
+  policy_ranking_max_tasks: 100
+  quality_preference:
+  - rbm-1m-ood
+  reward_alignment:
+  - villekuosmanen_armnetbench_robometer_v01_so101
+  - ykorkmaz_molmoact2_so100_101_rbm_molmoact2_so100_101
+  reward_alignment_max_trajectories: 10
+  subsample_n_frames: null
+  use_frame_steps: true
+data:
+  data_source_weights: null
+  dataloader_num_workers: 8
+  dataloader_persistent_workers: true
+  dataloader_pin_memory: true
+  dataset_preference_ratio: 0.7
+  dataset_success_cutoff_file: robometer/data/dataset_success_cutoff.txt
+  dataset_type: rbm
+  eval_datasets:
+  - villekuosmanen_armnetbench_robometer_v01_so101
+  - ykorkmaz_molmoact2_so100_101_rbm_molmoact2_so100_101
+  eval_subset_size: null
+  load_embeddings: false
+  max_frames: 8
+  max_frames_after_preprocessing: 64
+  max_success: 1.0
+  max_trajectories: -1
+  min_frames_per_trajectory: 5
+  min_success: 0.5
+  partial_success_threshold: 0.2
+  predict_last_frame_partial_progress: false
+  preference_strategy_ratio:
+  - 1.0
+  - 1.0
+  - 1.0
+  - 1.0
+  progress_discrete_bins: 10
+  progress_loss_type: discrete
+  progress_pred_type: absolute_first_frame
+  progress_strategy_ratio:
+  - 1.0
+  - 1.0
+  - 1.0
+  - 1.0
+  resized_height: null
+  resized_width: null
+  sample_type_ratio:
+  - 1.0
+  - 0.0
+  - 0.0
+  seed: 42
+  shuffle: true
+  shuffle_progress_frames: false
+  train_datasets:
+  - villekuosmanen_armnetbench_robometer_v01_so101
+  - ykorkmaz_molmoact2_so100_101_rbm_molmoact2_so100_101
+  traj_same_source_prob: 0.5
+  use_multi_image: true
+  use_per_frame_progress_token: true
+debug: false
+logging:
+  log_level: INFO
+  log_to: []
+  save_best:
+    greater_is_better:
+    - true
+    - true
+    - true
+    hub_private: false
+    hub_save_every: 1000
+    hub_token: null
+    keep_top_k: 5
+    metric_names:
+    - eval_rew_align/pearson_villekuosmanen_armnetbench_robometer_v01_so101
+    - eval_p_rank/kendall_last_villekuosmanen_armnetbench_robometer_v01_so101
+    - eval_rew_align/pearson_ykorkmaz_molmoact2_so100_101_rbm_molmoact2_so100_101
+    save_every: 1000
+    upload_to_hub: false
+  save_model: true
+  save_processor: true
+  wandb_entity: jesbu1
+  wandb_mode: null
+  wandb_notes: training Robometer
+  wandb_project: robometer
+loss:
+  predict_last_frame_progress: false
+  progress_discrete_bins: 10
+  progress_loss_type: discrete
+  success_positive_weight: 1.0
+mode: train
+model:
+  average_temporal_patches: true
+  base_model_id: Qwen/Qwen3-VL-4B-Instruct
+  frame_pooling: mean
+  frame_pooling_attn_temperature: 1.0
+  model_type: default
+  peft_vision_encoder: false
+  progress_discrete_bins: 10
+  progress_loss_type: discrete
+  quantization: false
+  rewind: null
+  torch_dtype: bfloat16
+  train_language_model: true
+  train_preference_head: true
+  train_progress_head: true
+  train_success_head: true
+  train_vision_encoder: false
+  trust_remote_code: true
+  use_multi_image: true
+  use_peft: false
+  use_per_frame_progress_token: true
+  use_unsloth: true
+peft:
+  bias: none
+  lora_alpha: 64
+  lora_dropout: 0.05
+  peft_vision_encoder: false
+  r: 32
+  target_modules:
+  - q_proj
+  - k_proj
+  - v_proj
+  - o_proj
+  - gate_proj
+  - up_proj
+  - down_proj
+trainer_cls: rbm_heads
+training:
+  beta: 0.1
+  bf16: true
+  custom_eval_steps: 250
+  dataloader_num_workers: 8
+  dataloader_persistent_workers: true
+  dataloader_pin_memory: true
+  ddp_bucket_cap_mb: 25
+  ddp_find_unused_parameters: false
+  do_eval: true
+  eval_steps: 250
+  evaluation_strategy: steps
+  exp_name: rbm_fft_both_so101_qwen3_4b_16bs_4gpu
+  fp16: false
+  gradient_accumulation_steps: 1
+  gradient_checkpointing: true
+  learning_rate: 2.0e-05
+  load_from_checkpoint: robometer/Robometer-4B
+  logging_steps: 1
+  lr_scheduler_type: cosine
+  max_grad_norm: 10.0
+  max_seq_length: 1024
+  max_steps: 1500
+  num_gpus: 2
+  num_train_epochs: -1
+  optim: adamw_8bit
+  output_dir: ./logs
+  overwrite_output_dir: true
+  per_device_eval_batch_size: 16
+  per_device_train_batch_size: 16
+  predict_pref_progress: true
+  prediction_loss_only: true
+  remove_unused_columns: false
+  resume_from_checkpoint: null
+  run_default_eval: false
+  save_steps: 200
+  save_strategy: 'no'
+  torch_compile: false
+  torch_compile_backend: inductor
+  vision_encoder_lr: 1.0e-05
+  vision_encoder_num_layers: 3
+  warmup_ratio: 0.1
+  warmup_steps: 0
+  weight_decay: 0.01

metrics.json

@@ -0,0 +1,64 @@
+{
+  "step": 750,
+  "metrics": {
+    "eval_rew_align/success_auprc_villekuosmanen_armnetbench_robometer_v01_so101": 0.6899881016045726,
+    "eval_rew_align/positive_success_acc_villekuosmanen_armnetbench_robometer_v01_so101": 0.775,
+    "eval_rew_align/negative_success_acc_villekuosmanen_armnetbench_robometer_v01_so101": 0.9880952380952381,
+    "eval_rew_align/loss_villekuosmanen_armnetbench_robometer_v01_so101": 1.5189456224441529,
+    "eval_rew_align/pearson_villekuosmanen_armnetbench_robometer_v01_so101": 0.7819677384703819,
+    "eval_p_rank/kendall_last_villekuosmanen_armnetbench_robometer_v01_so101": 0.94,
+    "eval_p_rank/kendall_rewind_last_villekuosmanen_armnetbench_robometer_v01_so101": 1.0,
+    "eval_p_rank/avg_succ_subopt_diff_last_villekuosmanen_armnetbench_robometer_v01_so101": 0.26076613664627074,
+    "eval_p_rank/min_succ_subopt_diff_last_villekuosmanen_armnetbench_robometer_v01_so101": 0.2388161063194275,
+    "eval_p_rank/max_succ_subopt_diff_last_villekuosmanen_armnetbench_robometer_v01_so101": 0.28271616697311397,
+    "eval_p_rank/avg_subopt_fail_diff_last_villekuosmanen_armnetbench_robometer_v01_so101": 0.25212376415729526,
+    "eval_p_rank/min_subopt_fail_diff_last_villekuosmanen_armnetbench_robometer_v01_so101": 0.147011947631836,
+    "eval_p_rank/max_subopt_fail_diff_last_villekuosmanen_armnetbench_robometer_v01_so101": 0.3572355806827545,
+    "eval_p_rank/avg_succ_fail_diff_last_villekuosmanen_armnetbench_robometer_v01_so101": 0.5191158263012767,
+    "eval_p_rank/min_succ_fail_diff_last_villekuosmanen_armnetbench_robometer_v01_so101": 0.25460606813430786,
+    "eval_p_rank/max_succ_fail_diff_last_villekuosmanen_armnetbench_robometer_v01_so101": 0.6971883609890939,
+    "eval_p_rank/ranking_acc_last_villekuosmanen_armnetbench_robometer_v01_so101": 0.9733333333333334,
+    "eval_p_rank/ranking_acc_all_pairs_last_villekuosmanen_armnetbench_robometer_v01_so101": 0.9733333333333334,
+    "eval_p_rank/ranking_acc_failure_vs_successful_last_villekuosmanen_armnetbench_robometer_v01_so101": 0.975,
+    "eval_p_rank/ranking_acc_suboptimal_vs_successful_last_villekuosmanen_armnetbench_robometer_v01_so101": 0.98,
+    "eval_p_rank/ranking_acc_failure_vs_suboptimal_last_villekuosmanen_armnetbench_robometer_v01_so101": 0.96,
+    "eval_p_rank/kendall_avg_villekuosmanen_armnetbench_robometer_v01_so101": 0.94,
+    "eval_p_rank/kendall_rewind_avg_villekuosmanen_armnetbench_robometer_v01_so101": 1.0,
+    "eval_p_rank/avg_succ_subopt_diff_avg_villekuosmanen_armnetbench_robometer_v01_so101": 0.26076613664627074,
+    "eval_p_rank/min_succ_subopt_diff_avg_villekuosmanen_armnetbench_robometer_v01_so101": 0.2388161063194275,
+    "eval_p_rank/max_succ_subopt_diff_avg_villekuosmanen_armnetbench_robometer_v01_so101": 0.28271616697311397,
+    "eval_p_rank/avg_subopt_fail_diff_avg_villekuosmanen_armnetbench_robometer_v01_so101": 0.25212376415729526,
+    "eval_p_rank/min_subopt_fail_diff_avg_villekuosmanen_armnetbench_robometer_v01_so101": 0.147011947631836,
+    "eval_p_rank/max_subopt_fail_diff_avg_villekuosmanen_armnetbench_robometer_v01_so101": 0.3572355806827545,
+    "eval_p_rank/avg_succ_fail_diff_avg_villekuosmanen_armnetbench_robometer_v01_so101": 0.5191158263012767,
+    "eval_p_rank/min_succ_fail_diff_avg_villekuosmanen_armnetbench_robometer_v01_so101": 0.25460606813430786,
+    "eval_p_rank/max_succ_fail_diff_avg_villekuosmanen_armnetbench_robometer_v01_so101": 0.6971883609890939,
+    "eval_p_rank/ranking_acc_avg_villekuosmanen_armnetbench_robometer_v01_so101": 0.9733333333333334,
+    "eval_p_rank/ranking_acc_all_pairs_avg_villekuosmanen_armnetbench_robometer_v01_so101": 0.9733333333333334,
+    "eval_p_rank/ranking_acc_failure_vs_successful_avg_villekuosmanen_armnetbench_robometer_v01_so101": 0.975,
+    "eval_p_rank/ranking_acc_suboptimal_vs_successful_avg_villekuosmanen_armnetbench_robometer_v01_so101": 0.98,
+    "eval_p_rank/ranking_acc_failure_vs_suboptimal_avg_villekuosmanen_armnetbench_robometer_v01_so101": 0.96,
+    "eval_p_rank/kendall_sum_villekuosmanen_armnetbench_robometer_v01_so101": 0.94,
+    "eval_p_rank/kendall_rewind_sum_villekuosmanen_armnetbench_robometer_v01_so101": 1.0,
+    "eval_p_rank/avg_succ_subopt_diff_sum_villekuosmanen_armnetbench_robometer_v01_so101": 0.26076613664627074,
+    "eval_p_rank/min_succ_subopt_diff_sum_villekuosmanen_armnetbench_robometer_v01_so101": 0.2388161063194275,
+    "eval_p_rank/max_succ_subopt_diff_sum_villekuosmanen_armnetbench_robometer_v01_so101": 0.28271616697311397,
+    "eval_p_rank/avg_subopt_fail_diff_sum_villekuosmanen_armnetbench_robometer_v01_so101": 0.25212376415729526,
+    "eval_p_rank/min_subopt_fail_diff_sum_villekuosmanen_armnetbench_robometer_v01_so101": 0.147011947631836,
+    "eval_p_rank/max_subopt_fail_diff_sum_villekuosmanen_armnetbench_robometer_v01_so101": 0.3572355806827545,
+    "eval_p_rank/avg_succ_fail_diff_sum_villekuosmanen_armnetbench_robometer_v01_so101": 0.5191158263012767,
+    "eval_p_rank/min_succ_fail_diff_sum_villekuosmanen_armnetbench_robometer_v01_so101": 0.25460606813430786,
+    "eval_p_rank/max_succ_fail_diff_sum_villekuosmanen_armnetbench_robometer_v01_so101": 0.6971883609890939,
+    "eval_p_rank/ranking_acc_sum_villekuosmanen_armnetbench_robometer_v01_so101": 0.9733333333333334,
+    "eval_p_rank/ranking_acc_all_pairs_sum_villekuosmanen_armnetbench_robometer_v01_so101": 0.9733333333333334,
+    "eval_p_rank/ranking_acc_failure_vs_successful_sum_villekuosmanen_armnetbench_robometer_v01_so101": 0.975,
+    "eval_p_rank/ranking_acc_suboptimal_vs_successful_sum_villekuosmanen_armnetbench_robometer_v01_so101": 0.98,
+    "eval_p_rank/ranking_acc_failure_vs_suboptimal_sum_villekuosmanen_armnetbench_robometer_v01_so101": 0.96,
+    "eval_rew_align/success_auprc_ykorkmaz_molmoact2_so100_101_rbm_molmoact2_so100_101": 0.3786145132198085,
+    "eval_rew_align/positive_success_acc_ykorkmaz_molmoact2_so100_101_rbm_molmoact2_so100_101": 0.9,
+    "eval_rew_align/negative_success_acc_ykorkmaz_molmoact2_so100_101_rbm_molmoact2_so100_101": 0.9583333333333334,
+    "eval_rew_align/loss_ykorkmaz_molmoact2_so100_101_rbm_molmoact2_so100_101": 1.2838820934295654,
+    "eval_rew_align/pearson_ykorkmaz_molmoact2_so100_101_rbm_molmoact2_so100_101": 0.9019924825489823,
+    "time/custom_evaluations": 41.701750837266445
+  }
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d5e3cb513d526e4da34041619a5967dab1e94523375d5430b9c370717bbcc82a
+size 8894099488