Add files using upload-large-folder tool
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- healed/grid_math/reap_keep25_s1225/args.json +62 -0
- healed/grid_math/reap_keep25_s1225/train_log.jsonl +150 -0
- healed/grid_math/uniform_keep25_s1226/train_log.jsonl +150 -0
- healed/grid_math/uniform_keep75_s1226/args.json +62 -0
- healed/grid_math/uniform_keep75_s1226/train_log.jsonl +150 -0
- healed/liger_mb8/step0004/chat_template.jinja +9 -0
- healed/liger_mb8/step0004/config.json +887 -0
- healed/liger_mb8/step0004/configuration_pruned_olmoe.py +27 -0
- healed/liger_mb8/step0004/generation_config.json +6 -0
- healed/liger_mb8/step0004/model.safetensors.index.json +0 -0
- healed/liger_mb8/step0004/modeling_pruned_olmoe.py +66 -0
- healed/liger_mb8/step0004/special_tokens_map.json +23 -0
- healed/liger_mb8/step0004/tokenizer.json +0 -0
- healed/liger_mb8/step0004/tokenizer_config.json +247 -0
- healed/liger_smoke/step0006/chat_template.jinja +9 -0
- healed/liger_smoke/step0006/config.json +887 -0
- healed/liger_smoke/step0006/configuration_pruned_olmoe.py +27 -0
- healed/liger_smoke/step0006/generation_config.json +6 -0
- healed/liger_smoke/step0006/model.safetensors.index.json +0 -0
- healed/liger_smoke/step0006/modeling_pruned_olmoe.py +66 -0
- healed/liger_smoke/step0006/special_tokens_map.json +23 -0
- healed/liger_smoke/step0006/tokenizer.json +0 -0
- healed/liger_smoke/step0006/tokenizer_config.json +247 -0
- healed/mixonly_keep50/vllm_live/chat_template.jinja +9 -0
- healed/mixonly_keep50/vllm_live/config.json +887 -0
- healed/mixonly_keep50/vllm_live/configuration_pruned_olmoe.py +27 -0
- healed/mixonly_keep50/vllm_live/generation_config.json +6 -0
- healed/mixonly_keep50/vllm_live/model.safetensors.index.json +0 -0
- healed/mixonly_keep50/vllm_live/modeling_pruned_olmoe.py +66 -0
- healed/mixonly_keep50/vllm_live/special_tokens_map.json +23 -0
- healed/mixonly_keep50/vllm_live/tokenizer.json +0 -0
- healed/mixonly_keep50/vllm_live/tokenizer_config.json +247 -0
- healed/mixonly_keep50/wandb/debug-internal.log +11 -0
- healed/mixonly_keep50/wandb/debug.log +19 -0
- healed/mixonly_keep50/wandb/offline-run-20260731_011513-gexxktae/files/requirements.txt +130 -0
- healed/mixonly_keep50/wandb/offline-run-20260731_011513-gexxktae/logs/debug-core.log +6 -0
- healed/mixonly_keep50/wandb/offline-run-20260731_011513-gexxktae/logs/debug-internal.log +11 -0
- healed/mixonly_keep50/wandb/offline-run-20260731_011513-gexxktae/logs/debug.log +19 -0
- healed/opd_warm_fixed_keep50/step0120/chat_template.jinja +9 -0
- healed/opd_warm_fixed_keep50/step0120/config.json +887 -0
- healed/opd_warm_fixed_keep50/step0120/configuration_pruned_olmoe.py +27 -0
- healed/opd_warm_fixed_keep50/step0120/generation_config.json +6 -0
- healed/opd_warm_fixed_keep50/step0120/model.safetensors.index.json +0 -0
- healed/opd_warm_fixed_keep50/step0120/modeling_pruned_olmoe.py +66 -0
- healed/opd_warm_fixed_keep50/step0120/special_tokens_map.json +23 -0
- healed/opd_warm_fixed_keep50/step0120/tokenizer.json +0 -0
- healed/opd_warm_fixed_keep50/step0120/tokenizer_config.json +247 -0
- healed/opd_warm_fixed_keep50/step0200/chat_template.jinja +9 -0
- healed/opd_warm_fixed_keep50/step0200/config.json +887 -0
- healed/opd_warm_fixed_keep50/step0200/configuration_pruned_olmoe.py +27 -0
healed/grid_math/reap_keep25_s1225/args.json
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"student": "outputs/pruned/reap-0125inst-math-keep25",
|
| 3 |
+
"teacher": "allenai/OLMoE-1B-7B-0125-Instruct",
|
| 4 |
+
"training_mode": "off-policy",
|
| 5 |
+
"kl_direction": "forward",
|
| 6 |
+
"dataset": "allenai/RLVR-MATH",
|
| 7 |
+
"dataset_sources": null,
|
| 8 |
+
"max_difficulty": null,
|
| 9 |
+
"trajectories": "outputs/teacher_trajectories/dolci_math_curated.jsonl",
|
| 10 |
+
"trajectory_dataset": "allenai/Dolci-Instruct-RL",
|
| 11 |
+
"off_policy_frames": "chat",
|
| 12 |
+
"off_policy_max_seq_len": 2048,
|
| 13 |
+
"topk_targets": "outputs/teacher_trajectories/dolci_math_curated_opd_top128",
|
| 14 |
+
"max_loss_tokens": null,
|
| 15 |
+
"loss_tokens_per_step": 120000,
|
| 16 |
+
"teacher_device": "cuda:0",
|
| 17 |
+
"student_device": "cuda:0",
|
| 18 |
+
"lr": 3e-05,
|
| 19 |
+
"optimizer": "adamw8bit",
|
| 20 |
+
"weight_decay": 0.1,
|
| 21 |
+
"epochs": 3,
|
| 22 |
+
"prompts_per_step": 256,
|
| 23 |
+
"group_size": 1,
|
| 24 |
+
"rollout_batch": 64,
|
| 25 |
+
"micro_batch": 3,
|
| 26 |
+
"max_new_tokens": 256,
|
| 27 |
+
"max_prompt_len": 1024,
|
| 28 |
+
"warmup_steps": 10,
|
| 29 |
+
"max_grad_norm": 1.0,
|
| 30 |
+
"eval_every": 10,
|
| 31 |
+
"gsm8k_every": 0,
|
| 32 |
+
"gsm8k_n": 256,
|
| 33 |
+
"gsm8k_batch": 16,
|
| 34 |
+
"gsm8k_max_new_tokens": 512,
|
| 35 |
+
"gsm8k_frames": "chat",
|
| 36 |
+
"save_every": 50,
|
| 37 |
+
"out_dir": "outputs/healed/grid_math/reap_keep25_s1225",
|
| 38 |
+
"sweep": 150,
|
| 39 |
+
"wandb": true,
|
| 40 |
+
"wandb_project": "glean-grid",
|
| 41 |
+
"wandb_run_name": "reap-math-keep25-s1225",
|
| 42 |
+
"wandb_run_id": null,
|
| 43 |
+
"wandb_resume": null,
|
| 44 |
+
"wandb_mode": "online",
|
| 45 |
+
"no_wandb_sync": false,
|
| 46 |
+
"debug": false,
|
| 47 |
+
"resume_from": null,
|
| 48 |
+
"start_step": 0,
|
| 49 |
+
"no_grad_checkpointing": false,
|
| 50 |
+
"seed": 1225,
|
| 51 |
+
"no_teacher_overlap": false,
|
| 52 |
+
"sync_checkpoints": false,
|
| 53 |
+
"rollout_engine": "hf",
|
| 54 |
+
"vllm_gpu": null,
|
| 55 |
+
"vllm_port": 8377,
|
| 56 |
+
"vllm_refresh_every": 5,
|
| 57 |
+
"vllm_serve_bin": "vllm-plugin/.venv/bin/python",
|
| 58 |
+
"vllm_gpu_mem_util": 0.85,
|
| 59 |
+
"vllm_refresh_mode": "reload",
|
| 60 |
+
"vllm_live_dir": null,
|
| 61 |
+
"resolved_kl_direction": "forward"
|
| 62 |
+
}
|
healed/grid_math/reap_keep25_s1225/train_log.jsonl
ADDED
|
@@ -0,0 +1,150 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"step": 1, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 5.5052759854316715, "tokens": 120000, "cumulative_loss_tokens": 120000, "grad_norm": 91.0, "lr": 6e-06, "finish_rate": 0.733, "comp_len": 628.3, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.5, "frames": {"chat": 191}, "mem_gb": 9.93}
|
| 2 |
+
{"step": 2, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 5.34218070195516, "tokens": 120000, "cumulative_loss_tokens": 240000, "grad_norm": 72.5, "lr": 9e-06, "finish_rate": 0.845, "comp_len": 547.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 41.7, "frames": {"chat": 219}, "mem_gb": 9.99}
|
| 3 |
+
{"step": 3, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 5.2662361368576684, "tokens": 120000, "cumulative_loss_tokens": 360000, "grad_norm": 54.5, "lr": 1.2e-05, "finish_rate": 0.778, "comp_len": 579.7, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 41.4, "frames": {"chat": 207}, "mem_gb": 10.0}
|
| 4 |
+
{"step": 4, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 4.323533807410796, "tokens": 120000, "cumulative_loss_tokens": 480000, "grad_norm": 49.75, "lr": 1.5e-05, "finish_rate": 0.755, "comp_len": 576.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 42.4, "frames": {"chat": 208}, "mem_gb": 9.95}
|
| 5 |
+
{"step": 5, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 3.656937780322631, "tokens": 120000, "cumulative_loss_tokens": 600000, "grad_norm": 42.5, "lr": 1.8e-05, "finish_rate": 0.799, "comp_len": 547.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.2, "frames": {"chat": 219}, "mem_gb": 9.99}
|
| 6 |
+
{"step": 6, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 3.098056636095047, "tokens": 120000, "cumulative_loss_tokens": 720000, "grad_norm": 32.25, "lr": 2.1e-05, "finish_rate": 0.915, "comp_len": 487.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.9, "frames": {"chat": 246}, "mem_gb": 9.87}
|
| 7 |
+
{"step": 7, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 2.628579444358746, "tokens": 120000, "cumulative_loss_tokens": 840000, "grad_norm": 41.0, "lr": 2.4e-05, "finish_rate": 0.704, "comp_len": 582.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.1, "frames": {"chat": 206}, "mem_gb": 10.02}
|
| 8 |
+
{"step": 8, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 2.0468826131433246, "tokens": 120000, "cumulative_loss_tokens": 960000, "grad_norm": 26.75, "lr": 2.7000000000000002e-05, "finish_rate": 0.876, "comp_len": 515.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.1, "frames": {"chat": 233}, "mem_gb": 10.0}
|
| 9 |
+
{"step": 9, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 1.5576237986435493, "tokens": 120000, "cumulative_loss_tokens": 1080000, "grad_norm": 16.125, "lr": 3e-05, "finish_rate": 0.847, "comp_len": 524.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 42.7, "frames": {"chat": 229}, "mem_gb": 9.86}
|
| 10 |
+
{"step": 10, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 1.2283661879966656, "tokens": 120000, "cumulative_loss_tokens": 1200000, "grad_norm": 9.4375, "lr": 3e-05, "finish_rate": 0.864, "comp_len": 508.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.2, "frames": {"chat": 236}, "mem_gb": 9.9}
|
| 11 |
+
{"step": 11, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.9977921682407459, "tokens": 120000, "cumulative_loss_tokens": 1320000, "grad_norm": 6.375, "lr": 3e-05, "finish_rate": 0.87, "comp_len": 502.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.2, "frames": {"chat": 239}, "mem_gb": 9.78}
|
| 12 |
+
{"step": 12, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.8474529584680994, "tokens": 120000, "cumulative_loss_tokens": 1440000, "grad_norm": 6.25, "lr": 3e-05, "finish_rate": 0.867, "comp_len": 497.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.9, "frames": {"chat": 241}, "mem_gb": 9.9}
|
| 13 |
+
{"step": 13, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.7915342311960956, "tokens": 120000, "cumulative_loss_tokens": 1560000, "grad_norm": 15.375, "lr": 3e-05, "finish_rate": 0.863, "comp_len": 531.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.8, "frames": {"chat": 226}, "mem_gb": 9.87}
|
| 14 |
+
{"step": 14, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.6635135169697305, "tokens": 120000, "cumulative_loss_tokens": 1680000, "grad_norm": 3.984375, "lr": 3e-05, "finish_rate": 0.893, "comp_len": 512.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.7, "frames": {"chat": 234}, "mem_gb": 10.0}
|
| 15 |
+
{"step": 15, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.6129945642106235, "tokens": 120000, "cumulative_loss_tokens": 1800000, "grad_norm": 2.453125, "lr": 3e-05, "finish_rate": 0.914, "comp_len": 466.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.2, "frames": {"chat": 257}, "mem_gb": 9.99}
|
| 16 |
+
{"step": 16, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.714731409107397, "tokens": 120000, "cumulative_loss_tokens": 1920000, "grad_norm": 3.296875, "lr": 3e-05, "finish_rate": 0.76, "comp_len": 576.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.6, "frames": {"chat": 208}, "mem_gb": 10.04}
|
| 17 |
+
{"step": 17, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.5896475110622744, "tokens": 120000, "cumulative_loss_tokens": 2040000, "grad_norm": 5.53125, "lr": 3e-05, "finish_rate": 0.763, "comp_len": 568.7, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.6, "frames": {"chat": 211}, "mem_gb": 10.02}
|
| 18 |
+
{"step": 18, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.517651187770317, "tokens": 120000, "cumulative_loss_tokens": 2160000, "grad_norm": 1.390625, "lr": 3e-05, "finish_rate": 0.806, "comp_len": 528.6, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.0, "frames": {"chat": 227}, "mem_gb": 10.0}
|
| 19 |
+
{"step": 19, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.5277626443808278, "tokens": 120000, "cumulative_loss_tokens": 2280000, "grad_norm": 1.3984375, "lr": 3e-05, "finish_rate": 0.796, "comp_len": 568.7, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.0, "frames": {"chat": 211}, "mem_gb": 9.98}
|
| 20 |
+
{"step": 20, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.4388222487750153, "tokens": 120000, "cumulative_loss_tokens": 2400000, "grad_norm": 1.1875, "lr": 3e-05, "finish_rate": 0.861, "comp_len": 504.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.8, "frames": {"chat": 238}, "mem_gb": 9.99}
|
| 21 |
+
{"step": 21, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.42888357511336606, "tokens": 120000, "cumulative_loss_tokens": 2520000, "grad_norm": 1.046875, "lr": 3e-05, "finish_rate": 0.835, "comp_len": 506.3, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 46.7, "frames": {"chat": 237}, "mem_gb": 10.03}
|
| 22 |
+
{"step": 22, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.44955972477830947, "tokens": 120000, "cumulative_loss_tokens": 2640000, "grad_norm": 1.171875, "lr": 3e-05, "finish_rate": 0.721, "comp_len": 576.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.5, "frames": {"chat": 208}, "mem_gb": 10.03}
|
| 23 |
+
{"step": 23, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.40363404167294503, "tokens": 120000, "cumulative_loss_tokens": 2760000, "grad_norm": 0.9921875, "lr": 3e-05, "finish_rate": 0.801, "comp_len": 543.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 42.8, "frames": {"chat": 221}, "mem_gb": 10.12}
|
| 24 |
+
{"step": 24, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.36760696430206297, "tokens": 120000, "cumulative_loss_tokens": 2880000, "grad_norm": 0.8828125, "lr": 3e-05, "finish_rate": 0.853, "comp_len": 517.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.4, "frames": {"chat": 232}, "mem_gb": 9.96}
|
| 25 |
+
{"step": 25, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.3675604015878091, "tokens": 120000, "cumulative_loss_tokens": 3000000, "grad_norm": 0.8515625, "lr": 3e-05, "finish_rate": 0.764, "comp_len": 576.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.1, "frames": {"chat": 208}, "mem_gb": 9.99}
|
| 26 |
+
{"step": 26, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.3380553434039156, "tokens": 120000, "cumulative_loss_tokens": 3120000, "grad_norm": 0.7734375, "lr": 3e-05, "finish_rate": 0.837, "comp_len": 528.6, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.2, "frames": {"chat": 227}, "mem_gb": 9.91}
|
| 27 |
+
{"step": 27, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.3542323801631729, "tokens": 120000, "cumulative_loss_tokens": 3240000, "grad_norm": 0.7578125, "lr": 3e-05, "finish_rate": 0.824, "comp_len": 543.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 42.9, "frames": {"chat": 221}, "mem_gb": 9.94}
|
| 28 |
+
{"step": 28, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.3140212787228326, "tokens": 120000, "cumulative_loss_tokens": 3360000, "grad_norm": 0.7421875, "lr": 3e-05, "finish_rate": 0.815, "comp_len": 517.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.5, "frames": {"chat": 232}, "mem_gb": 10.0}
|
| 29 |
+
{"step": 29, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.2983292136088014, "tokens": 120000, "cumulative_loss_tokens": 3480000, "grad_norm": 0.71875, "lr": 3e-05, "finish_rate": 0.822, "comp_len": 547.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 42.8, "frames": {"chat": 219}, "mem_gb": 10.0}
|
| 30 |
+
{"step": 30, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.3193080648737649, "tokens": 120000, "cumulative_loss_tokens": 3600000, "grad_norm": 0.70703125, "lr": 3e-05, "finish_rate": 0.713, "comp_len": 615.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 41.8, "frames": {"chat": 195}, "mem_gb": 10.09}
|
| 31 |
+
{"step": 31, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.31196173311385017, "tokens": 120000, "cumulative_loss_tokens": 3720000, "grad_norm": 0.80859375, "lr": 3e-05, "finish_rate": 0.833, "comp_len": 555.6, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 41.9, "frames": {"chat": 216}, "mem_gb": 10.0}
|
| 32 |
+
{"step": 32, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.2535243794289728, "tokens": 120000, "cumulative_loss_tokens": 3840000, "grad_norm": 0.60546875, "lr": 3e-05, "finish_rate": 0.788, "comp_len": 576.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 41.9, "frames": {"chat": 208}, "mem_gb": 9.88}
|
| 33 |
+
{"step": 33, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.263915772453323, "tokens": 120000, "cumulative_loss_tokens": 3960000, "grad_norm": 0.70703125, "lr": 3e-05, "finish_rate": 0.919, "comp_len": 510.6, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 42.9, "frames": {"chat": 235}, "mem_gb": 9.88}
|
| 34 |
+
{"step": 34, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.27100975296981633, "tokens": 120000, "cumulative_loss_tokens": 4080000, "grad_norm": 0.7265625, "lr": 3e-05, "finish_rate": 0.853, "comp_len": 533.3, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 41.9, "frames": {"chat": 225}, "mem_gb": 9.99}
|
| 35 |
+
{"step": 35, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.34947378008762997, "tokens": 120000, "cumulative_loss_tokens": 4200000, "grad_norm": 0.76953125, "lr": 3e-05, "finish_rate": 0.77, "comp_len": 563.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.2, "frames": {"chat": 213}, "mem_gb": 10.08}
|
| 36 |
+
{"step": 36, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.2621212104951342, "tokens": 120000, "cumulative_loss_tokens": 4320000, "grad_norm": 0.71875, "lr": 3e-05, "finish_rate": 0.922, "comp_len": 466.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 46.1, "frames": {"chat": 257}, "mem_gb": 9.75}
|
| 37 |
+
{"step": 37, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.2858927722416818, "tokens": 120000, "cumulative_loss_tokens": 4440000, "grad_norm": 0.6640625, "lr": 3e-05, "finish_rate": 0.792, "comp_len": 566.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.6, "frames": {"chat": 212}, "mem_gb": 10.02}
|
| 38 |
+
{"step": 38, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.24749040407985448, "tokens": 120000, "cumulative_loss_tokens": 4560000, "grad_norm": 0.55859375, "lr": 3e-05, "finish_rate": 0.833, "comp_len": 543.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 42.5, "frames": {"chat": 221}, "mem_gb": 10.0}
|
| 39 |
+
{"step": 39, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.24993702379036695, "tokens": 120000, "cumulative_loss_tokens": 4680000, "grad_norm": 0.6328125, "lr": 3e-05, "finish_rate": 0.868, "comp_len": 495.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.7, "frames": {"chat": 242}, "mem_gb": 9.99}
|
| 40 |
+
{"step": 40, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.22095589282835523, "tokens": 120000, "cumulative_loss_tokens": 4800000, "grad_norm": 0.5390625, "lr": 3e-05, "finish_rate": 0.836, "comp_len": 545.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.1, "frames": {"chat": 220}, "mem_gb": 9.96}
|
| 41 |
+
{"step": 41, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.24182025297402093, "tokens": 120000, "cumulative_loss_tokens": 4920000, "grad_norm": 0.62890625, "lr": 3e-05, "finish_rate": 0.896, "comp_len": 500.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.0, "frames": {"chat": 240}, "mem_gb": 9.85}
|
| 42 |
+
{"step": 42, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.24479971672501416, "tokens": 120000, "cumulative_loss_tokens": 5040000, "grad_norm": 0.60546875, "lr": 3e-05, "finish_rate": 0.728, "comp_len": 582.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 41.6, "frames": {"chat": 206}, "mem_gb": 9.98}
|
| 43 |
+
{"step": 43, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.2458774261167273, "tokens": 120000, "cumulative_loss_tokens": 5160000, "grad_norm": 0.640625, "lr": 3e-05, "finish_rate": 0.867, "comp_len": 531.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.1, "frames": {"chat": 226}, "mem_gb": 10.0}
|
| 44 |
+
{"step": 44, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.31763079155124724, "tokens": 120000, "cumulative_loss_tokens": 5280000, "grad_norm": 1.1640625, "lr": 3e-05, "finish_rate": 0.877, "comp_len": 491.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.2, "frames": {"chat": 244}, "mem_gb": 9.78}
|
| 45 |
+
{"step": 45, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.26981097101569174, "tokens": 120000, "cumulative_loss_tokens": 5400000, "grad_norm": 0.6015625, "lr": 3e-05, "finish_rate": 0.804, "comp_len": 535.7, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.6, "frames": {"chat": 224}, "mem_gb": 10.0}
|
| 46 |
+
{"step": 46, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.2382314425634841, "tokens": 120000, "cumulative_loss_tokens": 5520000, "grad_norm": 0.625, "lr": 3e-05, "finish_rate": 0.923, "comp_len": 442.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 47.1, "frames": {"chat": 271}, "mem_gb": 9.72}
|
| 47 |
+
{"step": 47, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.21951183261151114, "tokens": 120000, "cumulative_loss_tokens": 5640000, "grad_norm": 0.546875, "lr": 3e-05, "finish_rate": 0.856, "comp_len": 508.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.6, "frames": {"chat": 236}, "mem_gb": 10.01}
|
| 48 |
+
{"step": 48, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.24863433394009868, "tokens": 120000, "cumulative_loss_tokens": 5760000, "grad_norm": 0.63671875, "lr": 3e-05, "finish_rate": 0.841, "comp_len": 517.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.5, "frames": {"chat": 232}, "mem_gb": 9.88}
|
| 49 |
+
{"step": 49, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.19559897173379237, "tokens": 120000, "cumulative_loss_tokens": 5880000, "grad_norm": 0.50390625, "lr": 3e-05, "finish_rate": 0.79, "comp_len": 571.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 42.5, "frames": {"chat": 210}, "mem_gb": 9.93}
|
| 50 |
+
{"step": 50, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.18844997002581756, "tokens": 120000, "cumulative_loss_tokens": 6000000, "grad_norm": 0.470703125, "lr": 3e-05, "finish_rate": 0.811, "comp_len": 553.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 42.2, "frames": {"chat": 217}, "mem_gb": 9.9}
|
| 51 |
+
{"step": 51, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.25687755517810584, "tokens": 120000, "cumulative_loss_tokens": 6120000, "grad_norm": 0.58203125, "lr": 3e-05, "finish_rate": 0.839, "comp_len": 535.7, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.9, "frames": {"chat": 224}, "mem_gb": 10.02}
|
| 52 |
+
{"step": 52, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.28079532670788465, "tokens": 120000, "cumulative_loss_tokens": 6240000, "grad_norm": 0.61328125, "lr": 3e-05, "finish_rate": 0.749, "comp_len": 591.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.1, "frames": {"chat": 203}, "mem_gb": 9.87}
|
| 53 |
+
{"step": 53, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.2312026971814533, "tokens": 120000, "cumulative_loss_tokens": 6360000, "grad_norm": 0.5546875, "lr": 3e-05, "finish_rate": 0.887, "comp_len": 502.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.4, "frames": {"chat": 239}, "mem_gb": 9.97}
|
| 54 |
+
{"step": 54, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.19427155094649642, "tokens": 120000, "cumulative_loss_tokens": 6480000, "grad_norm": 0.5, "lr": 3e-05, "finish_rate": 0.902, "comp_len": 472.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.4, "frames": {"chat": 254}, "mem_gb": 9.88}
|
| 55 |
+
{"step": 55, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.18302697168818363, "tokens": 120000, "cumulative_loss_tokens": 6600000, "grad_norm": 0.474609375, "lr": 3e-05, "finish_rate": 0.876, "comp_len": 497.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.5, "frames": {"chat": 241}, "mem_gb": 9.97}
|
| 56 |
+
{"step": 56, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.24095728847576925, "tokens": 120000, "cumulative_loss_tokens": 6720000, "grad_norm": 0.51953125, "lr": 3e-05, "finish_rate": 0.746, "comp_len": 563.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.0, "frames": {"chat": 213}, "mem_gb": 10.0}
|
| 57 |
+
{"step": 57, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.2061793549572428, "tokens": 120000, "cumulative_loss_tokens": 6840000, "grad_norm": 0.74609375, "lr": 3e-05, "finish_rate": 0.864, "comp_len": 543.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.2, "frames": {"chat": 221}, "mem_gb": 10.05}
|
| 58 |
+
{"step": 58, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.20495547246101, "tokens": 120000, "cumulative_loss_tokens": 6960000, "grad_norm": 0.48046875, "lr": 3e-05, "finish_rate": 0.745, "comp_len": 612.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 41.3, "frames": {"chat": 196}, "mem_gb": 10.01}
|
| 59 |
+
{"step": 59, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.17600186436952403, "tokens": 120000, "cumulative_loss_tokens": 7080000, "grad_norm": 0.498046875, "lr": 3e-05, "finish_rate": 0.926, "comp_len": 444.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 48.0, "frames": {"chat": 270}, "mem_gb": 9.81}
|
| 60 |
+
{"step": 60, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.15102526473104955, "tokens": 120000, "cumulative_loss_tokens": 7200000, "grad_norm": 0.4140625, "lr": 3e-05, "finish_rate": 0.815, "comp_len": 555.6, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 42.6, "frames": {"chat": 216}, "mem_gb": 9.99}
|
| 61 |
+
{"step": 61, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.21443645971684405, "tokens": 120000, "cumulative_loss_tokens": 7320000, "grad_norm": 0.57421875, "lr": 3e-05, "finish_rate": 0.775, "comp_len": 600.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 40.6, "frames": {"chat": 200}, "mem_gb": 9.96}
|
| 62 |
+
{"step": 62, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.1430999824684734, "tokens": 120000, "cumulative_loss_tokens": 7440000, "grad_norm": 0.419921875, "lr": 3e-05, "finish_rate": 0.767, "comp_len": 582.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 41.9, "frames": {"chat": 206}, "mem_gb": 9.91}
|
| 63 |
+
{"step": 63, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.1475067599070569, "tokens": 120000, "cumulative_loss_tokens": 7560000, "grad_norm": 0.40234375, "lr": 3e-05, "finish_rate": 0.902, "comp_len": 512.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 42.9, "frames": {"chat": 234}, "mem_gb": 9.94}
|
| 64 |
+
{"step": 64, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.17827725302210698, "tokens": 120000, "cumulative_loss_tokens": 7680000, "grad_norm": 0.439453125, "lr": 3e-05, "finish_rate": 0.823, "comp_len": 558.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 41.3, "frames": {"chat": 215}, "mem_gb": 9.95}
|
| 65 |
+
{"step": 65, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.15364940941271682, "tokens": 120000, "cumulative_loss_tokens": 7800000, "grad_norm": 0.494140625, "lr": 3e-05, "finish_rate": 0.922, "comp_len": 470.6, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.8, "frames": {"chat": 255}, "mem_gb": 9.94}
|
| 66 |
+
{"step": 66, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.18826355347931384, "tokens": 120000, "cumulative_loss_tokens": 7920000, "grad_norm": 0.546875, "lr": 3e-05, "finish_rate": 0.892, "comp_len": 480.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.2, "frames": {"chat": 250}, "mem_gb": 9.82}
|
| 67 |
+
{"step": 67, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.17189633793290704, "tokens": 120000, "cumulative_loss_tokens": 8040000, "grad_norm": 0.4375, "lr": 3e-05, "finish_rate": 0.884, "comp_len": 495.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.5, "frames": {"chat": 242}, "mem_gb": 9.99}
|
| 68 |
+
{"step": 68, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.2132726859041179, "tokens": 120000, "cumulative_loss_tokens": 8160000, "grad_norm": 0.5234375, "lr": 3e-05, "finish_rate": 0.729, "comp_len": 603.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 42.8, "frames": {"chat": 199}, "mem_gb": 10.0}
|
| 69 |
+
{"step": 69, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.24157150672028463, "tokens": 120000, "cumulative_loss_tokens": 8280000, "grad_norm": 0.578125, "lr": 3e-05, "finish_rate": 0.784, "comp_len": 576.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.0, "frames": {"chat": 208}, "mem_gb": 10.03}
|
| 70 |
+
{"step": 70, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.18809422614208113, "tokens": 120000, "cumulative_loss_tokens": 8400000, "grad_norm": 0.515625, "lr": 3e-05, "finish_rate": 0.764, "comp_len": 576.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 41.3, "frames": {"chat": 208}, "mem_gb": 9.97}
|
| 71 |
+
{"step": 71, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.19989227577894927, "tokens": 120000, "cumulative_loss_tokens": 8520000, "grad_norm": 0.4765625, "lr": 3e-05, "finish_rate": 0.732, "comp_len": 574.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.1, "frames": {"chat": 209}, "mem_gb": 10.12}
|
| 72 |
+
{"step": 72, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.1725833379857863, "tokens": 120000, "cumulative_loss_tokens": 8640000, "grad_norm": 0.484375, "lr": 3e-05, "finish_rate": 0.855, "comp_len": 510.6, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.9, "frames": {"chat": 235}, "mem_gb": 9.95}
|
| 73 |
+
{"step": 73, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.1478453546665609, "tokens": 120000, "cumulative_loss_tokens": 8760000, "grad_norm": 0.41015625, "lr": 3e-05, "finish_rate": 0.74, "comp_len": 588.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 41.8, "frames": {"chat": 204}, "mem_gb": 9.94}
|
| 74 |
+
{"step": 74, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.21500386069975794, "tokens": 120000, "cumulative_loss_tokens": 8880000, "grad_norm": 0.5, "lr": 3e-05, "finish_rate": 0.745, "comp_len": 576.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.1, "frames": {"chat": 208}, "mem_gb": 10.0}
|
| 75 |
+
{"step": 75, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.1554392906052371, "tokens": 120000, "cumulative_loss_tokens": 9000000, "grad_norm": 0.44140625, "lr": 3e-05, "finish_rate": 0.825, "comp_len": 500.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.9, "frames": {"chat": 240}, "mem_gb": 10.0}
|
| 76 |
+
{"step": 76, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.1832204219336311, "tokens": 120000, "cumulative_loss_tokens": 9120000, "grad_norm": 0.50390625, "lr": 3e-05, "finish_rate": 0.89, "comp_len": 487.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.4, "frames": {"chat": 246}, "mem_gb": 9.99}
|
| 77 |
+
{"step": 77, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.17209678962721178, "tokens": 120000, "cumulative_loss_tokens": 9240000, "grad_norm": 0.4453125, "lr": 3e-05, "finish_rate": 0.909, "comp_len": 493.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.1, "frames": {"chat": 243}, "mem_gb": 9.81}
|
| 78 |
+
{"step": 78, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.20465335384116817, "tokens": 120000, "cumulative_loss_tokens": 9360000, "grad_norm": 0.5078125, "lr": 3e-05, "finish_rate": 0.745, "comp_len": 576.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 41.8, "frames": {"chat": 208}, "mem_gb": 10.01}
|
| 79 |
+
{"step": 79, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.1694449459930261, "tokens": 120000, "cumulative_loss_tokens": 9480000, "grad_norm": 0.3984375, "lr": 3e-05, "finish_rate": 0.817, "comp_len": 547.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 42.7, "frames": {"chat": 219}, "mem_gb": 10.0}
|
| 80 |
+
{"step": 80, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.22224440845083446, "tokens": 120000, "cumulative_loss_tokens": 9600000, "grad_norm": 0.5234375, "lr": 3e-05, "finish_rate": 0.782, "comp_len": 568.7, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.6, "frames": {"chat": 211}, "mem_gb": 10.01}
|
| 81 |
+
{"step": 81, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.1509838725623054, "tokens": 120000, "cumulative_loss_tokens": 9720000, "grad_norm": 0.41015625, "lr": 3e-05, "finish_rate": 0.862, "comp_len": 517.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.9, "frames": {"chat": 232}, "mem_gb": 9.97}
|
| 82 |
+
{"step": 82, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.17593013638195892, "tokens": 120000, "cumulative_loss_tokens": 9840000, "grad_norm": 0.47265625, "lr": 3e-05, "finish_rate": 0.804, "comp_len": 560.7, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 42.9, "frames": {"chat": 214}, "mem_gb": 10.0}
|
| 83 |
+
{"step": 83, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.15619046537938217, "tokens": 120000, "cumulative_loss_tokens": 9960000, "grad_norm": 0.39453125, "lr": 3e-05, "finish_rate": 0.819, "comp_len": 531.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.5, "frames": {"chat": 226}, "mem_gb": 9.89}
|
| 84 |
+
{"step": 84, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.17479738595858216, "tokens": 120000, "cumulative_loss_tokens": 10080000, "grad_norm": 0.4609375, "lr": 3e-05, "finish_rate": 0.819, "comp_len": 571.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.1, "frames": {"chat": 210}, "mem_gb": 10.01}
|
| 85 |
+
{"step": 85, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.1404316657436701, "tokens": 120000, "cumulative_loss_tokens": 10200000, "grad_norm": 0.404296875, "lr": 3e-05, "finish_rate": 0.858, "comp_len": 550.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 42.9, "frames": {"chat": 218}, "mem_gb": 9.83}
|
| 86 |
+
{"step": 86, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.1403924273949427, "tokens": 120000, "cumulative_loss_tokens": 10320000, "grad_norm": 0.3828125, "lr": 3e-05, "finish_rate": 0.858, "comp_len": 515.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.0, "frames": {"chat": 233}, "mem_gb": 9.98}
|
| 87 |
+
{"step": 87, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.17982150458991528, "tokens": 120000, "cumulative_loss_tokens": 10440000, "grad_norm": 0.427734375, "lr": 3e-05, "finish_rate": 0.786, "comp_len": 558.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.8, "frames": {"chat": 215}, "mem_gb": 10.0}
|
| 88 |
+
{"step": 88, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.18029257206898183, "tokens": 120000, "cumulative_loss_tokens": 10560000, "grad_norm": 0.462890625, "lr": 3e-05, "finish_rate": 0.845, "comp_len": 515.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.0, "frames": {"chat": 233}, "mem_gb": 9.99}
|
| 89 |
+
{"step": 89, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.14494490633426854, "tokens": 120000, "cumulative_loss_tokens": 10680000, "grad_norm": 0.380859375, "lr": 3e-05, "finish_rate": 0.766, "comp_len": 574.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 42.6, "frames": {"chat": 209}, "mem_gb": 9.94}
|
| 90 |
+
{"step": 90, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.15547993779405953, "tokens": 120000, "cumulative_loss_tokens": 10800000, "grad_norm": 0.4453125, "lr": 3e-05, "finish_rate": 0.908, "comp_len": 458.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 46.3, "frames": {"chat": 262}, "mem_gb": 9.87}
|
| 91 |
+
{"step": 91, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.13846087997810294, "tokens": 120000, "cumulative_loss_tokens": 10920000, "grad_norm": 0.384765625, "lr": 3e-05, "finish_rate": 0.9, "comp_len": 481.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.2, "frames": {"chat": 249}, "mem_gb": 9.96}
|
| 92 |
+
{"step": 92, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.20217994603576758, "tokens": 120000, "cumulative_loss_tokens": 11040000, "grad_norm": 0.4765625, "lr": 3e-05, "finish_rate": 0.819, "comp_len": 528.6, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.6, "frames": {"chat": 227}, "mem_gb": 9.99}
|
| 93 |
+
{"step": 93, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.14583957826520005, "tokens": 120000, "cumulative_loss_tokens": 11160000, "grad_norm": 0.388671875, "lr": 3e-05, "finish_rate": 0.814, "comp_len": 543.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.3, "frames": {"chat": 221}, "mem_gb": 9.99}
|
| 94 |
+
{"step": 94, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.1660599309977765, "tokens": 120000, "cumulative_loss_tokens": 11280000, "grad_norm": 0.4453125, "lr": 3e-05, "finish_rate": 0.859, "comp_len": 512.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.1, "frames": {"chat": 234}, "mem_gb": 10.01}
|
| 95 |
+
{"step": 95, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.12514705213187263, "tokens": 120000, "cumulative_loss_tokens": 11400000, "grad_norm": 0.365234375, "lr": 3e-05, "finish_rate": 0.817, "comp_len": 563.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 41.6, "frames": {"chat": 213}, "mem_gb": 9.95}
|
| 96 |
+
{"step": 96, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.12465972695561747, "tokens": 120000, "cumulative_loss_tokens": 11520000, "grad_norm": 0.349609375, "lr": 3e-05, "finish_rate": 0.836, "comp_len": 563.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 41.9, "frames": {"chat": 213}, "mem_gb": 9.89}
|
| 97 |
+
{"step": 97, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.15661951549227038, "tokens": 120000, "cumulative_loss_tokens": 11640000, "grad_norm": 0.443359375, "lr": 3e-05, "finish_rate": 0.906, "comp_len": 512.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.8, "frames": {"chat": 234}, "mem_gb": 9.92}
|
| 98 |
+
{"step": 98, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.14978609310248867, "tokens": 120000, "cumulative_loss_tokens": 11760000, "grad_norm": 0.416015625, "lr": 3e-05, "finish_rate": 0.793, "comp_len": 540.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.3, "frames": {"chat": 222}, "mem_gb": 9.99}
|
| 99 |
+
{"step": 99, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.16640961758115638, "tokens": 120000, "cumulative_loss_tokens": 11880000, "grad_norm": 0.419921875, "lr": 3e-05, "finish_rate": 0.806, "comp_len": 528.6, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 48.3, "frames": {"chat": 227}, "mem_gb": 10.0}
|
| 100 |
+
{"step": 100, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.16042816521745795, "tokens": 120000, "cumulative_loss_tokens": 12000000, "grad_norm": 0.427734375, "lr": 3e-05, "finish_rate": 0.835, "comp_len": 550.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.1, "frames": {"chat": 218}, "mem_gb": 10.04}
|
| 101 |
+
{"step": 101, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.17738757199452568, "tokens": 120000, "cumulative_loss_tokens": 12120000, "grad_norm": 0.44921875, "lr": 3e-05, "finish_rate": 0.798, "comp_len": 538.1, "t_data_s": 0.1, "t_rollout_s": 0.0, "t_step_s": 43.9, "frames": {"chat": 223}, "mem_gb": 10.01}
|
| 102 |
+
{"step": 102, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.18987772140322873, "tokens": 120000, "cumulative_loss_tokens": 12240000, "grad_norm": 0.45703125, "lr": 3e-05, "finish_rate": 0.772, "comp_len": 582.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 42.6, "frames": {"chat": 206}, "mem_gb": 10.0}
|
| 103 |
+
{"step": 103, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.15001419565274068, "tokens": 120000, "cumulative_loss_tokens": 12360000, "grad_norm": 0.41015625, "lr": 3e-05, "finish_rate": 0.784, "comp_len": 563.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 42.8, "frames": {"chat": 213}, "mem_gb": 9.92}
|
| 104 |
+
{"step": 104, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.19792243093705425, "tokens": 120000, "cumulative_loss_tokens": 12480000, "grad_norm": 0.451171875, "lr": 3e-05, "finish_rate": 0.843, "comp_len": 538.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 46.1, "frames": {"chat": 223}, "mem_gb": 9.86}
|
| 105 |
+
{"step": 105, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.15784500931380316, "tokens": 120000, "cumulative_loss_tokens": 12600000, "grad_norm": 0.3984375, "lr": 3e-05, "finish_rate": 0.828, "comp_len": 528.6, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.3, "frames": {"chat": 227}, "mem_gb": 9.97}
|
| 106 |
+
{"step": 106, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.15715095629387846, "tokens": 120000, "cumulative_loss_tokens": 12720000, "grad_norm": 0.4375, "lr": 3e-05, "finish_rate": 0.889, "comp_len": 474.3, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.9, "frames": {"chat": 253}, "mem_gb": 10.0}
|
| 107 |
+
{"step": 107, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.16428898396513736, "tokens": 120000, "cumulative_loss_tokens": 12840000, "grad_norm": 0.48828125, "lr": 3e-05, "finish_rate": 0.792, "comp_len": 555.6, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 42.4, "frames": {"chat": 216}, "mem_gb": 10.0}
|
| 108 |
+
{"step": 108, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.13419136733350653, "tokens": 120000, "cumulative_loss_tokens": 12960000, "grad_norm": 0.39453125, "lr": 3e-05, "finish_rate": 0.766, "comp_len": 585.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 42.2, "frames": {"chat": 205}, "mem_gb": 9.97}
|
| 109 |
+
{"step": 109, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.17433834038786591, "tokens": 120000, "cumulative_loss_tokens": 13080000, "grad_norm": 0.41796875, "lr": 3e-05, "finish_rate": 0.729, "comp_len": 579.7, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 46.7, "frames": {"chat": 207}, "mem_gb": 10.06}
|
| 110 |
+
{"step": 110, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.14855496705075105, "tokens": 120000, "cumulative_loss_tokens": 13200000, "grad_norm": 0.388671875, "lr": 3e-05, "finish_rate": 0.814, "comp_len": 558.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.6, "frames": {"chat": 215}, "mem_gb": 9.98}
|
| 111 |
+
{"step": 111, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.11281266368478536, "tokens": 120000, "cumulative_loss_tokens": 13320000, "grad_norm": 0.43359375, "lr": 3e-05, "finish_rate": 0.86, "comp_len": 526.3, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.7, "frames": {"chat": 228}, "mem_gb": 10.0}
|
| 112 |
+
{"step": 112, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.14154021223733823, "tokens": 120000, "cumulative_loss_tokens": 13440000, "grad_norm": 0.4453125, "lr": 3e-05, "finish_rate": 0.747, "comp_len": 543.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.8, "frames": {"chat": 221}, "mem_gb": 10.04}
|
| 113 |
+
{"step": 113, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.11087060506536314, "tokens": 120000, "cumulative_loss_tokens": 13560000, "grad_norm": 0.36328125, "lr": 3e-05, "finish_rate": 0.882, "comp_len": 472.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.2, "frames": {"chat": 254}, "mem_gb": 9.84}
|
| 114 |
+
{"step": 114, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.09634949580542743, "tokens": 120000, "cumulative_loss_tokens": 13680000, "grad_norm": 0.37109375, "lr": 3e-05, "finish_rate": 0.843, "comp_len": 571.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 42.5, "frames": {"chat": 210}, "mem_gb": 9.96}
|
| 115 |
+
{"step": 115, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.11305949703895797, "tokens": 120000, "cumulative_loss_tokens": 13800000, "grad_norm": 0.361328125, "lr": 3e-05, "finish_rate": 0.827, "comp_len": 531.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.1, "frames": {"chat": 226}, "mem_gb": 9.92}
|
| 116 |
+
{"step": 116, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.11892387241683901, "tokens": 120000, "cumulative_loss_tokens": 13920000, "grad_norm": 0.359375, "lr": 3e-05, "finish_rate": 0.802, "comp_len": 566.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 42.9, "frames": {"chat": 212}, "mem_gb": 9.99}
|
| 117 |
+
{"step": 117, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.14203277846990775, "tokens": 120000, "cumulative_loss_tokens": 14040000, "grad_norm": 0.388671875, "lr": 3e-05, "finish_rate": 0.754, "comp_len": 568.7, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.0, "frames": {"chat": 211}, "mem_gb": 9.92}
|
| 118 |
+
{"step": 118, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.1415941094346655, "tokens": 120000, "cumulative_loss_tokens": 14160000, "grad_norm": 0.3984375, "lr": 3e-05, "finish_rate": 0.776, "comp_len": 612.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 40.4, "frames": {"chat": 196}, "mem_gb": 9.97}
|
| 119 |
+
{"step": 119, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.10169251247107362, "tokens": 120000, "cumulative_loss_tokens": 14280000, "grad_norm": 0.33203125, "lr": 3e-05, "finish_rate": 0.811, "comp_len": 566.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 42.5, "frames": {"chat": 212}, "mem_gb": 9.99}
|
| 120 |
+
{"step": 120, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.10335419424610834, "tokens": 120000, "cumulative_loss_tokens": 14400000, "grad_norm": 0.326171875, "lr": 3e-05, "finish_rate": 0.877, "comp_len": 491.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.8, "frames": {"chat": 244}, "mem_gb": 9.91}
|
| 121 |
+
{"step": 121, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.1078583001211596, "tokens": 120000, "cumulative_loss_tokens": 14520000, "grad_norm": 0.333984375, "lr": 3e-05, "finish_rate": 0.838, "comp_len": 540.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.2, "frames": {"chat": 222}, "mem_gb": 9.95}
|
| 122 |
+
{"step": 122, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.1223276870971856, "tokens": 120000, "cumulative_loss_tokens": 14640000, "grad_norm": 0.3671875, "lr": 3e-05, "finish_rate": 0.78, "comp_len": 550.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.0, "frames": {"chat": 218}, "mem_gb": 10.0}
|
| 123 |
+
{"step": 123, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.12229978259069224, "tokens": 120000, "cumulative_loss_tokens": 14760000, "grad_norm": 0.349609375, "lr": 3e-05, "finish_rate": 0.913, "comp_len": 476.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 46.8, "frames": {"chat": 252}, "mem_gb": 9.87}
|
| 124 |
+
{"step": 124, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.1320776972546242, "tokens": 120000, "cumulative_loss_tokens": 14880000, "grad_norm": 0.3984375, "lr": 3e-05, "finish_rate": 0.728, "comp_len": 594.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 42.2, "frames": {"chat": 202}, "mem_gb": 10.05}
|
| 125 |
+
{"step": 125, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.1445485326328315, "tokens": 120000, "cumulative_loss_tokens": 15000000, "grad_norm": 0.408203125, "lr": 3e-05, "finish_rate": 0.835, "comp_len": 506.3, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.8, "frames": {"chat": 237}, "mem_gb": 10.0}
|
| 126 |
+
{"step": 126, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.1339294173414198, "tokens": 120000, "cumulative_loss_tokens": 15120000, "grad_norm": 0.388671875, "lr": 3e-05, "finish_rate": 0.868, "comp_len": 512.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.6, "frames": {"chat": 234}, "mem_gb": 9.98}
|
| 127 |
+
{"step": 127, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.09875204861378298, "tokens": 120000, "cumulative_loss_tokens": 15240000, "grad_norm": 0.330078125, "lr": 3e-05, "finish_rate": 0.809, "comp_len": 558.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.2, "frames": {"chat": 215}, "mem_gb": 10.0}
|
| 128 |
+
{"step": 128, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.10042380276505525, "tokens": 120000, "cumulative_loss_tokens": 15360000, "grad_norm": 0.333984375, "lr": 3e-05, "finish_rate": 0.876, "comp_len": 512.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.8, "frames": {"chat": 234}, "mem_gb": 9.93}
|
| 129 |
+
{"step": 129, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.0915526939183784, "tokens": 120000, "cumulative_loss_tokens": 15480000, "grad_norm": 0.330078125, "lr": 3e-05, "finish_rate": 0.801, "comp_len": 555.6, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 42.1, "frames": {"chat": 216}, "mem_gb": 9.98}
|
| 130 |
+
{"step": 130, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.10634777345014736, "tokens": 120000, "cumulative_loss_tokens": 15600000, "grad_norm": 0.326171875, "lr": 3e-05, "finish_rate": 0.805, "comp_len": 571.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 41.7, "frames": {"chat": 210}, "mem_gb": 9.95}
|
| 131 |
+
{"step": 131, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.12015755086155452, "tokens": 120000, "cumulative_loss_tokens": 15720000, "grad_norm": 0.353515625, "lr": 3e-05, "finish_rate": 0.719, "comp_len": 603.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 41.8, "frames": {"chat": 199}, "mem_gb": 10.0}
|
| 132 |
+
{"step": 132, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.1101692976130483, "tokens": 120000, "cumulative_loss_tokens": 15840000, "grad_norm": 0.337890625, "lr": 3e-05, "finish_rate": 0.824, "comp_len": 571.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 42.7, "frames": {"chat": 210}, "mem_gb": 10.01}
|
| 133 |
+
{"step": 133, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.11059571424775447, "tokens": 120000, "cumulative_loss_tokens": 15960000, "grad_norm": 0.357421875, "lr": 3e-05, "finish_rate": 0.902, "comp_len": 533.3, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 42.9, "frames": {"chat": 225}, "mem_gb": 9.95}
|
| 134 |
+
{"step": 134, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.10813538532971094, "tokens": 120000, "cumulative_loss_tokens": 16080000, "grad_norm": 0.328125, "lr": 3e-05, "finish_rate": 0.913, "comp_len": 474.3, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.3, "frames": {"chat": 253}, "mem_gb": 9.85}
|
| 135 |
+
{"step": 135, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.12084263433534652, "tokens": 120000, "cumulative_loss_tokens": 16200000, "grad_norm": 0.36328125, "lr": 3e-05, "finish_rate": 0.903, "comp_len": 485.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 46.0, "frames": {"chat": 247}, "mem_gb": 9.97}
|
| 136 |
+
{"step": 136, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.13201027247390398, "tokens": 120000, "cumulative_loss_tokens": 16320000, "grad_norm": 0.388671875, "lr": 3e-05, "finish_rate": 0.836, "comp_len": 504.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.3, "frames": {"chat": 238}, "mem_gb": 9.97}
|
| 137 |
+
{"step": 137, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.13818833304295938, "tokens": 120000, "cumulative_loss_tokens": 16440000, "grad_norm": 0.40625, "lr": 3e-05, "finish_rate": 0.86, "comp_len": 510.6, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.4, "frames": {"chat": 235}, "mem_gb": 9.99}
|
| 138 |
+
{"step": 138, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.1345420230248322, "tokens": 120000, "cumulative_loss_tokens": 16560000, "grad_norm": 0.423828125, "lr": 3e-05, "finish_rate": 0.805, "comp_len": 558.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.8, "frames": {"chat": 215}, "mem_gb": 9.96}
|
| 139 |
+
{"step": 139, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.10943617184103156, "tokens": 120000, "cumulative_loss_tokens": 16680000, "grad_norm": 0.365234375, "lr": 3e-05, "finish_rate": 0.925, "comp_len": 447.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.5, "frames": {"chat": 268}, "mem_gb": 9.97}
|
| 140 |
+
{"step": 140, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.12807247791942208, "tokens": 120000, "cumulative_loss_tokens": 16800000, "grad_norm": 0.380859375, "lr": 3e-05, "finish_rate": 0.825, "comp_len": 526.3, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.3, "frames": {"chat": 228}, "mem_gb": 10.0}
|
| 141 |
+
{"step": 141, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.12706402302297454, "tokens": 120000, "cumulative_loss_tokens": 16920000, "grad_norm": 0.38671875, "lr": 3e-05, "finish_rate": 0.881, "comp_len": 476.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.3, "frames": {"chat": 252}, "mem_gb": 9.93}
|
| 142 |
+
{"step": 142, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.12039303982133667, "tokens": 120000, "cumulative_loss_tokens": 17040000, "grad_norm": 0.36328125, "lr": 3e-05, "finish_rate": 0.821, "comp_len": 538.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 42.1, "frames": {"chat": 223}, "mem_gb": 10.01}
|
| 143 |
+
{"step": 143, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.14205731437917177, "tokens": 120000, "cumulative_loss_tokens": 17160000, "grad_norm": 0.3828125, "lr": 3e-05, "finish_rate": 0.805, "comp_len": 531.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.7, "frames": {"chat": 226}, "mem_gb": 9.99}
|
| 144 |
+
{"step": 144, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.1410405628043848, "tokens": 120000, "cumulative_loss_tokens": 17280000, "grad_norm": 0.40625, "lr": 3e-05, "finish_rate": 0.731, "comp_len": 576.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 46.9, "frames": {"chat": 208}, "mem_gb": 10.04}
|
| 145 |
+
{"step": 145, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.09851235140593101, "tokens": 120000, "cumulative_loss_tokens": 17400000, "grad_norm": 0.31640625, "lr": 3e-05, "finish_rate": 0.883, "comp_len": 500.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.6, "frames": {"chat": 240}, "mem_gb": 9.93}
|
| 146 |
+
{"step": 146, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.13880995498470342, "tokens": 120000, "cumulative_loss_tokens": 17520000, "grad_norm": 0.396484375, "lr": 3e-05, "finish_rate": 0.842, "comp_len": 540.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.3, "frames": {"chat": 222}, "mem_gb": 9.92}
|
| 147 |
+
{"step": 147, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.10342208843190843, "tokens": 120000, "cumulative_loss_tokens": 17640000, "grad_norm": 0.328125, "lr": 3e-05, "finish_rate": 0.881, "comp_len": 508.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.5, "frames": {"chat": 236}, "mem_gb": 9.99}
|
| 148 |
+
{"step": 148, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.1016613077900062, "tokens": 120000, "cumulative_loss_tokens": 17760000, "grad_norm": 0.353515625, "lr": 3e-05, "finish_rate": 0.834, "comp_len": 553.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 42.6, "frames": {"chat": 217}, "mem_gb": 9.96}
|
| 149 |
+
{"step": 149, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.12922203347664327, "tokens": 120000, "cumulative_loss_tokens": 17880000, "grad_norm": 0.48046875, "lr": 3e-05, "finish_rate": 0.921, "comp_len": 476.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.9, "frames": {"chat": 252}, "mem_gb": 9.87}
|
| 150 |
+
{"step": 150, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.10404405277933304, "tokens": 120000, "cumulative_loss_tokens": 18000000, "grad_norm": 0.359375, "lr": 3e-05, "finish_rate": 0.847, "comp_len": 540.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 42.8, "frames": {"chat": 222}, "mem_gb": 9.99}
|
healed/grid_math/uniform_keep25_s1226/train_log.jsonl
ADDED
|
@@ -0,0 +1,150 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"step": 1, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 1.2495578979462385, "tokens": 120000, "cumulative_loss_tokens": 120000, "grad_norm": 14.5625, "lr": 6e-06, "finish_rate": 0.902, "comp_len": 472.4, "t_data_s": 0.2, "t_rollout_s": 0.0, "t_step_s": 36.8, "frames": {"chat": 254}, "mem_gb": 9.82}
|
| 2 |
+
{"step": 2, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 1.2842795796026787, "tokens": 120000, "cumulative_loss_tokens": 240000, "grad_norm": 14.25, "lr": 9e-06, "finish_rate": 0.876, "comp_len": 497.9, "t_data_s": 0.1, "t_rollout_s": 0.0, "t_step_s": 29.4, "frames": {"chat": 241}, "mem_gb": 9.98}
|
| 3 |
+
{"step": 3, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 1.2860378812308113, "tokens": 120000, "cumulative_loss_tokens": 360000, "grad_norm": 13.3125, "lr": 1.2e-05, "finish_rate": 0.746, "comp_len": 563.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.7, "frames": {"chat": 213}, "mem_gb": 10.01}
|
| 4 |
+
{"step": 4, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 1.0296244965081414, "tokens": 120000, "cumulative_loss_tokens": 480000, "grad_norm": 8.625, "lr": 1.5e-05, "finish_rate": 0.864, "comp_len": 543.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.3, "frames": {"chat": 221}, "mem_gb": 10.05}
|
| 5 |
+
{"step": 5, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 1.0173526370272041, "tokens": 120000, "cumulative_loss_tokens": 600000, "grad_norm": 6.9375, "lr": 1.8e-05, "finish_rate": 0.745, "comp_len": 612.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 27.0, "frames": {"chat": 196}, "mem_gb": 10.01}
|
| 6 |
+
{"step": 6, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.7652708411070208, "tokens": 120000, "cumulative_loss_tokens": 720000, "grad_norm": 4.1875, "lr": 2.1e-05, "finish_rate": 0.926, "comp_len": 444.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 31.7, "frames": {"chat": 270}, "mem_gb": 9.82}
|
| 7 |
+
{"step": 7, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.7675839649726948, "tokens": 120000, "cumulative_loss_tokens": 840000, "grad_norm": 3.734375, "lr": 2.4e-05, "finish_rate": 0.815, "comp_len": 555.6, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.0, "frames": {"chat": 216}, "mem_gb": 10.0}
|
| 8 |
+
{"step": 8, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.7070816349441806, "tokens": 120000, "cumulative_loss_tokens": 960000, "grad_norm": 3.28125, "lr": 2.7000000000000002e-05, "finish_rate": 0.775, "comp_len": 600.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 26.7, "frames": {"chat": 200}, "mem_gb": 9.96}
|
| 9 |
+
{"step": 9, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.6169270259405176, "tokens": 120000, "cumulative_loss_tokens": 1080000, "grad_norm": 2.75, "lr": 3e-05, "finish_rate": 0.767, "comp_len": 582.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 27.5, "frames": {"chat": 206}, "mem_gb": 9.91}
|
| 10 |
+
{"step": 10, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.5323196953435739, "tokens": 120000, "cumulative_loss_tokens": 1200000, "grad_norm": 1.9609375, "lr": 3e-05, "finish_rate": 0.902, "comp_len": 512.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.3, "frames": {"chat": 234}, "mem_gb": 9.95}
|
| 11 |
+
{"step": 11, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.5534217075144251, "tokens": 120000, "cumulative_loss_tokens": 1320000, "grad_norm": 1.640625, "lr": 3e-05, "finish_rate": 0.823, "comp_len": 558.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 27.2, "frames": {"chat": 215}, "mem_gb": 9.96}
|
| 12 |
+
{"step": 12, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.470129220243295, "tokens": 120000, "cumulative_loss_tokens": 1440000, "grad_norm": 1.1953125, "lr": 3e-05, "finish_rate": 0.922, "comp_len": 470.6, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 29.1, "frames": {"chat": 255}, "mem_gb": 9.94}
|
| 13 |
+
{"step": 13, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.4660646371759474, "tokens": 120000, "cumulative_loss_tokens": 1560000, "grad_norm": 1.09375, "lr": 3e-05, "finish_rate": 0.892, "comp_len": 480.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 30.0, "frames": {"chat": 250}, "mem_gb": 9.82}
|
| 14 |
+
{"step": 14, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.4519369126672546, "tokens": 120000, "cumulative_loss_tokens": 1680000, "grad_norm": 1.0234375, "lr": 3e-05, "finish_rate": 0.884, "comp_len": 495.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.8, "frames": {"chat": 242}, "mem_gb": 10.0}
|
| 15 |
+
{"step": 15, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.48208508322536947, "tokens": 120000, "cumulative_loss_tokens": 1800000, "grad_norm": 0.97265625, "lr": 3e-05, "finish_rate": 0.729, "comp_len": 603.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.1, "frames": {"chat": 199}, "mem_gb": 10.0}
|
| 16 |
+
{"step": 16, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.505666505916665, "tokens": 120000, "cumulative_loss_tokens": 1920000, "grad_norm": 0.98046875, "lr": 3e-05, "finish_rate": 0.784, "comp_len": 576.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 29.4, "frames": {"chat": 208}, "mem_gb": 10.04}
|
| 17 |
+
{"step": 17, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.42821578803832333, "tokens": 120000, "cumulative_loss_tokens": 2040000, "grad_norm": 1.0078125, "lr": 3e-05, "finish_rate": 0.764, "comp_len": 576.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 27.3, "frames": {"chat": 208}, "mem_gb": 9.97}
|
| 18 |
+
{"step": 18, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.4488865850756566, "tokens": 120000, "cumulative_loss_tokens": 2160000, "grad_norm": 0.8359375, "lr": 3e-05, "finish_rate": 0.732, "comp_len": 574.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 29.5, "frames": {"chat": 209}, "mem_gb": 10.12}
|
| 19 |
+
{"step": 19, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.3713087936893105, "tokens": 120000, "cumulative_loss_tokens": 2280000, "grad_norm": 12.4375, "lr": 3e-05, "finish_rate": 0.855, "comp_len": 510.6, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 29.6, "frames": {"chat": 235}, "mem_gb": 9.96}
|
| 20 |
+
{"step": 20, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.3715890112740298, "tokens": 120000, "cumulative_loss_tokens": 2400000, "grad_norm": 0.80078125, "lr": 3e-05, "finish_rate": 0.74, "comp_len": 588.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 27.4, "frames": {"chat": 204}, "mem_gb": 9.95}
|
| 21 |
+
{"step": 21, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.44148291802903017, "tokens": 120000, "cumulative_loss_tokens": 2520000, "grad_norm": 0.9375, "lr": 3e-05, "finish_rate": 0.745, "comp_len": 576.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 29.5, "frames": {"chat": 208}, "mem_gb": 10.01}
|
| 22 |
+
{"step": 22, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.36412479583832125, "tokens": 120000, "cumulative_loss_tokens": 2640000, "grad_norm": 0.80859375, "lr": 3e-05, "finish_rate": 0.825, "comp_len": 500.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 30.2, "frames": {"chat": 240}, "mem_gb": 10.0}
|
| 23 |
+
{"step": 23, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.35719255141379935, "tokens": 120000, "cumulative_loss_tokens": 2760000, "grad_norm": 0.69921875, "lr": 3e-05, "finish_rate": 0.89, "comp_len": 487.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 29.5, "frames": {"chat": 246}, "mem_gb": 9.99}
|
| 24 |
+
{"step": 24, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.3621041040317466, "tokens": 120000, "cumulative_loss_tokens": 2880000, "grad_norm": 0.67578125, "lr": 3e-05, "finish_rate": 0.909, "comp_len": 493.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 29.1, "frames": {"chat": 243}, "mem_gb": 9.82}
|
| 25 |
+
{"step": 25, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.38283104023163517, "tokens": 120000, "cumulative_loss_tokens": 3000000, "grad_norm": 0.68359375, "lr": 3e-05, "finish_rate": 0.745, "comp_len": 576.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 27.5, "frames": {"chat": 208}, "mem_gb": 10.01}
|
| 26 |
+
{"step": 26, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.3491992857553065, "tokens": 120000, "cumulative_loss_tokens": 3120000, "grad_norm": 0.67578125, "lr": 3e-05, "finish_rate": 0.817, "comp_len": 547.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.1, "frames": {"chat": 219}, "mem_gb": 10.0}
|
| 27 |
+
{"step": 27, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.3949156841669232, "tokens": 120000, "cumulative_loss_tokens": 3240000, "grad_norm": 0.80859375, "lr": 3e-05, "finish_rate": 0.782, "comp_len": 568.7, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 29.0, "frames": {"chat": 211}, "mem_gb": 10.01}
|
| 28 |
+
{"step": 28, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.3246795868317286, "tokens": 120000, "cumulative_loss_tokens": 3360000, "grad_norm": 0.69140625, "lr": 3e-05, "finish_rate": 0.862, "comp_len": 517.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 29.0, "frames": {"chat": 232}, "mem_gb": 9.98}
|
| 29 |
+
{"step": 29, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.3552298259820789, "tokens": 120000, "cumulative_loss_tokens": 3480000, "grad_norm": 0.73046875, "lr": 3e-05, "finish_rate": 0.804, "comp_len": 560.7, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.1, "frames": {"chat": 214}, "mem_gb": 10.01}
|
| 30 |
+
{"step": 30, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.3321069305093338, "tokens": 120000, "cumulative_loss_tokens": 3600000, "grad_norm": 0.65625, "lr": 3e-05, "finish_rate": 0.819, "comp_len": 531.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.7, "frames": {"chat": 226}, "mem_gb": 9.9}
|
| 31 |
+
{"step": 31, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.32228971752213936, "tokens": 120000, "cumulative_loss_tokens": 3720000, "grad_norm": 0.64453125, "lr": 3e-05, "finish_rate": 0.819, "comp_len": 571.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.2, "frames": {"chat": 210}, "mem_gb": 10.01}
|
| 32 |
+
{"step": 32, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.2798056041251868, "tokens": 120000, "cumulative_loss_tokens": 3840000, "grad_norm": 0.58203125, "lr": 3e-05, "finish_rate": 0.858, "comp_len": 550.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.4, "frames": {"chat": 218}, "mem_gb": 9.83}
|
| 33 |
+
{"step": 33, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.29189739949926735, "tokens": 120000, "cumulative_loss_tokens": 3960000, "grad_norm": 0.61328125, "lr": 3e-05, "finish_rate": 0.858, "comp_len": 515.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 29.5, "frames": {"chat": 233}, "mem_gb": 9.99}
|
| 34 |
+
{"step": 34, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.35060337477376063, "tokens": 120000, "cumulative_loss_tokens": 4080000, "grad_norm": 0.64453125, "lr": 3e-05, "finish_rate": 0.786, "comp_len": 558.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 29.8, "frames": {"chat": 215}, "mem_gb": 10.01}
|
| 35 |
+
{"step": 35, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.33954473176573713, "tokens": 120000, "cumulative_loss_tokens": 4200000, "grad_norm": 0.640625, "lr": 3e-05, "finish_rate": 0.845, "comp_len": 515.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.3, "frames": {"chat": 233}, "mem_gb": 9.99}
|
| 36 |
+
{"step": 36, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.2843651156159739, "tokens": 120000, "cumulative_loss_tokens": 4320000, "grad_norm": 1.828125, "lr": 3e-05, "finish_rate": 0.766, "comp_len": 574.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 27.9, "frames": {"chat": 209}, "mem_gb": 9.94}
|
| 37 |
+
{"step": 37, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.2947497258403649, "tokens": 120000, "cumulative_loss_tokens": 4440000, "grad_norm": 0.72265625, "lr": 3e-05, "finish_rate": 0.908, "comp_len": 458.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 30.6, "frames": {"chat": 262}, "mem_gb": 9.88}
|
| 38 |
+
{"step": 38, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.28674538816077016, "tokens": 120000, "cumulative_loss_tokens": 4560000, "grad_norm": 0.56640625, "lr": 3e-05, "finish_rate": 0.9, "comp_len": 481.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 29.9, "frames": {"chat": 249}, "mem_gb": 9.96}
|
| 39 |
+
{"step": 39, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.33914187191314993, "tokens": 120000, "cumulative_loss_tokens": 4680000, "grad_norm": 1.4921875, "lr": 3e-05, "finish_rate": 0.819, "comp_len": 528.6, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 29.3, "frames": {"chat": 227}, "mem_gb": 10.0}
|
| 40 |
+
{"step": 40, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.26380810491734497, "tokens": 120000, "cumulative_loss_tokens": 4800000, "grad_norm": 0.5234375, "lr": 3e-05, "finish_rate": 0.814, "comp_len": 543.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.4, "frames": {"chat": 221}, "mem_gb": 10.0}
|
| 41 |
+
{"step": 41, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.302417300863564, "tokens": 120000, "cumulative_loss_tokens": 4920000, "grad_norm": 0.5546875, "lr": 3e-05, "finish_rate": 0.859, "comp_len": 512.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.4, "frames": {"chat": 234}, "mem_gb": 10.01}
|
| 42 |
+
{"step": 42, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.24419383710200587, "tokens": 120000, "cumulative_loss_tokens": 5040000, "grad_norm": 0.52734375, "lr": 3e-05, "finish_rate": 0.817, "comp_len": 563.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 27.3, "frames": {"chat": 213}, "mem_gb": 9.96}
|
| 43 |
+
{"step": 43, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.24537141441050916, "tokens": 120000, "cumulative_loss_tokens": 5160000, "grad_norm": 0.51953125, "lr": 3e-05, "finish_rate": 0.836, "comp_len": 563.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 27.5, "frames": {"chat": 213}, "mem_gb": 9.9}
|
| 44 |
+
{"step": 44, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.27504100236129014, "tokens": 120000, "cumulative_loss_tokens": 5280000, "grad_norm": 0.53125, "lr": 3e-05, "finish_rate": 0.906, "comp_len": 512.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.8, "frames": {"chat": 234}, "mem_gb": 9.93}
|
| 45 |
+
{"step": 45, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.2705650013284758, "tokens": 120000, "cumulative_loss_tokens": 5400000, "grad_norm": 0.5, "lr": 3e-05, "finish_rate": 0.793, "comp_len": 540.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.4, "frames": {"chat": 222}, "mem_gb": 10.0}
|
| 46 |
+
{"step": 46, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.31124431811695297, "tokens": 120000, "cumulative_loss_tokens": 5520000, "grad_norm": 0.59375, "lr": 3e-05, "finish_rate": 0.806, "comp_len": 528.6, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 31.3, "frames": {"chat": 227}, "mem_gb": 10.01}
|
| 47 |
+
{"step": 47, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.2672547916886707, "tokens": 120000, "cumulative_loss_tokens": 5640000, "grad_norm": 0.51171875, "lr": 3e-05, "finish_rate": 0.835, "comp_len": 550.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.2, "frames": {"chat": 218}, "mem_gb": 10.04}
|
| 48 |
+
{"step": 48, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.3164061588189254, "tokens": 120000, "cumulative_loss_tokens": 5760000, "grad_norm": 0.57421875, "lr": 3e-05, "finish_rate": 0.798, "comp_len": 538.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.8, "frames": {"chat": 223}, "mem_gb": 10.01}
|
| 49 |
+
{"step": 49, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.31090367152442533, "tokens": 120000, "cumulative_loss_tokens": 5880000, "grad_norm": 0.56640625, "lr": 3e-05, "finish_rate": 0.772, "comp_len": 582.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 27.8, "frames": {"chat": 206}, "mem_gb": 10.01}
|
| 50 |
+
{"step": 50, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.2863034582992395, "tokens": 120000, "cumulative_loss_tokens": 6000000, "grad_norm": 0.5625, "lr": 3e-05, "finish_rate": 0.784, "comp_len": 563.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 27.9, "frames": {"chat": 213}, "mem_gb": 9.92}
|
| 51 |
+
{"step": 51, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.33238482810370623, "tokens": 120000, "cumulative_loss_tokens": 6120000, "grad_norm": 0.5859375, "lr": 3e-05, "finish_rate": 0.843, "comp_len": 538.1, "t_data_s": 0.1, "t_rollout_s": 0.0, "t_step_s": 30.3, "frames": {"chat": 223}, "mem_gb": 9.86}
|
| 52 |
+
{"step": 52, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.30147026825944584, "tokens": 120000, "cumulative_loss_tokens": 6240000, "grad_norm": 0.5703125, "lr": 3e-05, "finish_rate": 0.828, "comp_len": 528.6, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.3, "frames": {"chat": 227}, "mem_gb": 9.97}
|
| 53 |
+
{"step": 53, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.2694238721400499, "tokens": 120000, "cumulative_loss_tokens": 6360000, "grad_norm": 1.2109375, "lr": 3e-05, "finish_rate": 0.889, "comp_len": 474.3, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 30.2, "frames": {"chat": 253}, "mem_gb": 10.0}
|
| 54 |
+
{"step": 54, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.2599986071868489, "tokens": 120000, "cumulative_loss_tokens": 6480000, "grad_norm": 0.609375, "lr": 3e-05, "finish_rate": 0.792, "comp_len": 555.6, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 27.7, "frames": {"chat": 216}, "mem_gb": 10.01}
|
| 55 |
+
{"step": 55, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.26703055294280253, "tokens": 120000, "cumulative_loss_tokens": 6600000, "grad_norm": 0.53125, "lr": 3e-05, "finish_rate": 0.766, "comp_len": 585.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 27.4, "frames": {"chat": 205}, "mem_gb": 9.98}
|
| 56 |
+
{"step": 56, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.2845016695648432, "tokens": 120000, "cumulative_loss_tokens": 6720000, "grad_norm": 0.6328125, "lr": 3e-05, "finish_rate": 0.729, "comp_len": 579.7, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 30.1, "frames": {"chat": 207}, "mem_gb": 10.06}
|
| 57 |
+
{"step": 57, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.2710578287235151, "tokens": 120000, "cumulative_loss_tokens": 6840000, "grad_norm": 0.53515625, "lr": 3e-05, "finish_rate": 0.814, "comp_len": 558.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 29.5, "frames": {"chat": 215}, "mem_gb": 9.98}
|
| 58 |
+
{"step": 58, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.2025449087051054, "tokens": 120000, "cumulative_loss_tokens": 6960000, "grad_norm": 0.4765625, "lr": 3e-05, "finish_rate": 0.86, "comp_len": 526.3, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.6, "frames": {"chat": 228}, "mem_gb": 10.0}
|
| 59 |
+
{"step": 59, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.2637373869329691, "tokens": 120000, "cumulative_loss_tokens": 7080000, "grad_norm": 0.5625, "lr": 3e-05, "finish_rate": 0.747, "comp_len": 543.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 29.7, "frames": {"chat": 221}, "mem_gb": 10.04}
|
| 60 |
+
{"step": 60, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.20766283342484385, "tokens": 120000, "cumulative_loss_tokens": 7200000, "grad_norm": 0.53515625, "lr": 3e-05, "finish_rate": 0.882, "comp_len": 472.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 29.7, "frames": {"chat": 254}, "mem_gb": 9.84}
|
| 61 |
+
{"step": 61, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.17684483035219212, "tokens": 120000, "cumulative_loss_tokens": 7320000, "grad_norm": 0.5078125, "lr": 3e-05, "finish_rate": 0.843, "comp_len": 571.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 27.7, "frames": {"chat": 210}, "mem_gb": 9.97}
|
| 62 |
+
{"step": 62, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.21151079111825674, "tokens": 120000, "cumulative_loss_tokens": 7440000, "grad_norm": 0.48046875, "lr": 3e-05, "finish_rate": 0.827, "comp_len": 531.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.8, "frames": {"chat": 226}, "mem_gb": 9.92}
|
| 63 |
+
{"step": 63, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.22429327983123562, "tokens": 120000, "cumulative_loss_tokens": 7560000, "grad_norm": 0.48046875, "lr": 3e-05, "finish_rate": 0.802, "comp_len": 566.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.1, "frames": {"chat": 212}, "mem_gb": 9.99}
|
| 64 |
+
{"step": 64, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.23901226744391024, "tokens": 120000, "cumulative_loss_tokens": 7680000, "grad_norm": 0.4921875, "lr": 3e-05, "finish_rate": 0.754, "comp_len": 568.7, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 29.3, "frames": {"chat": 211}, "mem_gb": 9.93}
|
| 65 |
+
{"step": 65, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.2521503586698324, "tokens": 120000, "cumulative_loss_tokens": 7800000, "grad_norm": 0.53125, "lr": 3e-05, "finish_rate": 0.776, "comp_len": 612.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 26.5, "frames": {"chat": 196}, "mem_gb": 9.98}
|
| 66 |
+
{"step": 66, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.19568856958678613, "tokens": 120000, "cumulative_loss_tokens": 7920000, "grad_norm": 0.447265625, "lr": 3e-05, "finish_rate": 0.811, "comp_len": 566.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 27.8, "frames": {"chat": 212}, "mem_gb": 10.0}
|
| 67 |
+
{"step": 67, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.2056147424393023, "tokens": 120000, "cumulative_loss_tokens": 8040000, "grad_norm": 0.447265625, "lr": 3e-05, "finish_rate": 0.877, "comp_len": 491.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 29.5, "frames": {"chat": 244}, "mem_gb": 9.91}
|
| 68 |
+
{"step": 68, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.19970010116541137, "tokens": 120000, "cumulative_loss_tokens": 8160000, "grad_norm": 0.435546875, "lr": 3e-05, "finish_rate": 0.838, "comp_len": 540.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.3, "frames": {"chat": 222}, "mem_gb": 9.95}
|
| 69 |
+
{"step": 69, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.2406689625217269, "tokens": 120000, "cumulative_loss_tokens": 8280000, "grad_norm": 0.5, "lr": 3e-05, "finish_rate": 0.78, "comp_len": 550.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.1, "frames": {"chat": 218}, "mem_gb": 10.0}
|
| 70 |
+
{"step": 70, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.2351806751595189, "tokens": 120000, "cumulative_loss_tokens": 8400000, "grad_norm": 0.494140625, "lr": 3e-05, "finish_rate": 0.913, "comp_len": 476.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 30.7, "frames": {"chat": 252}, "mem_gb": 9.88}
|
| 71 |
+
{"step": 71, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.23980661761003236, "tokens": 120000, "cumulative_loss_tokens": 8520000, "grad_norm": 0.486328125, "lr": 3e-05, "finish_rate": 0.728, "comp_len": 594.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 27.5, "frames": {"chat": 202}, "mem_gb": 10.05}
|
| 72 |
+
{"step": 72, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.2756894440931578, "tokens": 120000, "cumulative_loss_tokens": 8640000, "grad_norm": 0.59765625, "lr": 3e-05, "finish_rate": 0.835, "comp_len": 506.3, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 29.9, "frames": {"chat": 237}, "mem_gb": 10.0}
|
| 73 |
+
{"step": 73, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.22606769043393432, "tokens": 120000, "cumulative_loss_tokens": 8760000, "grad_norm": 0.486328125, "lr": 3e-05, "finish_rate": 0.868, "comp_len": 512.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 29.3, "frames": {"chat": 234}, "mem_gb": 9.99}
|
| 74 |
+
{"step": 74, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.17098218618429575, "tokens": 120000, "cumulative_loss_tokens": 8880000, "grad_norm": 0.4375, "lr": 3e-05, "finish_rate": 0.809, "comp_len": 558.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.2, "frames": {"chat": 215}, "mem_gb": 10.0}
|
| 75 |
+
{"step": 75, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.18496631520005563, "tokens": 120000, "cumulative_loss_tokens": 9000000, "grad_norm": 0.4375, "lr": 3e-05, "finish_rate": 0.876, "comp_len": 512.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.7, "frames": {"chat": 234}, "mem_gb": 9.93}
|
| 76 |
+
{"step": 76, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.16087721765795723, "tokens": 120000, "cumulative_loss_tokens": 9120000, "grad_norm": 0.404296875, "lr": 3e-05, "finish_rate": 0.801, "comp_len": 555.6, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 27.6, "frames": {"chat": 216}, "mem_gb": 9.99}
|
| 77 |
+
{"step": 77, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.1994732022792101, "tokens": 120000, "cumulative_loss_tokens": 9240000, "grad_norm": 0.4609375, "lr": 3e-05, "finish_rate": 0.805, "comp_len": 571.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 27.5, "frames": {"chat": 210}, "mem_gb": 9.95}
|
| 78 |
+
{"step": 78, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.20232079331736702, "tokens": 120000, "cumulative_loss_tokens": 9360000, "grad_norm": 0.443359375, "lr": 3e-05, "finish_rate": 0.719, "comp_len": 603.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 27.2, "frames": {"chat": 199}, "mem_gb": 10.0}
|
| 79 |
+
{"step": 79, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.18569014142416418, "tokens": 120000, "cumulative_loss_tokens": 9480000, "grad_norm": 0.431640625, "lr": 3e-05, "finish_rate": 0.824, "comp_len": 571.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 27.9, "frames": {"chat": 210}, "mem_gb": 10.01}
|
| 80 |
+
{"step": 80, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.20507156935961296, "tokens": 120000, "cumulative_loss_tokens": 9600000, "grad_norm": 0.5390625, "lr": 3e-05, "finish_rate": 0.902, "comp_len": 533.3, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.1, "frames": {"chat": 225}, "mem_gb": 9.95}
|
| 81 |
+
{"step": 81, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.20882732380144298, "tokens": 120000, "cumulative_loss_tokens": 9720000, "grad_norm": 0.486328125, "lr": 3e-05, "finish_rate": 0.913, "comp_len": 474.3, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 29.9, "frames": {"chat": 253}, "mem_gb": 9.85}
|
| 82 |
+
{"step": 82, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.22202617851061127, "tokens": 120000, "cumulative_loss_tokens": 9840000, "grad_norm": 0.458984375, "lr": 3e-05, "finish_rate": 0.903, "comp_len": 485.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 30.2, "frames": {"chat": 247}, "mem_gb": 9.98}
|
| 83 |
+
{"step": 83, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.2211942642432948, "tokens": 120000, "cumulative_loss_tokens": 9960000, "grad_norm": 0.51171875, "lr": 3e-05, "finish_rate": 0.836, "comp_len": 504.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 29.1, "frames": {"chat": 238}, "mem_gb": 9.98}
|
| 84 |
+
{"step": 84, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.23084418179870894, "tokens": 120000, "cumulative_loss_tokens": 10080000, "grad_norm": 0.51171875, "lr": 3e-05, "finish_rate": 0.86, "comp_len": 510.6, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.6, "frames": {"chat": 235}, "mem_gb": 10.0}
|
| 85 |
+
{"step": 85, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.2283516202347974, "tokens": 120000, "cumulative_loss_tokens": 10200000, "grad_norm": 0.498046875, "lr": 3e-05, "finish_rate": 0.805, "comp_len": 558.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.5, "frames": {"chat": 215}, "mem_gb": 9.97}
|
| 86 |
+
{"step": 86, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.22033791828608762, "tokens": 120000, "cumulative_loss_tokens": 10320000, "grad_norm": 0.53515625, "lr": 3e-05, "finish_rate": 0.925, "comp_len": 447.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 30.0, "frames": {"chat": 268}, "mem_gb": 9.97}
|
| 87 |
+
{"step": 87, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.22125197298427424, "tokens": 120000, "cumulative_loss_tokens": 10440000, "grad_norm": 0.486328125, "lr": 3e-05, "finish_rate": 0.825, "comp_len": 526.3, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 29.0, "frames": {"chat": 228}, "mem_gb": 10.0}
|
| 88 |
+
{"step": 88, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.21613482079487295, "tokens": 120000, "cumulative_loss_tokens": 10560000, "grad_norm": 0.49609375, "lr": 3e-05, "finish_rate": 0.881, "comp_len": 476.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 29.8, "frames": {"chat": 252}, "mem_gb": 9.93}
|
| 89 |
+
{"step": 89, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.1895898001347358, "tokens": 120000, "cumulative_loss_tokens": 10680000, "grad_norm": 0.46484375, "lr": 3e-05, "finish_rate": 0.821, "comp_len": 538.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 27.6, "frames": {"chat": 223}, "mem_gb": 10.01}
|
| 90 |
+
{"step": 90, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.26797097403767206, "tokens": 120000, "cumulative_loss_tokens": 10800000, "grad_norm": 0.55078125, "lr": 3e-05, "finish_rate": 0.805, "comp_len": 531.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 29.2, "frames": {"chat": 226}, "mem_gb": 10.0}
|
| 91 |
+
{"step": 91, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.24367389039595921, "tokens": 120000, "cumulative_loss_tokens": 10920000, "grad_norm": 0.55078125, "lr": 3e-05, "finish_rate": 0.731, "comp_len": 576.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 30.4, "frames": {"chat": 208}, "mem_gb": 10.05}
|
| 92 |
+
{"step": 92, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.17548983993784836, "tokens": 120000, "cumulative_loss_tokens": 11040000, "grad_norm": 0.4296875, "lr": 3e-05, "finish_rate": 0.883, "comp_len": 500.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 29.2, "frames": {"chat": 240}, "mem_gb": 9.93}
|
| 93 |
+
{"step": 93, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.21608840946884206, "tokens": 120000, "cumulative_loss_tokens": 11160000, "grad_norm": 0.47265625, "lr": 3e-05, "finish_rate": 0.842, "comp_len": 540.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 29.5, "frames": {"chat": 222}, "mem_gb": 9.93}
|
| 94 |
+
{"step": 94, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.1903983515452904, "tokens": 120000, "cumulative_loss_tokens": 11280000, "grad_norm": 0.455078125, "lr": 3e-05, "finish_rate": 0.881, "comp_len": 508.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.5, "frames": {"chat": 236}, "mem_gb": 10.0}
|
| 95 |
+
{"step": 95, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.17124010507706552, "tokens": 120000, "cumulative_loss_tokens": 11400000, "grad_norm": 0.427734375, "lr": 3e-05, "finish_rate": 0.834, "comp_len": 553.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 27.8, "frames": {"chat": 217}, "mem_gb": 9.97}
|
| 96 |
+
{"step": 96, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.21378996573444456, "tokens": 120000, "cumulative_loss_tokens": 11520000, "grad_norm": 0.59375, "lr": 3e-05, "finish_rate": 0.921, "comp_len": 476.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 29.6, "frames": {"chat": 252}, "mem_gb": 9.88}
|
| 97 |
+
{"step": 97, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.17583670812180888, "tokens": 120000, "cumulative_loss_tokens": 11640000, "grad_norm": 0.427734375, "lr": 3e-05, "finish_rate": 0.847, "comp_len": 540.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.1, "frames": {"chat": 222}, "mem_gb": 9.99}
|
| 98 |
+
{"step": 98, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.2008310172341764, "tokens": 120000, "cumulative_loss_tokens": 11760000, "grad_norm": 0.470703125, "lr": 3e-05, "finish_rate": 0.901, "comp_len": 495.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 29.1, "frames": {"chat": 242}, "mem_gb": 9.87}
|
| 99 |
+
{"step": 99, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.2504596530464788, "tokens": 120000, "cumulative_loss_tokens": 11880000, "grad_norm": 0.53515625, "lr": 3e-05, "finish_rate": 0.804, "comp_len": 547.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 29.7, "frames": {"chat": 219}, "mem_gb": 9.93}
|
| 100 |
+
{"step": 100, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.20776792142900327, "tokens": 120000, "cumulative_loss_tokens": 12000000, "grad_norm": 0.44140625, "lr": 3e-05, "finish_rate": 0.879, "comp_len": 538.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 27.8, "frames": {"chat": 223}, "mem_gb": 9.94}
|
| 101 |
+
{"step": 101, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.21407153125032782, "tokens": 120000, "cumulative_loss_tokens": 12120000, "grad_norm": 0.47265625, "lr": 3e-05, "finish_rate": 0.858, "comp_len": 517.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.3, "frames": {"chat": 232}, "mem_gb": 9.95}
|
| 102 |
+
{"step": 102, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.22760161069128662, "tokens": 120000, "cumulative_loss_tokens": 12240000, "grad_norm": 0.53125, "lr": 3e-05, "finish_rate": 0.832, "comp_len": 545.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 29.2, "frames": {"chat": 220}, "mem_gb": 10.0}
|
| 103 |
+
{"step": 103, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.23536558933630586, "tokens": 120000, "cumulative_loss_tokens": 12360000, "grad_norm": 0.51171875, "lr": 3e-05, "finish_rate": 0.776, "comp_len": 571.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.6, "frames": {"chat": 210}, "mem_gb": 10.04}
|
| 104 |
+
{"step": 104, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.20387262995596975, "tokens": 120000, "cumulative_loss_tokens": 12480000, "grad_norm": 0.482421875, "lr": 3e-05, "finish_rate": 0.81, "comp_len": 531.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 29.6, "frames": {"chat": 226}, "mem_gb": 9.97}
|
| 105 |
+
{"step": 105, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.1928112811392794, "tokens": 120000, "cumulative_loss_tokens": 12600000, "grad_norm": 0.466796875, "lr": 3e-05, "finish_rate": 0.741, "comp_len": 566.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 27.8, "frames": {"chat": 212}, "mem_gb": 10.0}
|
| 106 |
+
{"step": 106, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.18794621144427606, "tokens": 120000, "cumulative_loss_tokens": 12720000, "grad_norm": 0.48828125, "lr": 3e-05, "finish_rate": 0.839, "comp_len": 508.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 29.0, "frames": {"chat": 236}, "mem_gb": 10.01}
|
| 107 |
+
{"step": 107, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.16627150794851284, "tokens": 120000, "cumulative_loss_tokens": 12840000, "grad_norm": 0.56640625, "lr": 3e-05, "finish_rate": 0.928, "comp_len": 454.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 29.7, "frames": {"chat": 264}, "mem_gb": 9.88}
|
| 108 |
+
{"step": 108, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.21027346796846638, "tokens": 120000, "cumulative_loss_tokens": 12960000, "grad_norm": 0.48046875, "lr": 3e-05, "finish_rate": 0.834, "comp_len": 524.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.3, "frames": {"chat": 229}, "mem_gb": 9.98}
|
| 109 |
+
{"step": 109, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.16645300974740337, "tokens": 120000, "cumulative_loss_tokens": 13080000, "grad_norm": 0.5078125, "lr": 3e-05, "finish_rate": 0.903, "comp_len": 465.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 29.9, "frames": {"chat": 258}, "mem_gb": 9.85}
|
| 110 |
+
{"step": 110, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.21918750831211606, "tokens": 120000, "cumulative_loss_tokens": 13200000, "grad_norm": 0.55859375, "lr": 3e-05, "finish_rate": 0.755, "comp_len": 576.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.4, "frames": {"chat": 208}, "mem_gb": 10.01}
|
| 111 |
+
{"step": 111, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.18296561656640842, "tokens": 120000, "cumulative_loss_tokens": 13320000, "grad_norm": 0.48046875, "lr": 3e-05, "finish_rate": 0.88, "comp_len": 481.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 29.4, "frames": {"chat": 249}, "mem_gb": 9.93}
|
| 112 |
+
{"step": 112, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.14481682858659575, "tokens": 120000, "cumulative_loss_tokens": 13440000, "grad_norm": 0.41015625, "lr": 3e-05, "finish_rate": 0.845, "comp_len": 545.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 27.5, "frames": {"chat": 220}, "mem_gb": 9.99}
|
| 113 |
+
{"step": 113, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.1859534092683966, "tokens": 120000, "cumulative_loss_tokens": 13560000, "grad_norm": 0.56640625, "lr": 3e-05, "finish_rate": 0.834, "comp_len": 538.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 27.8, "frames": {"chat": 223}, "mem_gb": 9.99}
|
| 114 |
+
{"step": 114, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.13867307858555578, "tokens": 120000, "cumulative_loss_tokens": 13680000, "grad_norm": 0.458984375, "lr": 3e-05, "finish_rate": 0.833, "comp_len": 543.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.1, "frames": {"chat": 221}, "mem_gb": 10.0}
|
| 115 |
+
{"step": 115, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.15031419390533118, "tokens": 120000, "cumulative_loss_tokens": 13800000, "grad_norm": 0.427734375, "lr": 3e-05, "finish_rate": 0.9, "comp_len": 521.7, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.5, "frames": {"chat": 230}, "mem_gb": 9.9}
|
| 116 |
+
{"step": 116, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.17688426749395827, "tokens": 120000, "cumulative_loss_tokens": 13920000, "grad_norm": 0.458984375, "lr": 3e-05, "finish_rate": 0.776, "comp_len": 560.7, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.4, "frames": {"chat": 214}, "mem_gb": 9.97}
|
| 117 |
+
{"step": 117, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.2198834235218664, "tokens": 120000, "cumulative_loss_tokens": 14040000, "grad_norm": 0.51171875, "lr": 3e-05, "finish_rate": 0.766, "comp_len": 560.7, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 29.7, "frames": {"chat": 214}, "mem_gb": 9.99}
|
| 118 |
+
{"step": 118, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.1655972873053203, "tokens": 120000, "cumulative_loss_tokens": 14160000, "grad_norm": 0.478515625, "lr": 3e-05, "finish_rate": 0.786, "comp_len": 571.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.5, "frames": {"chat": 210}, "mem_gb": 10.04}
|
| 119 |
+
{"step": 119, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.18060686992689345, "tokens": 120000, "cumulative_loss_tokens": 14280000, "grad_norm": 0.45703125, "lr": 3e-05, "finish_rate": 0.776, "comp_len": 560.7, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 29.2, "frames": {"chat": 214}, "mem_gb": 10.0}
|
| 120 |
+
{"step": 120, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.18515299578898897, "tokens": 120000, "cumulative_loss_tokens": 14400000, "grad_norm": 0.451171875, "lr": 3e-05, "finish_rate": 0.791, "comp_len": 558.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 29.9, "frames": {"chat": 215}, "mem_gb": 9.96}
|
| 121 |
+
{"step": 121, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.19961161344274878, "tokens": 120000, "cumulative_loss_tokens": 14520000, "grad_norm": 0.4921875, "lr": 3e-05, "finish_rate": 0.721, "comp_len": 576.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 29.4, "frames": {"chat": 208}, "mem_gb": 9.99}
|
| 122 |
+
{"step": 122, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.150687341630583, "tokens": 120000, "cumulative_loss_tokens": 14640000, "grad_norm": 0.400390625, "lr": 3e-05, "finish_rate": 0.789, "comp_len": 550.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.4, "frames": {"chat": 218}, "mem_gb": 9.88}
|
| 123 |
+
{"step": 123, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.1382546880559375, "tokens": 120000, "cumulative_loss_tokens": 14760000, "grad_norm": 0.388671875, "lr": 3e-05, "finish_rate": 0.876, "comp_len": 515.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.7, "frames": {"chat": 233}, "mem_gb": 9.9}
|
| 124 |
+
{"step": 124, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.13145577659346164, "tokens": 120000, "cumulative_loss_tokens": 14880000, "grad_norm": 0.388671875, "lr": 3e-05, "finish_rate": 0.861, "comp_len": 519.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.8, "frames": {"chat": 231}, "mem_gb": 9.94}
|
| 125 |
+
{"step": 125, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.16622866188141827, "tokens": 120000, "cumulative_loss_tokens": 15000000, "grad_norm": 0.435546875, "lr": 3e-05, "finish_rate": 0.868, "comp_len": 510.6, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 29.8, "frames": {"chat": 235}, "mem_gb": 10.13}
|
| 126 |
+
{"step": 126, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.17218660681688538, "tokens": 120000, "cumulative_loss_tokens": 15120000, "grad_norm": 0.419921875, "lr": 3e-05, "finish_rate": 0.843, "comp_len": 555.6, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.1, "frames": {"chat": 216}, "mem_gb": 9.99}
|
| 127 |
+
{"step": 127, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.16202461753959457, "tokens": 120000, "cumulative_loss_tokens": 15240000, "grad_norm": 0.427734375, "lr": 3e-05, "finish_rate": 0.831, "comp_len": 506.3, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 29.1, "frames": {"chat": 237}, "mem_gb": 10.01}
|
| 128 |
+
{"step": 128, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.1946208799743404, "tokens": 120000, "cumulative_loss_tokens": 15360000, "grad_norm": 0.46484375, "lr": 3e-05, "finish_rate": 0.734, "comp_len": 591.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.9, "frames": {"chat": 203}, "mem_gb": 10.01}
|
| 129 |
+
{"step": 129, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.16476607479564845, "tokens": 120000, "cumulative_loss_tokens": 15480000, "grad_norm": 0.427734375, "lr": 3e-05, "finish_rate": 0.873, "comp_len": 508.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 29.5, "frames": {"chat": 236}, "mem_gb": 10.04}
|
| 130 |
+
{"step": 130, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.16959726118591303, "tokens": 120000, "cumulative_loss_tokens": 15600000, "grad_norm": 0.42578125, "lr": 3e-05, "finish_rate": 0.734, "comp_len": 560.7, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.0, "frames": {"chat": 214}, "mem_gb": 10.01}
|
| 131 |
+
{"step": 131, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.15791527282614262, "tokens": 120000, "cumulative_loss_tokens": 15720000, "grad_norm": 0.421875, "lr": 3e-05, "finish_rate": 0.78, "comp_len": 574.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.7, "frames": {"chat": 209}, "mem_gb": 10.0}
|
| 132 |
+
{"step": 132, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.16019434139728547, "tokens": 120000, "cumulative_loss_tokens": 15840000, "grad_norm": 0.443359375, "lr": 3e-05, "finish_rate": 0.906, "comp_len": 468.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 30.5, "frames": {"chat": 256}, "mem_gb": 10.0}
|
| 133 |
+
{"step": 133, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.1639191758962348, "tokens": 120000, "cumulative_loss_tokens": 15960000, "grad_norm": 0.455078125, "lr": 3e-05, "finish_rate": 0.878, "comp_len": 521.7, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 29.0, "frames": {"chat": 230}, "mem_gb": 9.87}
|
| 134 |
+
{"step": 134, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.16772623434948425, "tokens": 120000, "cumulative_loss_tokens": 16080000, "grad_norm": 0.439453125, "lr": 3e-05, "finish_rate": 0.822, "comp_len": 521.7, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 29.3, "frames": {"chat": 230}, "mem_gb": 10.06}
|
| 135 |
+
{"step": 135, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.1819166350507488, "tokens": 120000, "cumulative_loss_tokens": 16200000, "grad_norm": 0.44921875, "lr": 3e-05, "finish_rate": 0.881, "comp_len": 528.6, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.6, "frames": {"chat": 227}, "mem_gb": 9.95}
|
| 136 |
+
{"step": 136, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.1837459068759655, "tokens": 120000, "cumulative_loss_tokens": 16320000, "grad_norm": 0.455078125, "lr": 3e-05, "finish_rate": 0.755, "comp_len": 576.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 27.7, "frames": {"chat": 208}, "mem_gb": 10.01}
|
| 137 |
+
{"step": 137, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.19119155149323244, "tokens": 120000, "cumulative_loss_tokens": 16440000, "grad_norm": 0.474609375, "lr": 3e-05, "finish_rate": 0.699, "comp_len": 582.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.9, "frames": {"chat": 206}, "mem_gb": 10.03}
|
| 138 |
+
{"step": 138, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.17735388877652586, "tokens": 120000, "cumulative_loss_tokens": 16560000, "grad_norm": 0.48046875, "lr": 3e-05, "finish_rate": 0.82, "comp_len": 526.3, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.5, "frames": {"chat": 228}, "mem_gb": 9.9}
|
| 139 |
+
{"step": 139, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.17459255363605916, "tokens": 120000, "cumulative_loss_tokens": 16680000, "grad_norm": 0.462890625, "lr": 3e-05, "finish_rate": 0.835, "comp_len": 535.7, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.5, "frames": {"chat": 224}, "mem_gb": 10.0}
|
| 140 |
+
{"step": 140, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.15427039775537948, "tokens": 120000, "cumulative_loss_tokens": 16800000, "grad_norm": 0.408203125, "lr": 3e-05, "finish_rate": 0.66, "comp_len": 600.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.1, "frames": {"chat": 200}, "mem_gb": 10.03}
|
| 141 |
+
{"step": 141, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.17032653220028926, "tokens": 120000, "cumulative_loss_tokens": 16920000, "grad_norm": 0.45703125, "lr": 3e-05, "finish_rate": 0.714, "comp_len": 612.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 26.8, "frames": {"chat": 196}, "mem_gb": 10.01}
|
| 142 |
+
{"step": 142, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.14531434423498188, "tokens": 120000, "cumulative_loss_tokens": 17040000, "grad_norm": 0.400390625, "lr": 3e-05, "finish_rate": 0.834, "comp_len": 538.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.3, "frames": {"chat": 223}, "mem_gb": 10.0}
|
| 143 |
+
{"step": 143, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.13336851223480578, "tokens": 120000, "cumulative_loss_tokens": 17160000, "grad_norm": 0.376953125, "lr": 3e-05, "finish_rate": 0.869, "comp_len": 563.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 27.4, "frames": {"chat": 213}, "mem_gb": 9.89}
|
| 144 |
+
{"step": 144, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.14154418901971852, "tokens": 120000, "cumulative_loss_tokens": 17280000, "grad_norm": 0.427734375, "lr": 3e-05, "finish_rate": 0.879, "comp_len": 517.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.9, "frames": {"chat": 232}, "mem_gb": 9.93}
|
| 145 |
+
{"step": 145, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.15019642852085333, "tokens": 120000, "cumulative_loss_tokens": 17400000, "grad_norm": 0.447265625, "lr": 3e-05, "finish_rate": 0.861, "comp_len": 538.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.3, "frames": {"chat": 223}, "mem_gb": 9.93}
|
| 146 |
+
{"step": 146, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.15932164447531105, "tokens": 120000, "cumulative_loss_tokens": 17520000, "grad_norm": 0.421875, "lr": 3e-05, "finish_rate": 0.85, "comp_len": 515.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.7, "frames": {"chat": 233}, "mem_gb": 10.02}
|
| 147 |
+
{"step": 147, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.16258757215073952, "tokens": 120000, "cumulative_loss_tokens": 17640000, "grad_norm": 0.431640625, "lr": 3e-05, "finish_rate": 0.816, "comp_len": 553.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 27.9, "frames": {"chat": 217}, "mem_gb": 10.01}
|
| 148 |
+
{"step": 148, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.21068223652498175, "tokens": 120000, "cumulative_loss_tokens": 17760000, "grad_norm": 0.515625, "lr": 3e-05, "finish_rate": 0.752, "comp_len": 594.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.3, "frames": {"chat": 202}, "mem_gb": 10.08}
|
| 149 |
+
{"step": 149, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.14553914751367023, "tokens": 120000, "cumulative_loss_tokens": 17880000, "grad_norm": 0.4453125, "lr": 3e-05, "finish_rate": 0.858, "comp_len": 474.3, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 29.9, "frames": {"chat": 253}, "mem_gb": 9.93}
|
| 150 |
+
{"step": 150, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.13818583363940318, "tokens": 120000, "cumulative_loss_tokens": 18000000, "grad_norm": 0.396484375, "lr": 3e-05, "finish_rate": 0.879, "comp_len": 519.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 28.6, "frames": {"chat": 231}, "mem_gb": 9.94}
|
healed/grid_math/uniform_keep75_s1226/args.json
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"student": "outputs/pruned/uniform-0125inst-math-keep75",
|
| 3 |
+
"teacher": "allenai/OLMoE-1B-7B-0125-Instruct",
|
| 4 |
+
"training_mode": "off-policy",
|
| 5 |
+
"kl_direction": "forward",
|
| 6 |
+
"dataset": "allenai/RLVR-MATH",
|
| 7 |
+
"dataset_sources": null,
|
| 8 |
+
"max_difficulty": null,
|
| 9 |
+
"trajectories": "outputs/teacher_trajectories/dolci_math_curated.jsonl",
|
| 10 |
+
"trajectory_dataset": "allenai/Dolci-Instruct-RL",
|
| 11 |
+
"off_policy_frames": "chat",
|
| 12 |
+
"off_policy_max_seq_len": 2048,
|
| 13 |
+
"topk_targets": "outputs/teacher_trajectories/dolci_math_curated_opd_top128",
|
| 14 |
+
"max_loss_tokens": null,
|
| 15 |
+
"loss_tokens_per_step": 120000,
|
| 16 |
+
"teacher_device": "cuda:0",
|
| 17 |
+
"student_device": "cuda:0",
|
| 18 |
+
"lr": 3e-05,
|
| 19 |
+
"optimizer": "adamw8bit",
|
| 20 |
+
"weight_decay": 0.1,
|
| 21 |
+
"epochs": 3,
|
| 22 |
+
"prompts_per_step": 256,
|
| 23 |
+
"group_size": 1,
|
| 24 |
+
"rollout_batch": 64,
|
| 25 |
+
"micro_batch": 3,
|
| 26 |
+
"max_new_tokens": 256,
|
| 27 |
+
"max_prompt_len": 1024,
|
| 28 |
+
"warmup_steps": 10,
|
| 29 |
+
"max_grad_norm": 1.0,
|
| 30 |
+
"eval_every": 10,
|
| 31 |
+
"gsm8k_every": 0,
|
| 32 |
+
"gsm8k_n": 256,
|
| 33 |
+
"gsm8k_batch": 16,
|
| 34 |
+
"gsm8k_max_new_tokens": 512,
|
| 35 |
+
"gsm8k_frames": "chat",
|
| 36 |
+
"save_every": 50,
|
| 37 |
+
"out_dir": "outputs/healed/grid_math/uniform_keep75_s1226",
|
| 38 |
+
"sweep": 150,
|
| 39 |
+
"wandb": true,
|
| 40 |
+
"wandb_project": "glean-grid",
|
| 41 |
+
"wandb_run_name": "uniform-math-keep75-s1226",
|
| 42 |
+
"wandb_run_id": null,
|
| 43 |
+
"wandb_resume": null,
|
| 44 |
+
"wandb_mode": "online",
|
| 45 |
+
"no_wandb_sync": false,
|
| 46 |
+
"debug": false,
|
| 47 |
+
"resume_from": null,
|
| 48 |
+
"start_step": 0,
|
| 49 |
+
"no_grad_checkpointing": false,
|
| 50 |
+
"seed": 1226,
|
| 51 |
+
"no_teacher_overlap": false,
|
| 52 |
+
"sync_checkpoints": false,
|
| 53 |
+
"rollout_engine": "hf",
|
| 54 |
+
"vllm_gpu": null,
|
| 55 |
+
"vllm_port": 8377,
|
| 56 |
+
"vllm_refresh_every": 5,
|
| 57 |
+
"vllm_serve_bin": "vllm-plugin/.venv/bin/python",
|
| 58 |
+
"vllm_gpu_mem_util": 0.85,
|
| 59 |
+
"vllm_refresh_mode": "reload",
|
| 60 |
+
"vllm_live_dir": null,
|
| 61 |
+
"resolved_kl_direction": "forward"
|
| 62 |
+
}
|
healed/grid_math/uniform_keep75_s1226/train_log.jsonl
ADDED
|
@@ -0,0 +1,150 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"step": 1, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.11920493249129505, "tokens": 120000, "cumulative_loss_tokens": 120000, "grad_norm": 1.4453125, "lr": 6e-06, "finish_rate": 0.902, "comp_len": 472.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 54.8, "frames": {"chat": 254}, "mem_gb": 21.82}
|
| 2 |
+
{"step": 2, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.13180598530409238, "tokens": 120000, "cumulative_loss_tokens": 240000, "grad_norm": 1.640625, "lr": 9e-06, "finish_rate": 0.876, "comp_len": 497.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.6, "frames": {"chat": 241}, "mem_gb": 22.07}
|
| 3 |
+
{"step": 3, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.14182352375667542, "tokens": 120000, "cumulative_loss_tokens": 360000, "grad_norm": 1.40625, "lr": 1.2e-05, "finish_rate": 0.746, "comp_len": 563.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.6, "frames": {"chat": 213}, "mem_gb": 22.1}
|
| 4 |
+
{"step": 4, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.11608811415806412, "tokens": 120000, "cumulative_loss_tokens": 480000, "grad_norm": 0.94921875, "lr": 1.5e-05, "finish_rate": 0.864, "comp_len": 543.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.3, "frames": {"chat": 221}, "mem_gb": 22.15}
|
| 5 |
+
{"step": 5, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.12302177668878188, "tokens": 120000, "cumulative_loss_tokens": 600000, "grad_norm": 0.8515625, "lr": 1.8e-05, "finish_rate": 0.745, "comp_len": 612.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 42.1, "frames": {"chat": 196}, "mem_gb": 22.11}
|
| 6 |
+
{"step": 6, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.0868311198878102, "tokens": 120000, "cumulative_loss_tokens": 720000, "grad_norm": 0.75, "lr": 2.1e-05, "finish_rate": 0.926, "comp_len": 444.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 49.6, "frames": {"chat": 270}, "mem_gb": 21.91}
|
| 7 |
+
{"step": 7, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.09967558807097375, "tokens": 120000, "cumulative_loss_tokens": 840000, "grad_norm": 0.859375, "lr": 2.4e-05, "finish_rate": 0.815, "comp_len": 555.6, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.7, "frames": {"chat": 216}, "mem_gb": 22.09}
|
| 8 |
+
{"step": 8, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.09974337469547366, "tokens": 120000, "cumulative_loss_tokens": 960000, "grad_norm": 0.73828125, "lr": 2.7000000000000002e-05, "finish_rate": 0.775, "comp_len": 600.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 41.7, "frames": {"chat": 200}, "mem_gb": 22.06}
|
| 9 |
+
{"step": 9, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.08927157517972713, "tokens": 120000, "cumulative_loss_tokens": 1080000, "grad_norm": 0.64453125, "lr": 3e-05, "finish_rate": 0.767, "comp_len": 582.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 42.9, "frames": {"chat": 206}, "mem_gb": 22.01}
|
| 10 |
+
{"step": 10, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.0773138673888209, "tokens": 120000, "cumulative_loss_tokens": 1200000, "grad_norm": 0.54296875, "lr": 3e-05, "finish_rate": 0.902, "comp_len": 512.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.4, "frames": {"chat": 234}, "mem_gb": 22.04}
|
| 11 |
+
{"step": 11, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.08695185078081365, "tokens": 120000, "cumulative_loss_tokens": 1320000, "grad_norm": 0.51953125, "lr": 3e-05, "finish_rate": 0.823, "comp_len": 558.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 42.6, "frames": {"chat": 215}, "mem_gb": 22.05}
|
| 12 |
+
{"step": 12, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.07132623356043671, "tokens": 120000, "cumulative_loss_tokens": 1440000, "grad_norm": 0.455078125, "lr": 3e-05, "finish_rate": 0.922, "comp_len": 470.6, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.7, "frames": {"chat": 255}, "mem_gb": 22.04}
|
| 13 |
+
{"step": 13, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.0732923347460106, "tokens": 120000, "cumulative_loss_tokens": 1560000, "grad_norm": 0.453125, "lr": 3e-05, "finish_rate": 0.892, "comp_len": 480.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 46.8, "frames": {"chat": 250}, "mem_gb": 21.92}
|
| 14 |
+
{"step": 14, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.07680565097226451, "tokens": 120000, "cumulative_loss_tokens": 1680000, "grad_norm": 0.4296875, "lr": 3e-05, "finish_rate": 0.884, "comp_len": 495.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.1, "frames": {"chat": 242}, "mem_gb": 22.09}
|
| 15 |
+
{"step": 15, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.09355738684851676, "tokens": 120000, "cumulative_loss_tokens": 1800000, "grad_norm": 0.484375, "lr": 3e-05, "finish_rate": 0.729, "comp_len": 603.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.6, "frames": {"chat": 199}, "mem_gb": 22.1}
|
| 16 |
+
{"step": 16, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.10224050603043287, "tokens": 120000, "cumulative_loss_tokens": 1920000, "grad_norm": 0.57421875, "lr": 3e-05, "finish_rate": 0.784, "comp_len": 576.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.7, "frames": {"chat": 208}, "mem_gb": 22.13}
|
| 17 |
+
{"step": 17, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.0901082269590348, "tokens": 120000, "cumulative_loss_tokens": 2040000, "grad_norm": 0.6328125, "lr": 3e-05, "finish_rate": 0.764, "comp_len": 576.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 42.5, "frames": {"chat": 208}, "mem_gb": 22.07}
|
| 18 |
+
{"step": 18, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.09369800321385265, "tokens": 120000, "cumulative_loss_tokens": 2160000, "grad_norm": 0.46484375, "lr": 3e-05, "finish_rate": 0.732, "comp_len": 574.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.7, "frames": {"chat": 209}, "mem_gb": 22.22}
|
| 19 |
+
{"step": 19, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.06841314388358345, "tokens": 120000, "cumulative_loss_tokens": 2280000, "grad_norm": 0.380859375, "lr": 3e-05, "finish_rate": 0.855, "comp_len": 510.6, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 46.2, "frames": {"chat": 235}, "mem_gb": 22.05}
|
| 20 |
+
{"step": 20, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.07472310523393874, "tokens": 120000, "cumulative_loss_tokens": 2400000, "grad_norm": 0.396484375, "lr": 3e-05, "finish_rate": 0.74, "comp_len": 588.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 42.7, "frames": {"chat": 204}, "mem_gb": 22.04}
|
| 21 |
+
{"step": 21, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.09344625266479949, "tokens": 120000, "cumulative_loss_tokens": 2520000, "grad_norm": 0.55859375, "lr": 3e-05, "finish_rate": 0.745, "comp_len": 576.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.6, "frames": {"chat": 208}, "mem_gb": 22.1}
|
| 22 |
+
{"step": 22, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.07106327358403554, "tokens": 120000, "cumulative_loss_tokens": 2640000, "grad_norm": 0.392578125, "lr": 3e-05, "finish_rate": 0.825, "comp_len": 500.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 47.1, "frames": {"chat": 240}, "mem_gb": 22.09}
|
| 23 |
+
{"step": 23, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.06766866139359773, "tokens": 120000, "cumulative_loss_tokens": 2760000, "grad_norm": 0.40234375, "lr": 3e-05, "finish_rate": 0.89, "comp_len": 487.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 46.0, "frames": {"chat": 246}, "mem_gb": 22.09}
|
| 24 |
+
{"step": 24, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.06455388144083941, "tokens": 120000, "cumulative_loss_tokens": 2880000, "grad_norm": 0.384765625, "lr": 3e-05, "finish_rate": 0.909, "comp_len": 493.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.5, "frames": {"chat": 243}, "mem_gb": 21.91}
|
| 25 |
+
{"step": 25, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.07318340876198684, "tokens": 120000, "cumulative_loss_tokens": 3000000, "grad_norm": 0.38671875, "lr": 3e-05, "finish_rate": 0.745, "comp_len": 576.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 42.9, "frames": {"chat": 208}, "mem_gb": 22.1}
|
| 26 |
+
{"step": 26, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.07118238478129109, "tokens": 120000, "cumulative_loss_tokens": 3120000, "grad_norm": 0.36328125, "lr": 3e-05, "finish_rate": 0.817, "comp_len": 547.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.9, "frames": {"chat": 219}, "mem_gb": 22.1}
|
| 27 |
+
{"step": 27, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.0817199091798005, "tokens": 120000, "cumulative_loss_tokens": 3240000, "grad_norm": 0.431640625, "lr": 3e-05, "finish_rate": 0.782, "comp_len": 568.7, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.1, "frames": {"chat": 211}, "mem_gb": 22.11}
|
| 28 |
+
{"step": 28, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.06760180381714988, "tokens": 120000, "cumulative_loss_tokens": 3360000, "grad_norm": 0.3828125, "lr": 3e-05, "finish_rate": 0.862, "comp_len": 517.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.3, "frames": {"chat": 232}, "mem_gb": 22.07}
|
| 29 |
+
{"step": 29, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.0715414209818157, "tokens": 120000, "cumulative_loss_tokens": 3480000, "grad_norm": 0.365234375, "lr": 3e-05, "finish_rate": 0.804, "comp_len": 560.7, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.8, "frames": {"chat": 214}, "mem_gb": 22.1}
|
| 30 |
+
{"step": 30, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.06632242755762612, "tokens": 120000, "cumulative_loss_tokens": 3600000, "grad_norm": 0.3359375, "lr": 3e-05, "finish_rate": 0.819, "comp_len": 531.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.8, "frames": {"chat": 226}, "mem_gb": 21.99}
|
| 31 |
+
{"step": 31, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.06462202967116609, "tokens": 120000, "cumulative_loss_tokens": 3720000, "grad_norm": 0.33984375, "lr": 3e-05, "finish_rate": 0.819, "comp_len": 571.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.8, "frames": {"chat": 210}, "mem_gb": 22.11}
|
| 32 |
+
{"step": 32, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.06298202505940571, "tokens": 120000, "cumulative_loss_tokens": 3840000, "grad_norm": 0.404296875, "lr": 3e-05, "finish_rate": 0.858, "comp_len": 550.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.0, "frames": {"chat": 218}, "mem_gb": 21.93}
|
| 33 |
+
{"step": 33, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.05829383838605136, "tokens": 120000, "cumulative_loss_tokens": 3960000, "grad_norm": 0.330078125, "lr": 3e-05, "finish_rate": 0.858, "comp_len": 515.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 46.0, "frames": {"chat": 233}, "mem_gb": 22.08}
|
| 34 |
+
{"step": 34, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.07446961007329325, "tokens": 120000, "cumulative_loss_tokens": 4080000, "grad_norm": 0.388671875, "lr": 3e-05, "finish_rate": 0.786, "comp_len": 558.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 46.2, "frames": {"chat": 215}, "mem_gb": 22.1}
|
| 35 |
+
{"step": 35, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.06408025780933288, "tokens": 120000, "cumulative_loss_tokens": 4200000, "grad_norm": 0.341796875, "lr": 3e-05, "finish_rate": 0.845, "comp_len": 515.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.3, "frames": {"chat": 233}, "mem_gb": 22.08}
|
| 36 |
+
{"step": 36, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.06080306692530091, "tokens": 120000, "cumulative_loss_tokens": 4320000, "grad_norm": 0.353515625, "lr": 3e-05, "finish_rate": 0.766, "comp_len": 574.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.4, "frames": {"chat": 209}, "mem_gb": 22.04}
|
| 37 |
+
{"step": 37, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.05366904664818818, "tokens": 120000, "cumulative_loss_tokens": 4440000, "grad_norm": 0.310546875, "lr": 3e-05, "finish_rate": 0.908, "comp_len": 458.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 47.9, "frames": {"chat": 262}, "mem_gb": 21.97}
|
| 38 |
+
{"step": 38, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.05671166869318113, "tokens": 120000, "cumulative_loss_tokens": 4560000, "grad_norm": 0.326171875, "lr": 3e-05, "finish_rate": 0.9, "comp_len": 481.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 46.7, "frames": {"chat": 249}, "mem_gb": 22.06}
|
| 39 |
+
{"step": 39, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.07106226710767174, "tokens": 120000, "cumulative_loss_tokens": 4680000, "grad_norm": 0.3671875, "lr": 3e-05, "finish_rate": 0.819, "comp_len": 528.6, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.6, "frames": {"chat": 227}, "mem_gb": 22.09}
|
| 40 |
+
{"step": 40, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.05462301931890349, "tokens": 120000, "cumulative_loss_tokens": 4800000, "grad_norm": 0.33984375, "lr": 3e-05, "finish_rate": 0.814, "comp_len": 543.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.3, "frames": {"chat": 221}, "mem_gb": 22.09}
|
| 41 |
+
{"step": 41, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.0582238324320099, "tokens": 120000, "cumulative_loss_tokens": 4920000, "grad_norm": 0.333984375, "lr": 3e-05, "finish_rate": 0.859, "comp_len": 512.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.4, "frames": {"chat": 234}, "mem_gb": 22.1}
|
| 42 |
+
{"step": 42, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.05399296633934913, "tokens": 120000, "cumulative_loss_tokens": 5040000, "grad_norm": 0.330078125, "lr": 3e-05, "finish_rate": 0.817, "comp_len": 563.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 42.6, "frames": {"chat": 213}, "mem_gb": 22.05}
|
| 43 |
+
{"step": 43, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.05386831222480784, "tokens": 120000, "cumulative_loss_tokens": 5160000, "grad_norm": 0.298828125, "lr": 3e-05, "finish_rate": 0.836, "comp_len": 563.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 42.8, "frames": {"chat": 213}, "mem_gb": 21.99}
|
| 44 |
+
{"step": 44, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.05173095208670323, "tokens": 120000, "cumulative_loss_tokens": 5280000, "grad_norm": 0.314453125, "lr": 3e-05, "finish_rate": 0.906, "comp_len": 512.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.9, "frames": {"chat": 234}, "mem_gb": 22.02}
|
| 45 |
+
{"step": 45, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.05363421202797132, "tokens": 120000, "cumulative_loss_tokens": 5400000, "grad_norm": 0.322265625, "lr": 3e-05, "finish_rate": 0.793, "comp_len": 540.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.3, "frames": {"chat": 222}, "mem_gb": 22.09}
|
| 46 |
+
{"step": 46, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.0622261526023969, "tokens": 120000, "cumulative_loss_tokens": 5520000, "grad_norm": 0.361328125, "lr": 3e-05, "finish_rate": 0.806, "comp_len": 528.6, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 48.7, "frames": {"chat": 227}, "mem_gb": 22.1}
|
| 47 |
+
{"step": 47, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.05994350822979274, "tokens": 120000, "cumulative_loss_tokens": 5640000, "grad_norm": 0.32421875, "lr": 3e-05, "finish_rate": 0.835, "comp_len": 550.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.1, "frames": {"chat": 218}, "mem_gb": 22.14}
|
| 48 |
+
{"step": 48, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.06174757863963023, "tokens": 120000, "cumulative_loss_tokens": 5760000, "grad_norm": 0.341796875, "lr": 3e-05, "finish_rate": 0.798, "comp_len": 538.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.9, "frames": {"chat": 223}, "mem_gb": 22.1}
|
| 49 |
+
{"step": 49, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.06565356359497333, "tokens": 120000, "cumulative_loss_tokens": 5880000, "grad_norm": 0.359375, "lr": 3e-05, "finish_rate": 0.772, "comp_len": 582.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.4, "frames": {"chat": 206}, "mem_gb": 22.1}
|
| 50 |
+
{"step": 50, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.05670059742469651, "tokens": 120000, "cumulative_loss_tokens": 6000000, "grad_norm": 0.33984375, "lr": 3e-05, "finish_rate": 0.784, "comp_len": 563.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.7, "frames": {"chat": 213}, "mem_gb": 22.02}
|
| 51 |
+
{"step": 51, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.07032583207286273, "tokens": 120000, "cumulative_loss_tokens": 6120000, "grad_norm": 0.357421875, "lr": 3e-05, "finish_rate": 0.843, "comp_len": 538.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 46.2, "frames": {"chat": 223}, "mem_gb": 21.96}
|
| 52 |
+
{"step": 52, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.059393302978233746, "tokens": 120000, "cumulative_loss_tokens": 6240000, "grad_norm": 0.333984375, "lr": 3e-05, "finish_rate": 0.828, "comp_len": 528.6, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.3, "frames": {"chat": 227}, "mem_gb": 22.07}
|
| 53 |
+
{"step": 53, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.05184000949463807, "tokens": 120000, "cumulative_loss_tokens": 6360000, "grad_norm": 0.32421875, "lr": 3e-05, "finish_rate": 0.889, "comp_len": 474.3, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 47.2, "frames": {"chat": 253}, "mem_gb": 22.09}
|
| 54 |
+
{"step": 54, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.038828376208990815, "tokens": 120000, "cumulative_loss_tokens": 6480000, "grad_norm": 0.259765625, "lr": 3e-05, "finish_rate": 0.792, "comp_len": 555.6, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.3, "frames": {"chat": 216}, "mem_gb": 22.1}
|
| 55 |
+
{"step": 55, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.039238105912537624, "tokens": 120000, "cumulative_loss_tokens": 6600000, "grad_norm": 0.255859375, "lr": 3e-05, "finish_rate": 0.766, "comp_len": 585.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.0, "frames": {"chat": 205}, "mem_gb": 22.07}
|
| 56 |
+
{"step": 56, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.047572107557269434, "tokens": 120000, "cumulative_loss_tokens": 6720000, "grad_norm": 0.2890625, "lr": 3e-05, "finish_rate": 0.729, "comp_len": 579.7, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 46.7, "frames": {"chat": 207}, "mem_gb": 22.16}
|
| 57 |
+
{"step": 57, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.04661659479457885, "tokens": 120000, "cumulative_loss_tokens": 6840000, "grad_norm": 0.28125, "lr": 3e-05, "finish_rate": 0.814, "comp_len": 558.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.9, "frames": {"chat": 215}, "mem_gb": 22.08}
|
| 58 |
+
{"step": 58, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.034201694558095186, "tokens": 120000, "cumulative_loss_tokens": 6960000, "grad_norm": 0.27734375, "lr": 3e-05, "finish_rate": 0.86, "comp_len": 526.3, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.8, "frames": {"chat": 228}, "mem_gb": 22.1}
|
| 59 |
+
{"step": 59, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.04092416759391005, "tokens": 120000, "cumulative_loss_tokens": 7080000, "grad_norm": 0.267578125, "lr": 3e-05, "finish_rate": 0.747, "comp_len": 543.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 46.4, "frames": {"chat": 221}, "mem_gb": 22.14}
|
| 60 |
+
{"step": 60, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.029803095571851977, "tokens": 120000, "cumulative_loss_tokens": 7200000, "grad_norm": 0.2138671875, "lr": 3e-05, "finish_rate": 0.882, "comp_len": 472.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 46.7, "frames": {"chat": 254}, "mem_gb": 21.93}
|
| 61 |
+
{"step": 61, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.03530521812000467, "tokens": 120000, "cumulative_loss_tokens": 7320000, "grad_norm": 0.28515625, "lr": 3e-05, "finish_rate": 0.843, "comp_len": 571.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.3, "frames": {"chat": 210}, "mem_gb": 22.06}
|
| 62 |
+
{"step": 62, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.033094389010050025, "tokens": 120000, "cumulative_loss_tokens": 7440000, "grad_norm": 0.21484375, "lr": 3e-05, "finish_rate": 0.827, "comp_len": 531.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.1, "frames": {"chat": 226}, "mem_gb": 22.02}
|
| 63 |
+
{"step": 63, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.03637227460097832, "tokens": 120000, "cumulative_loss_tokens": 7560000, "grad_norm": 0.26953125, "lr": 3e-05, "finish_rate": 0.802, "comp_len": 566.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.8, "frames": {"chat": 212}, "mem_gb": 22.09}
|
| 64 |
+
{"step": 64, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.041987014663917945, "tokens": 120000, "cumulative_loss_tokens": 7680000, "grad_norm": 0.265625, "lr": 3e-05, "finish_rate": 0.754, "comp_len": 568.7, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.6, "frames": {"chat": 211}, "mem_gb": 22.02}
|
| 65 |
+
{"step": 65, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.03841204237483131, "tokens": 120000, "cumulative_loss_tokens": 7800000, "grad_norm": 0.26953125, "lr": 3e-05, "finish_rate": 0.776, "comp_len": 612.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 41.3, "frames": {"chat": 196}, "mem_gb": 22.07}
|
| 66 |
+
{"step": 66, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.03594996488027585, "tokens": 120000, "cumulative_loss_tokens": 7920000, "grad_norm": 0.267578125, "lr": 3e-05, "finish_rate": 0.811, "comp_len": 566.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.4, "frames": {"chat": 212}, "mem_gb": 22.09}
|
| 67 |
+
{"step": 67, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.032274080974639706, "tokens": 120000, "cumulative_loss_tokens": 8040000, "grad_norm": 0.2177734375, "lr": 3e-05, "finish_rate": 0.877, "comp_len": 491.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 46.2, "frames": {"chat": 244}, "mem_gb": 22.0}
|
| 68 |
+
{"step": 68, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.03448318504060929, "tokens": 120000, "cumulative_loss_tokens": 8160000, "grad_norm": 0.2392578125, "lr": 3e-05, "finish_rate": 0.838, "comp_len": 540.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.2, "frames": {"chat": 222}, "mem_gb": 22.05}
|
| 69 |
+
{"step": 69, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.035316669923362014, "tokens": 120000, "cumulative_loss_tokens": 8280000, "grad_norm": 0.232421875, "lr": 3e-05, "finish_rate": 0.78, "comp_len": 550.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.8, "frames": {"chat": 218}, "mem_gb": 22.09}
|
| 70 |
+
{"step": 70, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.0348655257951313, "tokens": 120000, "cumulative_loss_tokens": 8400000, "grad_norm": 0.2353515625, "lr": 3e-05, "finish_rate": 0.913, "comp_len": 476.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 48.0, "frames": {"chat": 252}, "mem_gb": 21.97}
|
| 71 |
+
{"step": 71, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.03988810336939059, "tokens": 120000, "cumulative_loss_tokens": 8520000, "grad_norm": 0.26171875, "lr": 3e-05, "finish_rate": 0.728, "comp_len": 594.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 42.8, "frames": {"chat": 202}, "mem_gb": 22.14}
|
| 72 |
+
{"step": 72, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.04227977843079716, "tokens": 120000, "cumulative_loss_tokens": 8640000, "grad_norm": 0.29296875, "lr": 3e-05, "finish_rate": 0.835, "comp_len": 506.3, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 46.8, "frames": {"chat": 237}, "mem_gb": 22.1}
|
| 73 |
+
{"step": 73, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.03656210743497747, "tokens": 120000, "cumulative_loss_tokens": 8760000, "grad_norm": 0.234375, "lr": 3e-05, "finish_rate": 0.868, "comp_len": 512.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.7, "frames": {"chat": 234}, "mem_gb": 22.08}
|
| 74 |
+
{"step": 74, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.03134735133058081, "tokens": 120000, "cumulative_loss_tokens": 8880000, "grad_norm": 0.25, "lr": 3e-05, "finish_rate": 0.809, "comp_len": 558.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.1, "frames": {"chat": 215}, "mem_gb": 22.1}
|
| 75 |
+
{"step": 75, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.03046899615646495, "tokens": 120000, "cumulative_loss_tokens": 9000000, "grad_norm": 0.216796875, "lr": 3e-05, "finish_rate": 0.876, "comp_len": 512.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.2, "frames": {"chat": 234}, "mem_gb": 22.03}
|
| 76 |
+
{"step": 76, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.029838484331507546, "tokens": 120000, "cumulative_loss_tokens": 9120000, "grad_norm": 0.2099609375, "lr": 3e-05, "finish_rate": 0.801, "comp_len": 555.6, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.1, "frames": {"chat": 216}, "mem_gb": 22.08}
|
| 77 |
+
{"step": 77, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.03392047594260269, "tokens": 120000, "cumulative_loss_tokens": 9240000, "grad_norm": 0.248046875, "lr": 3e-05, "finish_rate": 0.805, "comp_len": 571.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 42.6, "frames": {"chat": 210}, "mem_gb": 22.05}
|
| 78 |
+
{"step": 78, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.03507764483672411, "tokens": 120000, "cumulative_loss_tokens": 9360000, "grad_norm": 0.25, "lr": 3e-05, "finish_rate": 0.719, "comp_len": 603.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 42.5, "frames": {"chat": 199}, "mem_gb": 22.1}
|
| 79 |
+
{"step": 79, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.03239439557224202, "tokens": 120000, "cumulative_loss_tokens": 9480000, "grad_norm": 0.2216796875, "lr": 3e-05, "finish_rate": 0.824, "comp_len": 571.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.4, "frames": {"chat": 210}, "mem_gb": 22.11}
|
| 80 |
+
{"step": 80, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.03286070772272845, "tokens": 120000, "cumulative_loss_tokens": 9600000, "grad_norm": 0.265625, "lr": 3e-05, "finish_rate": 0.902, "comp_len": 533.3, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.0, "frames": {"chat": 225}, "mem_gb": 22.05}
|
| 81 |
+
{"step": 81, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.032128432613688834, "tokens": 120000, "cumulative_loss_tokens": 9720000, "grad_norm": 0.25, "lr": 3e-05, "finish_rate": 0.913, "comp_len": 474.3, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 46.7, "frames": {"chat": 253}, "mem_gb": 21.95}
|
| 82 |
+
{"step": 82, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.031100997417598652, "tokens": 120000, "cumulative_loss_tokens": 9840000, "grad_norm": 0.21875, "lr": 3e-05, "finish_rate": 0.903, "comp_len": 485.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 47.2, "frames": {"chat": 247}, "mem_gb": 22.07}
|
| 83 |
+
{"step": 83, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.0324900306135416, "tokens": 120000, "cumulative_loss_tokens": 9960000, "grad_norm": 0.2412109375, "lr": 3e-05, "finish_rate": 0.836, "comp_len": 504.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.5, "frames": {"chat": 238}, "mem_gb": 22.07}
|
| 84 |
+
{"step": 84, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.03485131102278829, "tokens": 120000, "cumulative_loss_tokens": 10080000, "grad_norm": 0.2451171875, "lr": 3e-05, "finish_rate": 0.86, "comp_len": 510.6, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.7, "frames": {"chat": 235}, "mem_gb": 22.09}
|
| 85 |
+
{"step": 85, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.03912405168330297, "tokens": 120000, "cumulative_loss_tokens": 10200000, "grad_norm": 0.265625, "lr": 3e-05, "finish_rate": 0.805, "comp_len": 558.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.4, "frames": {"chat": 215}, "mem_gb": 22.06}
|
| 86 |
+
{"step": 86, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.033098834516915185, "tokens": 120000, "cumulative_loss_tokens": 10320000, "grad_norm": 0.28125, "lr": 3e-05, "finish_rate": 0.925, "comp_len": 447.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 47.1, "frames": {"chat": 268}, "mem_gb": 22.06}
|
| 87 |
+
{"step": 87, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.034645479888437934, "tokens": 120000, "cumulative_loss_tokens": 10440000, "grad_norm": 0.232421875, "lr": 3e-05, "finish_rate": 0.825, "comp_len": 526.3, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.3, "frames": {"chat": 228}, "mem_gb": 22.1}
|
| 88 |
+
{"step": 88, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.03237436599015103, "tokens": 120000, "cumulative_loss_tokens": 10560000, "grad_norm": 0.240234375, "lr": 3e-05, "finish_rate": 0.881, "comp_len": 476.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 46.7, "frames": {"chat": 252}, "mem_gb": 22.03}
|
| 89 |
+
{"step": 89, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.03435483168452047, "tokens": 120000, "cumulative_loss_tokens": 10680000, "grad_norm": 0.2734375, "lr": 3e-05, "finish_rate": 0.821, "comp_len": 538.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.3, "frames": {"chat": 223}, "mem_gb": 22.11}
|
| 90 |
+
{"step": 90, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.04248241032837735, "tokens": 120000, "cumulative_loss_tokens": 10800000, "grad_norm": 0.294921875, "lr": 3e-05, "finish_rate": 0.805, "comp_len": 531.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.6, "frames": {"chat": 226}, "mem_gb": 22.09}
|
| 91 |
+
{"step": 91, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.04142716025705449, "tokens": 120000, "cumulative_loss_tokens": 10920000, "grad_norm": 0.283203125, "lr": 3e-05, "finish_rate": 0.731, "comp_len": 576.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 47.0, "frames": {"chat": 208}, "mem_gb": 22.14}
|
| 92 |
+
{"step": 92, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.031166394373402, "tokens": 120000, "cumulative_loss_tokens": 11040000, "grad_norm": 0.251953125, "lr": 3e-05, "finish_rate": 0.883, "comp_len": 500.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.8, "frames": {"chat": 240}, "mem_gb": 22.03}
|
| 93 |
+
{"step": 93, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.03443516800537861, "tokens": 120000, "cumulative_loss_tokens": 11160000, "grad_norm": 0.23046875, "lr": 3e-05, "finish_rate": 0.842, "comp_len": 540.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.9, "frames": {"chat": 222}, "mem_gb": 22.02}
|
| 94 |
+
{"step": 94, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.030224586171346407, "tokens": 120000, "cumulative_loss_tokens": 11280000, "grad_norm": 0.2333984375, "lr": 3e-05, "finish_rate": 0.881, "comp_len": 508.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.7, "frames": {"chat": 236}, "mem_gb": 22.09}
|
| 95 |
+
{"step": 95, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.03236451368537576, "tokens": 120000, "cumulative_loss_tokens": 11400000, "grad_norm": 0.2353515625, "lr": 3e-05, "finish_rate": 0.834, "comp_len": 553.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.5, "frames": {"chat": 217}, "mem_gb": 22.06}
|
| 96 |
+
{"step": 96, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.034898156173845445, "tokens": 120000, "cumulative_loss_tokens": 11520000, "grad_norm": 0.263671875, "lr": 3e-05, "finish_rate": 0.921, "comp_len": 476.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 46.3, "frames": {"chat": 252}, "mem_gb": 21.97}
|
| 97 |
+
{"step": 97, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.02913232143558562, "tokens": 120000, "cumulative_loss_tokens": 11640000, "grad_norm": 0.2255859375, "lr": 3e-05, "finish_rate": 0.847, "comp_len": 540.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.8, "frames": {"chat": 222}, "mem_gb": 22.08}
|
| 98 |
+
{"step": 98, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.029040355308757475, "tokens": 120000, "cumulative_loss_tokens": 11760000, "grad_norm": 0.21484375, "lr": 3e-05, "finish_rate": 0.901, "comp_len": 495.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.6, "frames": {"chat": 242}, "mem_gb": 21.97}
|
| 99 |
+
{"step": 99, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.041233757554925976, "tokens": 120000, "cumulative_loss_tokens": 11880000, "grad_norm": 0.2734375, "lr": 3e-05, "finish_rate": 0.804, "comp_len": 547.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 46.1, "frames": {"chat": 219}, "mem_gb": 22.03}
|
| 100 |
+
{"step": 100, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.031569313709802614, "tokens": 120000, "cumulative_loss_tokens": 12000000, "grad_norm": 0.2138671875, "lr": 3e-05, "finish_rate": 0.879, "comp_len": 538.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.6, "frames": {"chat": 223}, "mem_gb": 22.04}
|
| 101 |
+
{"step": 101, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.03175070390204589, "tokens": 120000, "cumulative_loss_tokens": 12120000, "grad_norm": 0.228515625, "lr": 3e-05, "finish_rate": 0.858, "comp_len": 517.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.5, "frames": {"chat": 232}, "mem_gb": 22.04}
|
| 102 |
+
{"step": 102, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.03924315899686578, "tokens": 120000, "cumulative_loss_tokens": 12240000, "grad_norm": 0.28125, "lr": 3e-05, "finish_rate": 0.832, "comp_len": 545.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.3, "frames": {"chat": 220}, "mem_gb": 22.09}
|
| 103 |
+
{"step": 103, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.03732912681673964, "tokens": 120000, "cumulative_loss_tokens": 12360000, "grad_norm": 0.236328125, "lr": 3e-05, "finish_rate": 0.776, "comp_len": 571.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.5, "frames": {"chat": 210}, "mem_gb": 22.14}
|
| 104 |
+
{"step": 104, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.03158854953125119, "tokens": 120000, "cumulative_loss_tokens": 12480000, "grad_norm": 0.205078125, "lr": 3e-05, "finish_rate": 0.81, "comp_len": 531.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 46.0, "frames": {"chat": 226}, "mem_gb": 22.06}
|
| 105 |
+
{"step": 105, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.033342002084447694, "tokens": 120000, "cumulative_loss_tokens": 12600000, "grad_norm": 0.259765625, "lr": 3e-05, "finish_rate": 0.741, "comp_len": 566.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.2, "frames": {"chat": 212}, "mem_gb": 22.09}
|
| 106 |
+
{"step": 106, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.030606075126398354, "tokens": 120000, "cumulative_loss_tokens": 12720000, "grad_norm": 0.232421875, "lr": 3e-05, "finish_rate": 0.839, "comp_len": 508.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.3, "frames": {"chat": 236}, "mem_gb": 22.1}
|
| 107 |
+
{"step": 107, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.021876435576969135, "tokens": 120000, "cumulative_loss_tokens": 12840000, "grad_norm": 0.2001953125, "lr": 3e-05, "finish_rate": 0.928, "comp_len": 454.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 46.7, "frames": {"chat": 264}, "mem_gb": 21.97}
|
| 108 |
+
{"step": 108, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.031198739765095525, "tokens": 120000, "cumulative_loss_tokens": 12960000, "grad_norm": 0.21875, "lr": 3e-05, "finish_rate": 0.834, "comp_len": 524.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.5, "frames": {"chat": 229}, "mem_gb": 22.07}
|
| 109 |
+
{"step": 109, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.02165067300312221, "tokens": 120000, "cumulative_loss_tokens": 13080000, "grad_norm": 0.1806640625, "lr": 3e-05, "finish_rate": 0.903, "comp_len": 465.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 46.9, "frames": {"chat": 258}, "mem_gb": 21.95}
|
| 110 |
+
{"step": 110, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.028640677657909692, "tokens": 120000, "cumulative_loss_tokens": 13200000, "grad_norm": 0.193359375, "lr": 3e-05, "finish_rate": 0.755, "comp_len": 576.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.1, "frames": {"chat": 208}, "mem_gb": 22.11}
|
| 111 |
+
{"step": 111, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.022872452891978902, "tokens": 120000, "cumulative_loss_tokens": 13320000, "grad_norm": 0.18359375, "lr": 3e-05, "finish_rate": 0.88, "comp_len": 481.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 46.0, "frames": {"chat": 249}, "mem_gb": 22.02}
|
| 112 |
+
{"step": 112, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.02100934052404482, "tokens": 120000, "cumulative_loss_tokens": 13440000, "grad_norm": 0.19140625, "lr": 3e-05, "finish_rate": 0.845, "comp_len": 545.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 42.8, "frames": {"chat": 220}, "mem_gb": 22.09}
|
| 113 |
+
{"step": 113, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.024147335090193275, "tokens": 120000, "cumulative_loss_tokens": 13560000, "grad_norm": 0.1806640625, "lr": 3e-05, "finish_rate": 0.834, "comp_len": 538.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.2, "frames": {"chat": 223}, "mem_gb": 22.08}
|
| 114 |
+
{"step": 114, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.024052943600167055, "tokens": 120000, "cumulative_loss_tokens": 13680000, "grad_norm": 0.201171875, "lr": 3e-05, "finish_rate": 0.833, "comp_len": 543.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.7, "frames": {"chat": 221}, "mem_gb": 22.09}
|
| 115 |
+
{"step": 115, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.02166470361133882, "tokens": 120000, "cumulative_loss_tokens": 13800000, "grad_norm": 0.162109375, "lr": 3e-05, "finish_rate": 0.9, "comp_len": 521.7, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.5, "frames": {"chat": 230}, "mem_gb": 22.0}
|
| 116 |
+
{"step": 116, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.02461200471681077, "tokens": 120000, "cumulative_loss_tokens": 13920000, "grad_norm": 0.171875, "lr": 3e-05, "finish_rate": 0.776, "comp_len": 560.7, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.2, "frames": {"chat": 214}, "mem_gb": 22.07}
|
| 117 |
+
{"step": 117, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.03192179344429945, "tokens": 120000, "cumulative_loss_tokens": 14040000, "grad_norm": 0.224609375, "lr": 3e-05, "finish_rate": 0.766, "comp_len": 560.7, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 46.2, "frames": {"chat": 214}, "mem_gb": 22.08}
|
| 118 |
+
{"step": 118, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.027875469126327275, "tokens": 120000, "cumulative_loss_tokens": 14160000, "grad_norm": 0.2041015625, "lr": 3e-05, "finish_rate": 0.786, "comp_len": 571.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.4, "frames": {"chat": 210}, "mem_gb": 22.13}
|
| 119 |
+
{"step": 119, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.028064899455732668, "tokens": 120000, "cumulative_loss_tokens": 14280000, "grad_norm": 0.1845703125, "lr": 3e-05, "finish_rate": 0.776, "comp_len": 560.7, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.7, "frames": {"chat": 214}, "mem_gb": 22.09}
|
| 120 |
+
{"step": 120, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.02598108633084533, "tokens": 120000, "cumulative_loss_tokens": 14400000, "grad_norm": 0.185546875, "lr": 3e-05, "finish_rate": 0.791, "comp_len": 558.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 51.0, "frames": {"chat": 215}, "mem_gb": 22.05}
|
| 121 |
+
{"step": 121, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.029542491081776097, "tokens": 120000, "cumulative_loss_tokens": 14520000, "grad_norm": 0.2060546875, "lr": 3e-05, "finish_rate": 0.721, "comp_len": 576.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 49.9, "frames": {"chat": 208}, "mem_gb": 22.09}
|
| 122 |
+
{"step": 122, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.023529452410464485, "tokens": 120000, "cumulative_loss_tokens": 14640000, "grad_norm": 0.201171875, "lr": 3e-05, "finish_rate": 0.789, "comp_len": 550.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 49.8, "frames": {"chat": 218}, "mem_gb": 21.97}
|
| 123 |
+
{"step": 123, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.021247040836008577, "tokens": 120000, "cumulative_loss_tokens": 14760000, "grad_norm": 0.16015625, "lr": 3e-05, "finish_rate": 0.876, "comp_len": 515.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 49.4, "frames": {"chat": 233}, "mem_gb": 21.99}
|
| 124 |
+
{"step": 124, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.02059493543021381, "tokens": 120000, "cumulative_loss_tokens": 14880000, "grad_norm": 0.173828125, "lr": 3e-05, "finish_rate": 0.861, "comp_len": 519.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 50.3, "frames": {"chat": 231}, "mem_gb": 22.03}
|
| 125 |
+
{"step": 125, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.024373427555907982, "tokens": 120000, "cumulative_loss_tokens": 15000000, "grad_norm": 0.1845703125, "lr": 3e-05, "finish_rate": 0.868, "comp_len": 510.6, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 52.0, "frames": {"chat": 235}, "mem_gb": 22.22}
|
| 126 |
+
{"step": 126, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.02307887886830916, "tokens": 120000, "cumulative_loss_tokens": 15120000, "grad_norm": 0.1650390625, "lr": 3e-05, "finish_rate": 0.843, "comp_len": 555.6, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 48.3, "frames": {"chat": 216}, "mem_gb": 22.08}
|
| 127 |
+
{"step": 127, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.02126138604179335, "tokens": 120000, "cumulative_loss_tokens": 15240000, "grad_norm": 0.162109375, "lr": 3e-05, "finish_rate": 0.831, "comp_len": 506.3, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 50.3, "frames": {"chat": 237}, "mem_gb": 22.1}
|
| 128 |
+
{"step": 128, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.029153408732265233, "tokens": 120000, "cumulative_loss_tokens": 15360000, "grad_norm": 0.2001953125, "lr": 3e-05, "finish_rate": 0.734, "comp_len": 591.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 49.1, "frames": {"chat": 203}, "mem_gb": 22.11}
|
| 129 |
+
{"step": 129, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.02260789417422687, "tokens": 120000, "cumulative_loss_tokens": 15480000, "grad_norm": 0.17578125, "lr": 3e-05, "finish_rate": 0.873, "comp_len": 508.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 50.6, "frames": {"chat": 236}, "mem_gb": 22.13}
|
| 130 |
+
{"step": 130, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.022835769553498055, "tokens": 120000, "cumulative_loss_tokens": 15600000, "grad_norm": 0.1728515625, "lr": 3e-05, "finish_rate": 0.734, "comp_len": 560.7, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 48.0, "frames": {"chat": 214}, "mem_gb": 22.1}
|
| 131 |
+
{"step": 131, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.026547468842783323, "tokens": 120000, "cumulative_loss_tokens": 15720000, "grad_norm": 0.189453125, "lr": 3e-05, "finish_rate": 0.78, "comp_len": 574.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 49.3, "frames": {"chat": 209}, "mem_gb": 22.09}
|
| 132 |
+
{"step": 132, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.022498934920489166, "tokens": 120000, "cumulative_loss_tokens": 15840000, "grad_norm": 0.1826171875, "lr": 3e-05, "finish_rate": 0.906, "comp_len": 468.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 52.3, "frames": {"chat": 256}, "mem_gb": 22.1}
|
| 133 |
+
{"step": 133, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.021946324599829193, "tokens": 120000, "cumulative_loss_tokens": 15960000, "grad_norm": 0.1787109375, "lr": 3e-05, "finish_rate": 0.878, "comp_len": 521.7, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 49.6, "frames": {"chat": 230}, "mem_gb": 21.96}
|
| 134 |
+
{"step": 134, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.024039489145313078, "tokens": 120000, "cumulative_loss_tokens": 16080000, "grad_norm": 0.18359375, "lr": 3e-05, "finish_rate": 0.822, "comp_len": 521.7, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 49.9, "frames": {"chat": 230}, "mem_gb": 22.15}
|
| 135 |
+
{"step": 135, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.02589866460277699, "tokens": 120000, "cumulative_loss_tokens": 16200000, "grad_norm": 0.1796875, "lr": 3e-05, "finish_rate": 0.881, "comp_len": 528.6, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 48.8, "frames": {"chat": 227}, "mem_gb": 22.05}
|
| 136 |
+
{"step": 136, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.023664263719754913, "tokens": 120000, "cumulative_loss_tokens": 16320000, "grad_norm": 0.166015625, "lr": 3e-05, "finish_rate": 0.755, "comp_len": 576.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 47.4, "frames": {"chat": 208}, "mem_gb": 22.11}
|
| 137 |
+
{"step": 137, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.027114476467870796, "tokens": 120000, "cumulative_loss_tokens": 16440000, "grad_norm": 0.1796875, "lr": 3e-05, "finish_rate": 0.699, "comp_len": 582.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 49.0, "frames": {"chat": 206}, "mem_gb": 22.12}
|
| 138 |
+
{"step": 138, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.02399630657126351, "tokens": 120000, "cumulative_loss_tokens": 16560000, "grad_norm": 0.1767578125, "lr": 3e-05, "finish_rate": 0.82, "comp_len": 526.3, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 48.7, "frames": {"chat": 228}, "mem_gb": 22.0}
|
| 139 |
+
{"step": 139, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.02343961157395194, "tokens": 120000, "cumulative_loss_tokens": 16680000, "grad_norm": 0.181640625, "lr": 3e-05, "finish_rate": 0.835, "comp_len": 535.7, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 48.9, "frames": {"chat": 224}, "mem_gb": 22.09}
|
| 140 |
+
{"step": 140, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.024890781568270178, "tokens": 120000, "cumulative_loss_tokens": 16800000, "grad_norm": 0.1962890625, "lr": 3e-05, "finish_rate": 0.66, "comp_len": 600.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 48.1, "frames": {"chat": 200}, "mem_gb": 22.13}
|
| 141 |
+
{"step": 141, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.02478467862996428, "tokens": 120000, "cumulative_loss_tokens": 16920000, "grad_norm": 0.1904296875, "lr": 3e-05, "finish_rate": 0.714, "comp_len": 612.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 47.1, "frames": {"chat": 196}, "mem_gb": 22.11}
|
| 142 |
+
{"step": 142, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.022272795487005108, "tokens": 120000, "cumulative_loss_tokens": 17040000, "grad_norm": 0.169921875, "lr": 3e-05, "finish_rate": 0.834, "comp_len": 538.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 49.4, "frames": {"chat": 223}, "mem_gb": 22.09}
|
| 143 |
+
{"step": 143, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.022915525561606045, "tokens": 120000, "cumulative_loss_tokens": 17160000, "grad_norm": 0.18359375, "lr": 3e-05, "finish_rate": 0.869, "comp_len": 563.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 47.0, "frames": {"chat": 213}, "mem_gb": 21.98}
|
| 144 |
+
{"step": 144, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.0212597823954653, "tokens": 120000, "cumulative_loss_tokens": 17280000, "grad_norm": 0.1953125, "lr": 3e-05, "finish_rate": 0.879, "comp_len": 517.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 49.4, "frames": {"chat": 232}, "mem_gb": 22.02}
|
| 145 |
+
{"step": 145, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.020988663152477237, "tokens": 120000, "cumulative_loss_tokens": 17400000, "grad_norm": 0.1796875, "lr": 3e-05, "finish_rate": 0.861, "comp_len": 538.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 50.2, "frames": {"chat": 223}, "mem_gb": 22.02}
|
| 146 |
+
{"step": 146, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.021486785067555806, "tokens": 120000, "cumulative_loss_tokens": 17520000, "grad_norm": 0.1630859375, "lr": 3e-05, "finish_rate": 0.85, "comp_len": 515.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 49.0, "frames": {"chat": 233}, "mem_gb": 22.11}
|
| 147 |
+
{"step": 147, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.02550031017503546, "tokens": 120000, "cumulative_loss_tokens": 17640000, "grad_norm": 0.2021484375, "lr": 3e-05, "finish_rate": 0.816, "comp_len": 553.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 47.8, "frames": {"chat": 217}, "mem_gb": 22.11}
|
| 148 |
+
{"step": 148, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.030582861898102175, "tokens": 120000, "cumulative_loss_tokens": 17760000, "grad_norm": 0.1962890625, "lr": 3e-05, "finish_rate": 0.752, "comp_len": 594.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 48.1, "frames": {"chat": 202}, "mem_gb": 22.17}
|
| 149 |
+
{"step": 149, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.021075663964031262, "tokens": 120000, "cumulative_loss_tokens": 17880000, "grad_norm": 0.1640625, "lr": 3e-05, "finish_rate": 0.858, "comp_len": 474.3, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 51.1, "frames": {"chat": 253}, "mem_gb": 22.03}
|
| 150 |
+
{"step": 150, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.02149440993195555, "tokens": 120000, "cumulative_loss_tokens": 18000000, "grad_norm": 0.30859375, "lr": 3e-05, "finish_rate": 0.879, "comp_len": 519.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 49.8, "frames": {"chat": 231}, "mem_gb": 22.03}
|
healed/liger_mb8/step0004/chat_template.jinja
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{{ bos_token }}{% for message in messages %}{% if message['role'] == 'system' %}{{ '<|system|>
|
| 2 |
+
' + message['content'] + '
|
| 3 |
+
' }}{% elif message['role'] == 'user' %}{{ '<|user|>
|
| 4 |
+
' + message['content'] + '
|
| 5 |
+
' }}{% elif message['role'] == 'assistant' %}{% if not loop.last %}{{ '<|assistant|>
|
| 6 |
+
' + message['content'] + eos_token + '
|
| 7 |
+
' }}{% else %}{{ '<|assistant|>
|
| 8 |
+
' + message['content'] + eos_token }}{% endif %}{% endif %}{% if loop.last and add_generation_prompt %}{{ '<|assistant|>
|
| 9 |
+
' }}{% endif %}{% endfor %}
|
healed/liger_mb8/step0004/config.json
ADDED
|
@@ -0,0 +1,887 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"architectures": [
|
| 3 |
+
"PrunedOlmoeForCausalLM"
|
| 4 |
+
],
|
| 5 |
+
"attention_bias": false,
|
| 6 |
+
"attention_dropout": 0.0,
|
| 7 |
+
"auto_map": {
|
| 8 |
+
"AutoConfig": "configuration_pruned_olmoe.PrunedOlmoeConfig",
|
| 9 |
+
"AutoModelForCausalLM": "modeling_pruned_olmoe.PrunedOlmoeForCausalLM"
|
| 10 |
+
},
|
| 11 |
+
"clip_qkv": null,
|
| 12 |
+
"dtype": "bfloat16",
|
| 13 |
+
"eos_token_id": 50279,
|
| 14 |
+
"expert_widths": [
|
| 15 |
+
[
|
| 16 |
+
1024,
|
| 17 |
+
384,
|
| 18 |
+
256,
|
| 19 |
+
256,
|
| 20 |
+
768,
|
| 21 |
+
1024,
|
| 22 |
+
128,
|
| 23 |
+
768,
|
| 24 |
+
384,
|
| 25 |
+
128,
|
| 26 |
+
1024,
|
| 27 |
+
384,
|
| 28 |
+
384,
|
| 29 |
+
128,
|
| 30 |
+
640,
|
| 31 |
+
896,
|
| 32 |
+
896,
|
| 33 |
+
256,
|
| 34 |
+
768,
|
| 35 |
+
128,
|
| 36 |
+
768,
|
| 37 |
+
768,
|
| 38 |
+
768,
|
| 39 |
+
768,
|
| 40 |
+
640,
|
| 41 |
+
896,
|
| 42 |
+
256,
|
| 43 |
+
128,
|
| 44 |
+
384,
|
| 45 |
+
896,
|
| 46 |
+
256,
|
| 47 |
+
768,
|
| 48 |
+
512,
|
| 49 |
+
640,
|
| 50 |
+
768,
|
| 51 |
+
896,
|
| 52 |
+
1024,
|
| 53 |
+
384,
|
| 54 |
+
384,
|
| 55 |
+
512,
|
| 56 |
+
384,
|
| 57 |
+
256,
|
| 58 |
+
768,
|
| 59 |
+
640,
|
| 60 |
+
896,
|
| 61 |
+
896,
|
| 62 |
+
512,
|
| 63 |
+
768,
|
| 64 |
+
512,
|
| 65 |
+
384,
|
| 66 |
+
1024,
|
| 67 |
+
896,
|
| 68 |
+
896,
|
| 69 |
+
768,
|
| 70 |
+
128
|
| 71 |
+
],
|
| 72 |
+
[
|
| 73 |
+
768,
|
| 74 |
+
896,
|
| 75 |
+
896,
|
| 76 |
+
256,
|
| 77 |
+
896,
|
| 78 |
+
384,
|
| 79 |
+
768,
|
| 80 |
+
768,
|
| 81 |
+
1024,
|
| 82 |
+
384,
|
| 83 |
+
512,
|
| 84 |
+
1024,
|
| 85 |
+
768,
|
| 86 |
+
768,
|
| 87 |
+
1024,
|
| 88 |
+
640,
|
| 89 |
+
128,
|
| 90 |
+
768,
|
| 91 |
+
384,
|
| 92 |
+
768,
|
| 93 |
+
128,
|
| 94 |
+
256,
|
| 95 |
+
512,
|
| 96 |
+
384,
|
| 97 |
+
896,
|
| 98 |
+
256,
|
| 99 |
+
1024,
|
| 100 |
+
1024,
|
| 101 |
+
896,
|
| 102 |
+
1024,
|
| 103 |
+
896,
|
| 104 |
+
768,
|
| 105 |
+
256,
|
| 106 |
+
768,
|
| 107 |
+
128,
|
| 108 |
+
1024,
|
| 109 |
+
768,
|
| 110 |
+
768,
|
| 111 |
+
640,
|
| 112 |
+
384,
|
| 113 |
+
1024,
|
| 114 |
+
384,
|
| 115 |
+
640,
|
| 116 |
+
896,
|
| 117 |
+
1024,
|
| 118 |
+
512,
|
| 119 |
+
512,
|
| 120 |
+
512,
|
| 121 |
+
640
|
| 122 |
+
],
|
| 123 |
+
[
|
| 124 |
+
768,
|
| 125 |
+
1024,
|
| 126 |
+
512,
|
| 127 |
+
384,
|
| 128 |
+
384,
|
| 129 |
+
768,
|
| 130 |
+
256,
|
| 131 |
+
384,
|
| 132 |
+
1024,
|
| 133 |
+
896,
|
| 134 |
+
640,
|
| 135 |
+
256,
|
| 136 |
+
1024,
|
| 137 |
+
1024,
|
| 138 |
+
1024,
|
| 139 |
+
640,
|
| 140 |
+
1024,
|
| 141 |
+
896,
|
| 142 |
+
512,
|
| 143 |
+
768,
|
| 144 |
+
128,
|
| 145 |
+
768,
|
| 146 |
+
1024,
|
| 147 |
+
768,
|
| 148 |
+
256,
|
| 149 |
+
1024,
|
| 150 |
+
896,
|
| 151 |
+
1024,
|
| 152 |
+
896,
|
| 153 |
+
512,
|
| 154 |
+
640,
|
| 155 |
+
384,
|
| 156 |
+
896,
|
| 157 |
+
384,
|
| 158 |
+
512,
|
| 159 |
+
640,
|
| 160 |
+
256,
|
| 161 |
+
768,
|
| 162 |
+
256,
|
| 163 |
+
768,
|
| 164 |
+
768,
|
| 165 |
+
640,
|
| 166 |
+
1024,
|
| 167 |
+
640,
|
| 168 |
+
896,
|
| 169 |
+
768,
|
| 170 |
+
768,
|
| 171 |
+
256
|
| 172 |
+
],
|
| 173 |
+
[
|
| 174 |
+
640,
|
| 175 |
+
640,
|
| 176 |
+
896,
|
| 177 |
+
512,
|
| 178 |
+
640,
|
| 179 |
+
896,
|
| 180 |
+
896,
|
| 181 |
+
384,
|
| 182 |
+
896,
|
| 183 |
+
896,
|
| 184 |
+
384,
|
| 185 |
+
896,
|
| 186 |
+
640,
|
| 187 |
+
256,
|
| 188 |
+
640,
|
| 189 |
+
1024,
|
| 190 |
+
1024,
|
| 191 |
+
768,
|
| 192 |
+
1024,
|
| 193 |
+
896,
|
| 194 |
+
1024,
|
| 195 |
+
768,
|
| 196 |
+
896,
|
| 197 |
+
256,
|
| 198 |
+
512,
|
| 199 |
+
768,
|
| 200 |
+
1024,
|
| 201 |
+
256,
|
| 202 |
+
768,
|
| 203 |
+
512,
|
| 204 |
+
256,
|
| 205 |
+
640,
|
| 206 |
+
1024,
|
| 207 |
+
1024,
|
| 208 |
+
512,
|
| 209 |
+
1024,
|
| 210 |
+
768,
|
| 211 |
+
256,
|
| 212 |
+
1024,
|
| 213 |
+
384,
|
| 214 |
+
896,
|
| 215 |
+
1024,
|
| 216 |
+
896,
|
| 217 |
+
1024,
|
| 218 |
+
384
|
| 219 |
+
],
|
| 220 |
+
[
|
| 221 |
+
256,
|
| 222 |
+
640,
|
| 223 |
+
640,
|
| 224 |
+
896,
|
| 225 |
+
768,
|
| 226 |
+
896,
|
| 227 |
+
768,
|
| 228 |
+
768,
|
| 229 |
+
896,
|
| 230 |
+
896,
|
| 231 |
+
1024,
|
| 232 |
+
256,
|
| 233 |
+
512,
|
| 234 |
+
1024,
|
| 235 |
+
640,
|
| 236 |
+
896,
|
| 237 |
+
512,
|
| 238 |
+
512,
|
| 239 |
+
384,
|
| 240 |
+
384,
|
| 241 |
+
256,
|
| 242 |
+
384,
|
| 243 |
+
384,
|
| 244 |
+
896,
|
| 245 |
+
896,
|
| 246 |
+
768,
|
| 247 |
+
640,
|
| 248 |
+
896,
|
| 249 |
+
768,
|
| 250 |
+
1024,
|
| 251 |
+
512,
|
| 252 |
+
640,
|
| 253 |
+
512,
|
| 254 |
+
640,
|
| 255 |
+
896,
|
| 256 |
+
512,
|
| 257 |
+
512,
|
| 258 |
+
384,
|
| 259 |
+
640,
|
| 260 |
+
896,
|
| 261 |
+
896,
|
| 262 |
+
896,
|
| 263 |
+
1024,
|
| 264 |
+
640,
|
| 265 |
+
1024,
|
| 266 |
+
640,
|
| 267 |
+
1024
|
| 268 |
+
],
|
| 269 |
+
[
|
| 270 |
+
1024,
|
| 271 |
+
512,
|
| 272 |
+
1024,
|
| 273 |
+
1024,
|
| 274 |
+
640,
|
| 275 |
+
896,
|
| 276 |
+
640,
|
| 277 |
+
1024,
|
| 278 |
+
896,
|
| 279 |
+
384,
|
| 280 |
+
1024,
|
| 281 |
+
128,
|
| 282 |
+
896,
|
| 283 |
+
768,
|
| 284 |
+
1024,
|
| 285 |
+
768,
|
| 286 |
+
640,
|
| 287 |
+
896,
|
| 288 |
+
768,
|
| 289 |
+
640,
|
| 290 |
+
512,
|
| 291 |
+
896,
|
| 292 |
+
512,
|
| 293 |
+
640,
|
| 294 |
+
256,
|
| 295 |
+
768,
|
| 296 |
+
640,
|
| 297 |
+
768,
|
| 298 |
+
384,
|
| 299 |
+
896,
|
| 300 |
+
512,
|
| 301 |
+
512,
|
| 302 |
+
256,
|
| 303 |
+
512,
|
| 304 |
+
896,
|
| 305 |
+
256,
|
| 306 |
+
384,
|
| 307 |
+
640,
|
| 308 |
+
512,
|
| 309 |
+
640,
|
| 310 |
+
896,
|
| 311 |
+
512,
|
| 312 |
+
1024,
|
| 313 |
+
256,
|
| 314 |
+
768,
|
| 315 |
+
1024,
|
| 316 |
+
768,
|
| 317 |
+
256,
|
| 318 |
+
256
|
| 319 |
+
],
|
| 320 |
+
[
|
| 321 |
+
640,
|
| 322 |
+
896,
|
| 323 |
+
1024,
|
| 324 |
+
896,
|
| 325 |
+
1024,
|
| 326 |
+
1024,
|
| 327 |
+
1024,
|
| 328 |
+
512,
|
| 329 |
+
256,
|
| 330 |
+
256,
|
| 331 |
+
1024,
|
| 332 |
+
768,
|
| 333 |
+
512,
|
| 334 |
+
768,
|
| 335 |
+
1024,
|
| 336 |
+
1024,
|
| 337 |
+
1024,
|
| 338 |
+
384,
|
| 339 |
+
512,
|
| 340 |
+
1024,
|
| 341 |
+
512,
|
| 342 |
+
1024,
|
| 343 |
+
128,
|
| 344 |
+
640,
|
| 345 |
+
640,
|
| 346 |
+
896,
|
| 347 |
+
768,
|
| 348 |
+
128,
|
| 349 |
+
256,
|
| 350 |
+
256,
|
| 351 |
+
256,
|
| 352 |
+
256,
|
| 353 |
+
896,
|
| 354 |
+
1024,
|
| 355 |
+
1024,
|
| 356 |
+
384,
|
| 357 |
+
896,
|
| 358 |
+
256,
|
| 359 |
+
896,
|
| 360 |
+
640,
|
| 361 |
+
1024,
|
| 362 |
+
384,
|
| 363 |
+
640,
|
| 364 |
+
256,
|
| 365 |
+
1024,
|
| 366 |
+
1024,
|
| 367 |
+
1024
|
| 368 |
+
],
|
| 369 |
+
[
|
| 370 |
+
1024,
|
| 371 |
+
384,
|
| 372 |
+
1024,
|
| 373 |
+
1024,
|
| 374 |
+
256,
|
| 375 |
+
128,
|
| 376 |
+
256,
|
| 377 |
+
384,
|
| 378 |
+
256,
|
| 379 |
+
384,
|
| 380 |
+
896,
|
| 381 |
+
768,
|
| 382 |
+
896,
|
| 383 |
+
896,
|
| 384 |
+
512,
|
| 385 |
+
896,
|
| 386 |
+
640,
|
| 387 |
+
384,
|
| 388 |
+
384,
|
| 389 |
+
896,
|
| 390 |
+
768,
|
| 391 |
+
384,
|
| 392 |
+
896,
|
| 393 |
+
768,
|
| 394 |
+
768,
|
| 395 |
+
512,
|
| 396 |
+
896,
|
| 397 |
+
768,
|
| 398 |
+
768,
|
| 399 |
+
896,
|
| 400 |
+
128,
|
| 401 |
+
896,
|
| 402 |
+
512,
|
| 403 |
+
256,
|
| 404 |
+
768,
|
| 405 |
+
128,
|
| 406 |
+
384,
|
| 407 |
+
256,
|
| 408 |
+
896,
|
| 409 |
+
896,
|
| 410 |
+
384,
|
| 411 |
+
768,
|
| 412 |
+
512,
|
| 413 |
+
640,
|
| 414 |
+
256,
|
| 415 |
+
768,
|
| 416 |
+
640,
|
| 417 |
+
896,
|
| 418 |
+
384,
|
| 419 |
+
512,
|
| 420 |
+
1024,
|
| 421 |
+
768,
|
| 422 |
+
384
|
| 423 |
+
],
|
| 424 |
+
[
|
| 425 |
+
512,
|
| 426 |
+
768,
|
| 427 |
+
512,
|
| 428 |
+
256,
|
| 429 |
+
128,
|
| 430 |
+
640,
|
| 431 |
+
384,
|
| 432 |
+
640,
|
| 433 |
+
768,
|
| 434 |
+
896,
|
| 435 |
+
640,
|
| 436 |
+
768,
|
| 437 |
+
256,
|
| 438 |
+
384,
|
| 439 |
+
1024,
|
| 440 |
+
896,
|
| 441 |
+
256,
|
| 442 |
+
896,
|
| 443 |
+
512,
|
| 444 |
+
256,
|
| 445 |
+
896,
|
| 446 |
+
768,
|
| 447 |
+
256,
|
| 448 |
+
896,
|
| 449 |
+
896,
|
| 450 |
+
384,
|
| 451 |
+
896,
|
| 452 |
+
640,
|
| 453 |
+
768,
|
| 454 |
+
512,
|
| 455 |
+
768,
|
| 456 |
+
768,
|
| 457 |
+
1024,
|
| 458 |
+
768,
|
| 459 |
+
640,
|
| 460 |
+
768,
|
| 461 |
+
384,
|
| 462 |
+
256,
|
| 463 |
+
512,
|
| 464 |
+
896,
|
| 465 |
+
128,
|
| 466 |
+
384,
|
| 467 |
+
256,
|
| 468 |
+
768,
|
| 469 |
+
384,
|
| 470 |
+
256,
|
| 471 |
+
1024,
|
| 472 |
+
1024,
|
| 473 |
+
896,
|
| 474 |
+
256,
|
| 475 |
+
1024,
|
| 476 |
+
256,
|
| 477 |
+
128,
|
| 478 |
+
896
|
| 479 |
+
],
|
| 480 |
+
[
|
| 481 |
+
640,
|
| 482 |
+
640,
|
| 483 |
+
896,
|
| 484 |
+
256,
|
| 485 |
+
1024,
|
| 486 |
+
512,
|
| 487 |
+
1024,
|
| 488 |
+
768,
|
| 489 |
+
384,
|
| 490 |
+
512,
|
| 491 |
+
256,
|
| 492 |
+
768,
|
| 493 |
+
896,
|
| 494 |
+
768,
|
| 495 |
+
512,
|
| 496 |
+
768,
|
| 497 |
+
768,
|
| 498 |
+
640,
|
| 499 |
+
384,
|
| 500 |
+
768,
|
| 501 |
+
512,
|
| 502 |
+
768,
|
| 503 |
+
768,
|
| 504 |
+
512,
|
| 505 |
+
768,
|
| 506 |
+
128,
|
| 507 |
+
896,
|
| 508 |
+
512,
|
| 509 |
+
768,
|
| 510 |
+
1024,
|
| 511 |
+
128,
|
| 512 |
+
384,
|
| 513 |
+
768,
|
| 514 |
+
768,
|
| 515 |
+
768,
|
| 516 |
+
384,
|
| 517 |
+
512,
|
| 518 |
+
640,
|
| 519 |
+
768,
|
| 520 |
+
512,
|
| 521 |
+
768,
|
| 522 |
+
1024,
|
| 523 |
+
640,
|
| 524 |
+
896,
|
| 525 |
+
256,
|
| 526 |
+
1024,
|
| 527 |
+
384,
|
| 528 |
+
768,
|
| 529 |
+
768,
|
| 530 |
+
768
|
| 531 |
+
],
|
| 532 |
+
[
|
| 533 |
+
896,
|
| 534 |
+
512,
|
| 535 |
+
896,
|
| 536 |
+
768,
|
| 537 |
+
384,
|
| 538 |
+
384,
|
| 539 |
+
768,
|
| 540 |
+
512,
|
| 541 |
+
768,
|
| 542 |
+
512,
|
| 543 |
+
1024,
|
| 544 |
+
640,
|
| 545 |
+
896,
|
| 546 |
+
896,
|
| 547 |
+
256,
|
| 548 |
+
640,
|
| 549 |
+
1024,
|
| 550 |
+
256,
|
| 551 |
+
896,
|
| 552 |
+
128,
|
| 553 |
+
128,
|
| 554 |
+
128,
|
| 555 |
+
768,
|
| 556 |
+
896,
|
| 557 |
+
384,
|
| 558 |
+
896,
|
| 559 |
+
512,
|
| 560 |
+
896,
|
| 561 |
+
384,
|
| 562 |
+
256,
|
| 563 |
+
640,
|
| 564 |
+
640,
|
| 565 |
+
896,
|
| 566 |
+
768,
|
| 567 |
+
640,
|
| 568 |
+
256,
|
| 569 |
+
896,
|
| 570 |
+
896,
|
| 571 |
+
512,
|
| 572 |
+
128,
|
| 573 |
+
896,
|
| 574 |
+
256,
|
| 575 |
+
256,
|
| 576 |
+
640,
|
| 577 |
+
896,
|
| 578 |
+
896,
|
| 579 |
+
128,
|
| 580 |
+
1024,
|
| 581 |
+
256,
|
| 582 |
+
384,
|
| 583 |
+
1024,
|
| 584 |
+
640,
|
| 585 |
+
896
|
| 586 |
+
],
|
| 587 |
+
[
|
| 588 |
+
768,
|
| 589 |
+
384,
|
| 590 |
+
640,
|
| 591 |
+
896,
|
| 592 |
+
256,
|
| 593 |
+
128,
|
| 594 |
+
384,
|
| 595 |
+
896,
|
| 596 |
+
128,
|
| 597 |
+
128,
|
| 598 |
+
896,
|
| 599 |
+
256,
|
| 600 |
+
384,
|
| 601 |
+
896,
|
| 602 |
+
512,
|
| 603 |
+
768,
|
| 604 |
+
768,
|
| 605 |
+
512,
|
| 606 |
+
512,
|
| 607 |
+
768,
|
| 608 |
+
896,
|
| 609 |
+
640,
|
| 610 |
+
768,
|
| 611 |
+
896,
|
| 612 |
+
896,
|
| 613 |
+
896,
|
| 614 |
+
640,
|
| 615 |
+
896,
|
| 616 |
+
640,
|
| 617 |
+
512,
|
| 618 |
+
896,
|
| 619 |
+
256,
|
| 620 |
+
512,
|
| 621 |
+
128,
|
| 622 |
+
512,
|
| 623 |
+
384,
|
| 624 |
+
768,
|
| 625 |
+
768,
|
| 626 |
+
1024,
|
| 627 |
+
256,
|
| 628 |
+
768,
|
| 629 |
+
256,
|
| 630 |
+
768,
|
| 631 |
+
512,
|
| 632 |
+
640,
|
| 633 |
+
1024,
|
| 634 |
+
128,
|
| 635 |
+
896,
|
| 636 |
+
896,
|
| 637 |
+
896,
|
| 638 |
+
896,
|
| 639 |
+
1024
|
| 640 |
+
],
|
| 641 |
+
[
|
| 642 |
+
512,
|
| 643 |
+
384,
|
| 644 |
+
768,
|
| 645 |
+
640,
|
| 646 |
+
640,
|
| 647 |
+
768,
|
| 648 |
+
1024,
|
| 649 |
+
896,
|
| 650 |
+
512,
|
| 651 |
+
256,
|
| 652 |
+
640,
|
| 653 |
+
768,
|
| 654 |
+
640,
|
| 655 |
+
896,
|
| 656 |
+
128,
|
| 657 |
+
256,
|
| 658 |
+
896,
|
| 659 |
+
1024,
|
| 660 |
+
256,
|
| 661 |
+
640,
|
| 662 |
+
512,
|
| 663 |
+
256,
|
| 664 |
+
128,
|
| 665 |
+
512,
|
| 666 |
+
256,
|
| 667 |
+
640,
|
| 668 |
+
768,
|
| 669 |
+
768,
|
| 670 |
+
128,
|
| 671 |
+
128,
|
| 672 |
+
768,
|
| 673 |
+
640,
|
| 674 |
+
1024,
|
| 675 |
+
1024,
|
| 676 |
+
768,
|
| 677 |
+
512,
|
| 678 |
+
896,
|
| 679 |
+
768,
|
| 680 |
+
896,
|
| 681 |
+
1024,
|
| 682 |
+
896,
|
| 683 |
+
896,
|
| 684 |
+
512,
|
| 685 |
+
640,
|
| 686 |
+
1024,
|
| 687 |
+
512,
|
| 688 |
+
1024,
|
| 689 |
+
512,
|
| 690 |
+
512,
|
| 691 |
+
512,
|
| 692 |
+
768
|
| 693 |
+
],
|
| 694 |
+
[
|
| 695 |
+
896,
|
| 696 |
+
896,
|
| 697 |
+
1024,
|
| 698 |
+
1024,
|
| 699 |
+
896,
|
| 700 |
+
128,
|
| 701 |
+
768,
|
| 702 |
+
256,
|
| 703 |
+
1024,
|
| 704 |
+
256,
|
| 705 |
+
1024,
|
| 706 |
+
640,
|
| 707 |
+
384,
|
| 708 |
+
256,
|
| 709 |
+
256,
|
| 710 |
+
512,
|
| 711 |
+
768,
|
| 712 |
+
896,
|
| 713 |
+
512,
|
| 714 |
+
768,
|
| 715 |
+
384,
|
| 716 |
+
1024,
|
| 717 |
+
896,
|
| 718 |
+
896,
|
| 719 |
+
1024,
|
| 720 |
+
896,
|
| 721 |
+
768,
|
| 722 |
+
896,
|
| 723 |
+
640,
|
| 724 |
+
1024,
|
| 725 |
+
512,
|
| 726 |
+
896,
|
| 727 |
+
512,
|
| 728 |
+
1024,
|
| 729 |
+
512,
|
| 730 |
+
512,
|
| 731 |
+
256,
|
| 732 |
+
256,
|
| 733 |
+
256,
|
| 734 |
+
512,
|
| 735 |
+
768,
|
| 736 |
+
128,
|
| 737 |
+
384,
|
| 738 |
+
512,
|
| 739 |
+
896,
|
| 740 |
+
896,
|
| 741 |
+
1024,
|
| 742 |
+
256,
|
| 743 |
+
384,
|
| 744 |
+
640
|
| 745 |
+
],
|
| 746 |
+
[
|
| 747 |
+
896,
|
| 748 |
+
640,
|
| 749 |
+
384,
|
| 750 |
+
512,
|
| 751 |
+
256,
|
| 752 |
+
640,
|
| 753 |
+
1024,
|
| 754 |
+
384,
|
| 755 |
+
1024,
|
| 756 |
+
1024,
|
| 757 |
+
768,
|
| 758 |
+
256,
|
| 759 |
+
1024,
|
| 760 |
+
768,
|
| 761 |
+
512,
|
| 762 |
+
896,
|
| 763 |
+
256,
|
| 764 |
+
1024,
|
| 765 |
+
768,
|
| 766 |
+
768,
|
| 767 |
+
768,
|
| 768 |
+
384,
|
| 769 |
+
384,
|
| 770 |
+
256,
|
| 771 |
+
1024,
|
| 772 |
+
384,
|
| 773 |
+
384,
|
| 774 |
+
384,
|
| 775 |
+
896,
|
| 776 |
+
768,
|
| 777 |
+
640,
|
| 778 |
+
768,
|
| 779 |
+
512,
|
| 780 |
+
896,
|
| 781 |
+
896,
|
| 782 |
+
896,
|
| 783 |
+
896,
|
| 784 |
+
256,
|
| 785 |
+
384,
|
| 786 |
+
128,
|
| 787 |
+
1024,
|
| 788 |
+
896,
|
| 789 |
+
256,
|
| 790 |
+
256,
|
| 791 |
+
768,
|
| 792 |
+
640,
|
| 793 |
+
896,
|
| 794 |
+
384,
|
| 795 |
+
768,
|
| 796 |
+
512,
|
| 797 |
+
640
|
| 798 |
+
],
|
| 799 |
+
[
|
| 800 |
+
896,
|
| 801 |
+
1024,
|
| 802 |
+
768,
|
| 803 |
+
1024,
|
| 804 |
+
896,
|
| 805 |
+
256,
|
| 806 |
+
768,
|
| 807 |
+
128,
|
| 808 |
+
128,
|
| 809 |
+
768,
|
| 810 |
+
512,
|
| 811 |
+
896,
|
| 812 |
+
384,
|
| 813 |
+
768,
|
| 814 |
+
1024,
|
| 815 |
+
256,
|
| 816 |
+
768,
|
| 817 |
+
768,
|
| 818 |
+
256,
|
| 819 |
+
512,
|
| 820 |
+
512,
|
| 821 |
+
640,
|
| 822 |
+
512,
|
| 823 |
+
256,
|
| 824 |
+
768,
|
| 825 |
+
896,
|
| 826 |
+
384,
|
| 827 |
+
1024,
|
| 828 |
+
640,
|
| 829 |
+
1024,
|
| 830 |
+
512,
|
| 831 |
+
512,
|
| 832 |
+
384,
|
| 833 |
+
512,
|
| 834 |
+
512,
|
| 835 |
+
1024,
|
| 836 |
+
384,
|
| 837 |
+
896,
|
| 838 |
+
768,
|
| 839 |
+
384,
|
| 840 |
+
384,
|
| 841 |
+
128,
|
| 842 |
+
384,
|
| 843 |
+
1024,
|
| 844 |
+
896,
|
| 845 |
+
640,
|
| 846 |
+
768,
|
| 847 |
+
768,
|
| 848 |
+
256,
|
| 849 |
+
640,
|
| 850 |
+
512,
|
| 851 |
+
640,
|
| 852 |
+
384
|
| 853 |
+
]
|
| 854 |
+
],
|
| 855 |
+
"glean_metadata": {
|
| 856 |
+
"base_model": "allenai/OLMoE-1B-7B-0125-Instruct",
|
| 857 |
+
"block_size": 128,
|
| 858 |
+
"criterion": "reap",
|
| 859 |
+
"dead_experts": 217,
|
| 860 |
+
"keep_fraction": 0.5,
|
| 861 |
+
"min_width": 128,
|
| 862 |
+
"params": 3697491968,
|
| 863 |
+
"scores": "outputs/scores_0125inst_dolmino-math/scores.pt"
|
| 864 |
+
},
|
| 865 |
+
"hidden_act": "silu",
|
| 866 |
+
"hidden_size": 2048,
|
| 867 |
+
"initializer_range": 0.02,
|
| 868 |
+
"intermediate_size": 1024,
|
| 869 |
+
"max_position_embeddings": 4096,
|
| 870 |
+
"model_type": "pruned_olmoe",
|
| 871 |
+
"norm_topk_prob": false,
|
| 872 |
+
"num_attention_heads": 16,
|
| 873 |
+
"num_experts": 64,
|
| 874 |
+
"num_experts_per_tok": 8,
|
| 875 |
+
"num_hidden_layers": 16,
|
| 876 |
+
"num_key_value_heads": 16,
|
| 877 |
+
"output_router_logits": false,
|
| 878 |
+
"pad_token_id": 1,
|
| 879 |
+
"rms_norm_eps": 1e-05,
|
| 880 |
+
"rope_scaling": null,
|
| 881 |
+
"rope_theta": 10000.0,
|
| 882 |
+
"router_aux_loss_coef": 0.01,
|
| 883 |
+
"tie_word_embeddings": false,
|
| 884 |
+
"transformers_version": "4.57.6",
|
| 885 |
+
"use_cache": false,
|
| 886 |
+
"vocab_size": 50304
|
| 887 |
+
}
|
healed/liger_mb8/step0004/configuration_pruned_olmoe.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Configuration for GLEAN-pruned OLMoE: variable-width, variable-count experts.
|
| 2 |
+
"""
|
| 3 |
+
|
| 4 |
+
from transformers.models.olmoe.configuration_olmoe import OlmoeConfig
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
class PrunedOlmoeConfig(OlmoeConfig):
|
| 8 |
+
"""OlmoeConfig plus a per-(layer, expert) width table.
|
| 9 |
+
|
| 10 |
+
``expert_widths[l]`` lists the SwiGLU intermediate width of each surviving
|
| 11 |
+
expert in decoder layer ``l``, in expert order. Lists are ragged: layers
|
| 12 |
+
may keep different numbers of experts (deleted experts simply don't
|
| 13 |
+
appear — the router in layer ``l`` has ``len(expert_widths[l])`` rows),
|
| 14 |
+
and each width may differ (multiples of the GEMM block size, 128, for
|
| 15 |
+
variable-MegaBlocks execution). ``None`` means an unpruned model
|
| 16 |
+
(uniform ``num_experts`` × ``intermediate_size``).
|
| 17 |
+
|
| 18 |
+
The inherited ``num_experts`` / ``intermediate_size`` keep their ORIGINAL
|
| 19 |
+
(pre-pruning) values for provenance; the width table is authoritative for
|
| 20 |
+
the built architecture.
|
| 21 |
+
"""
|
| 22 |
+
|
| 23 |
+
model_type = "pruned_olmoe"
|
| 24 |
+
|
| 25 |
+
def __init__(self, expert_widths: list[list[int]] | None = None, **kwargs):
|
| 26 |
+
super().__init__(**kwargs)
|
| 27 |
+
self.expert_widths = expert_widths
|
healed/liger_mb8/step0004/generation_config.json
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_from_model_config": true,
|
| 3 |
+
"eos_token_id": 50279,
|
| 4 |
+
"pad_token_id": 1,
|
| 5 |
+
"transformers_version": "4.57.6"
|
| 6 |
+
}
|
healed/liger_mb8/step0004/model.safetensors.index.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
healed/liger_mb8/step0004/modeling_pruned_olmoe.py
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""GLEAN-pruned OLMoE: HF-loadable model with ragged (variable-width) experts.
|
| 2 |
+
|
| 3 |
+
Pattern follows hbfreed/variable-flex-olmo's PrunedFlexOlmoForCausalLM
|
| 4 |
+
(docs/recon/prior-work-hbfreed.md), generalized from one scalar width to a
|
| 5 |
+
per-(layer, expert) width table: ``super().__init__`` builds the uniform
|
| 6 |
+
architecture from the config, then every MoE block is rebuilt to its pruned
|
| 7 |
+
shape — surviving experts only, each at its own width, router sliced to
|
| 8 |
+
match — so the state dict aligns exactly with what
|
| 9 |
+
``glean.prune.prune_channels_global`` leaves behind.
|
| 10 |
+
|
| 11 |
+
Caveat: ``output_router_logits=True`` (the load-balancing aux loss) assumes a
|
| 12 |
+
uniform ``config.num_experts`` and is unsupported on ragged models.
|
| 13 |
+
"""
|
| 14 |
+
|
| 15 |
+
import torch.nn as nn
|
| 16 |
+
from transformers.activations import ACT2FN
|
| 17 |
+
from transformers.models.olmoe.modeling_olmoe import OlmoeForCausalLM
|
| 18 |
+
|
| 19 |
+
from .configuration_pruned_olmoe import PrunedOlmoeConfig
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
class RaggedOlmoeMLP(nn.Module):
|
| 23 |
+
"""OlmoeMLP with an explicit intermediate width (SwiGLU, no biases)."""
|
| 24 |
+
|
| 25 |
+
def __init__(self, hidden_size: int, intermediate_size: int, hidden_act: str):
|
| 26 |
+
super().__init__()
|
| 27 |
+
self.hidden_size = hidden_size
|
| 28 |
+
self.intermediate_size = intermediate_size
|
| 29 |
+
self.gate_proj = nn.Linear(hidden_size, intermediate_size, bias=False)
|
| 30 |
+
self.up_proj = nn.Linear(hidden_size, intermediate_size, bias=False)
|
| 31 |
+
self.down_proj = nn.Linear(intermediate_size, hidden_size, bias=False)
|
| 32 |
+
self.act_fn = ACT2FN[hidden_act]
|
| 33 |
+
|
| 34 |
+
def forward(self, x):
|
| 35 |
+
return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
class PrunedOlmoeForCausalLM(OlmoeForCausalLM):
|
| 39 |
+
"""OLMoE with per-layer surviving-expert lists at per-expert widths."""
|
| 40 |
+
|
| 41 |
+
config_class = PrunedOlmoeConfig
|
| 42 |
+
|
| 43 |
+
def __init__(self, config: PrunedOlmoeConfig):
|
| 44 |
+
super().__init__(config)
|
| 45 |
+
widths_table = getattr(config, "expert_widths", None)
|
| 46 |
+
if widths_table is None:
|
| 47 |
+
return # unpruned: plain OLMoE
|
| 48 |
+
if len(widths_table) != len(self.model.layers):
|
| 49 |
+
raise ValueError(
|
| 50 |
+
f"expert_widths has {len(widths_table)} rows but the model has "
|
| 51 |
+
f"{len(self.model.layers)} decoder layers"
|
| 52 |
+
)
|
| 53 |
+
for layer, widths in zip(self.model.layers, widths_table):
|
| 54 |
+
if any(w <= 0 for w in widths):
|
| 55 |
+
raise ValueError("expert_widths must list surviving experts only (>0)")
|
| 56 |
+
block = layer.mlp
|
| 57 |
+
if len(widths) < block.top_k:
|
| 58 |
+
raise ValueError(
|
| 59 |
+
f"a layer keeps {len(widths)} experts < top_k={block.top_k}"
|
| 60 |
+
)
|
| 61 |
+
block.num_experts = len(widths)
|
| 62 |
+
block.gate = nn.Linear(config.hidden_size, len(widths), bias=False)
|
| 63 |
+
block.experts = nn.ModuleList(
|
| 64 |
+
RaggedOlmoeMLP(config.hidden_size, w, config.hidden_act)
|
| 65 |
+
for w in widths
|
| 66 |
+
)
|
healed/liger_mb8/step0004/special_tokens_map.json
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"bos_token": {
|
| 3 |
+
"content": "|||IP_ADDRESS|||",
|
| 4 |
+
"lstrip": false,
|
| 5 |
+
"normalized": true,
|
| 6 |
+
"rstrip": false,
|
| 7 |
+
"single_word": false
|
| 8 |
+
},
|
| 9 |
+
"eos_token": {
|
| 10 |
+
"content": "|||IP_ADDRESS|||",
|
| 11 |
+
"lstrip": false,
|
| 12 |
+
"normalized": true,
|
| 13 |
+
"rstrip": false,
|
| 14 |
+
"single_word": false
|
| 15 |
+
},
|
| 16 |
+
"pad_token": {
|
| 17 |
+
"content": "<pad>",
|
| 18 |
+
"lstrip": false,
|
| 19 |
+
"normalized": false,
|
| 20 |
+
"rstrip": false,
|
| 21 |
+
"single_word": false
|
| 22 |
+
}
|
| 23 |
+
}
|
healed/liger_mb8/step0004/tokenizer.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
healed/liger_mb8/step0004/tokenizer_config.json
ADDED
|
@@ -0,0 +1,247 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"add_bos_token": false,
|
| 3 |
+
"add_eos_token": false,
|
| 4 |
+
"add_prefix_space": false,
|
| 5 |
+
"added_tokens_decoder": {
|
| 6 |
+
"0": {
|
| 7 |
+
"content": "<|endoftext|>",
|
| 8 |
+
"lstrip": false,
|
| 9 |
+
"normalized": false,
|
| 10 |
+
"rstrip": false,
|
| 11 |
+
"single_word": false,
|
| 12 |
+
"special": true
|
| 13 |
+
},
|
| 14 |
+
"1": {
|
| 15 |
+
"content": "<|padding|>",
|
| 16 |
+
"lstrip": false,
|
| 17 |
+
"normalized": false,
|
| 18 |
+
"rstrip": false,
|
| 19 |
+
"single_word": false,
|
| 20 |
+
"special": true
|
| 21 |
+
},
|
| 22 |
+
"50254": {
|
| 23 |
+
"content": " ",
|
| 24 |
+
"lstrip": false,
|
| 25 |
+
"normalized": true,
|
| 26 |
+
"rstrip": false,
|
| 27 |
+
"single_word": false,
|
| 28 |
+
"special": false
|
| 29 |
+
},
|
| 30 |
+
"50255": {
|
| 31 |
+
"content": " ",
|
| 32 |
+
"lstrip": false,
|
| 33 |
+
"normalized": true,
|
| 34 |
+
"rstrip": false,
|
| 35 |
+
"single_word": false,
|
| 36 |
+
"special": false
|
| 37 |
+
},
|
| 38 |
+
"50256": {
|
| 39 |
+
"content": " ",
|
| 40 |
+
"lstrip": false,
|
| 41 |
+
"normalized": true,
|
| 42 |
+
"rstrip": false,
|
| 43 |
+
"single_word": false,
|
| 44 |
+
"special": false
|
| 45 |
+
},
|
| 46 |
+
"50257": {
|
| 47 |
+
"content": " ",
|
| 48 |
+
"lstrip": false,
|
| 49 |
+
"normalized": true,
|
| 50 |
+
"rstrip": false,
|
| 51 |
+
"single_word": false,
|
| 52 |
+
"special": false
|
| 53 |
+
},
|
| 54 |
+
"50258": {
|
| 55 |
+
"content": " ",
|
| 56 |
+
"lstrip": false,
|
| 57 |
+
"normalized": true,
|
| 58 |
+
"rstrip": false,
|
| 59 |
+
"single_word": false,
|
| 60 |
+
"special": false
|
| 61 |
+
},
|
| 62 |
+
"50259": {
|
| 63 |
+
"content": " ",
|
| 64 |
+
"lstrip": false,
|
| 65 |
+
"normalized": true,
|
| 66 |
+
"rstrip": false,
|
| 67 |
+
"single_word": false,
|
| 68 |
+
"special": false
|
| 69 |
+
},
|
| 70 |
+
"50260": {
|
| 71 |
+
"content": " ",
|
| 72 |
+
"lstrip": false,
|
| 73 |
+
"normalized": true,
|
| 74 |
+
"rstrip": false,
|
| 75 |
+
"single_word": false,
|
| 76 |
+
"special": false
|
| 77 |
+
},
|
| 78 |
+
"50261": {
|
| 79 |
+
"content": " ",
|
| 80 |
+
"lstrip": false,
|
| 81 |
+
"normalized": true,
|
| 82 |
+
"rstrip": false,
|
| 83 |
+
"single_word": false,
|
| 84 |
+
"special": false
|
| 85 |
+
},
|
| 86 |
+
"50262": {
|
| 87 |
+
"content": " ",
|
| 88 |
+
"lstrip": false,
|
| 89 |
+
"normalized": true,
|
| 90 |
+
"rstrip": false,
|
| 91 |
+
"single_word": false,
|
| 92 |
+
"special": false
|
| 93 |
+
},
|
| 94 |
+
"50263": {
|
| 95 |
+
"content": " ",
|
| 96 |
+
"lstrip": false,
|
| 97 |
+
"normalized": true,
|
| 98 |
+
"rstrip": false,
|
| 99 |
+
"single_word": false,
|
| 100 |
+
"special": false
|
| 101 |
+
},
|
| 102 |
+
"50264": {
|
| 103 |
+
"content": " ",
|
| 104 |
+
"lstrip": false,
|
| 105 |
+
"normalized": true,
|
| 106 |
+
"rstrip": false,
|
| 107 |
+
"single_word": false,
|
| 108 |
+
"special": false
|
| 109 |
+
},
|
| 110 |
+
"50265": {
|
| 111 |
+
"content": " ",
|
| 112 |
+
"lstrip": false,
|
| 113 |
+
"normalized": true,
|
| 114 |
+
"rstrip": false,
|
| 115 |
+
"single_word": false,
|
| 116 |
+
"special": false
|
| 117 |
+
},
|
| 118 |
+
"50266": {
|
| 119 |
+
"content": " ",
|
| 120 |
+
"lstrip": false,
|
| 121 |
+
"normalized": true,
|
| 122 |
+
"rstrip": false,
|
| 123 |
+
"single_word": false,
|
| 124 |
+
"special": false
|
| 125 |
+
},
|
| 126 |
+
"50267": {
|
| 127 |
+
"content": " ",
|
| 128 |
+
"lstrip": false,
|
| 129 |
+
"normalized": true,
|
| 130 |
+
"rstrip": false,
|
| 131 |
+
"single_word": false,
|
| 132 |
+
"special": false
|
| 133 |
+
},
|
| 134 |
+
"50268": {
|
| 135 |
+
"content": " ",
|
| 136 |
+
"lstrip": false,
|
| 137 |
+
"normalized": true,
|
| 138 |
+
"rstrip": false,
|
| 139 |
+
"single_word": false,
|
| 140 |
+
"special": false
|
| 141 |
+
},
|
| 142 |
+
"50269": {
|
| 143 |
+
"content": " ",
|
| 144 |
+
"lstrip": false,
|
| 145 |
+
"normalized": true,
|
| 146 |
+
"rstrip": false,
|
| 147 |
+
"single_word": false,
|
| 148 |
+
"special": false
|
| 149 |
+
},
|
| 150 |
+
"50270": {
|
| 151 |
+
"content": " ",
|
| 152 |
+
"lstrip": false,
|
| 153 |
+
"normalized": true,
|
| 154 |
+
"rstrip": false,
|
| 155 |
+
"single_word": false,
|
| 156 |
+
"special": false
|
| 157 |
+
},
|
| 158 |
+
"50271": {
|
| 159 |
+
"content": " ",
|
| 160 |
+
"lstrip": false,
|
| 161 |
+
"normalized": true,
|
| 162 |
+
"rstrip": false,
|
| 163 |
+
"single_word": false,
|
| 164 |
+
"special": false
|
| 165 |
+
},
|
| 166 |
+
"50272": {
|
| 167 |
+
"content": " ",
|
| 168 |
+
"lstrip": false,
|
| 169 |
+
"normalized": true,
|
| 170 |
+
"rstrip": false,
|
| 171 |
+
"single_word": false,
|
| 172 |
+
"special": false
|
| 173 |
+
},
|
| 174 |
+
"50273": {
|
| 175 |
+
"content": " ",
|
| 176 |
+
"lstrip": false,
|
| 177 |
+
"normalized": true,
|
| 178 |
+
"rstrip": false,
|
| 179 |
+
"single_word": false,
|
| 180 |
+
"special": false
|
| 181 |
+
},
|
| 182 |
+
"50274": {
|
| 183 |
+
"content": " ",
|
| 184 |
+
"lstrip": false,
|
| 185 |
+
"normalized": true,
|
| 186 |
+
"rstrip": false,
|
| 187 |
+
"single_word": false,
|
| 188 |
+
"special": false
|
| 189 |
+
},
|
| 190 |
+
"50275": {
|
| 191 |
+
"content": " ",
|
| 192 |
+
"lstrip": false,
|
| 193 |
+
"normalized": true,
|
| 194 |
+
"rstrip": false,
|
| 195 |
+
"single_word": false,
|
| 196 |
+
"special": false
|
| 197 |
+
},
|
| 198 |
+
"50276": {
|
| 199 |
+
"content": " ",
|
| 200 |
+
"lstrip": false,
|
| 201 |
+
"normalized": true,
|
| 202 |
+
"rstrip": false,
|
| 203 |
+
"single_word": false,
|
| 204 |
+
"special": false
|
| 205 |
+
},
|
| 206 |
+
"50277": {
|
| 207 |
+
"content": "|||EMAIL_ADDRESS|||",
|
| 208 |
+
"lstrip": false,
|
| 209 |
+
"normalized": true,
|
| 210 |
+
"rstrip": false,
|
| 211 |
+
"single_word": false,
|
| 212 |
+
"special": false
|
| 213 |
+
},
|
| 214 |
+
"50278": {
|
| 215 |
+
"content": "|||PHONE_NUMBER|||",
|
| 216 |
+
"lstrip": false,
|
| 217 |
+
"normalized": true,
|
| 218 |
+
"rstrip": false,
|
| 219 |
+
"single_word": false,
|
| 220 |
+
"special": false
|
| 221 |
+
},
|
| 222 |
+
"50279": {
|
| 223 |
+
"content": "|||IP_ADDRESS|||",
|
| 224 |
+
"lstrip": false,
|
| 225 |
+
"normalized": true,
|
| 226 |
+
"rstrip": false,
|
| 227 |
+
"single_word": false,
|
| 228 |
+
"special": true
|
| 229 |
+
},
|
| 230 |
+
"50280": {
|
| 231 |
+
"content": "<pad>",
|
| 232 |
+
"lstrip": false,
|
| 233 |
+
"normalized": false,
|
| 234 |
+
"rstrip": false,
|
| 235 |
+
"single_word": false,
|
| 236 |
+
"special": true
|
| 237 |
+
}
|
| 238 |
+
},
|
| 239 |
+
"bos_token": "|||IP_ADDRESS|||",
|
| 240 |
+
"clean_up_tokenization_spaces": false,
|
| 241 |
+
"eos_token": "|||IP_ADDRESS|||",
|
| 242 |
+
"extra_special_tokens": {},
|
| 243 |
+
"model_max_length": 1000000000000000019884624838656,
|
| 244 |
+
"pad_token": "<pad>",
|
| 245 |
+
"tokenizer_class": "GPTNeoXTokenizer",
|
| 246 |
+
"unk_token": null
|
| 247 |
+
}
|
healed/liger_smoke/step0006/chat_template.jinja
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{{ bos_token }}{% for message in messages %}{% if message['role'] == 'system' %}{{ '<|system|>
|
| 2 |
+
' + message['content'] + '
|
| 3 |
+
' }}{% elif message['role'] == 'user' %}{{ '<|user|>
|
| 4 |
+
' + message['content'] + '
|
| 5 |
+
' }}{% elif message['role'] == 'assistant' %}{% if not loop.last %}{{ '<|assistant|>
|
| 6 |
+
' + message['content'] + eos_token + '
|
| 7 |
+
' }}{% else %}{{ '<|assistant|>
|
| 8 |
+
' + message['content'] + eos_token }}{% endif %}{% endif %}{% if loop.last and add_generation_prompt %}{{ '<|assistant|>
|
| 9 |
+
' }}{% endif %}{% endfor %}
|
healed/liger_smoke/step0006/config.json
ADDED
|
@@ -0,0 +1,887 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"architectures": [
|
| 3 |
+
"PrunedOlmoeForCausalLM"
|
| 4 |
+
],
|
| 5 |
+
"attention_bias": false,
|
| 6 |
+
"attention_dropout": 0.0,
|
| 7 |
+
"auto_map": {
|
| 8 |
+
"AutoConfig": "configuration_pruned_olmoe.PrunedOlmoeConfig",
|
| 9 |
+
"AutoModelForCausalLM": "modeling_pruned_olmoe.PrunedOlmoeForCausalLM"
|
| 10 |
+
},
|
| 11 |
+
"clip_qkv": null,
|
| 12 |
+
"dtype": "bfloat16",
|
| 13 |
+
"eos_token_id": 50279,
|
| 14 |
+
"expert_widths": [
|
| 15 |
+
[
|
| 16 |
+
1024,
|
| 17 |
+
384,
|
| 18 |
+
256,
|
| 19 |
+
256,
|
| 20 |
+
768,
|
| 21 |
+
1024,
|
| 22 |
+
128,
|
| 23 |
+
768,
|
| 24 |
+
384,
|
| 25 |
+
128,
|
| 26 |
+
1024,
|
| 27 |
+
384,
|
| 28 |
+
384,
|
| 29 |
+
128,
|
| 30 |
+
640,
|
| 31 |
+
896,
|
| 32 |
+
896,
|
| 33 |
+
256,
|
| 34 |
+
768,
|
| 35 |
+
128,
|
| 36 |
+
768,
|
| 37 |
+
768,
|
| 38 |
+
768,
|
| 39 |
+
768,
|
| 40 |
+
640,
|
| 41 |
+
896,
|
| 42 |
+
256,
|
| 43 |
+
128,
|
| 44 |
+
384,
|
| 45 |
+
896,
|
| 46 |
+
256,
|
| 47 |
+
768,
|
| 48 |
+
512,
|
| 49 |
+
640,
|
| 50 |
+
768,
|
| 51 |
+
896,
|
| 52 |
+
1024,
|
| 53 |
+
384,
|
| 54 |
+
384,
|
| 55 |
+
512,
|
| 56 |
+
384,
|
| 57 |
+
256,
|
| 58 |
+
768,
|
| 59 |
+
640,
|
| 60 |
+
896,
|
| 61 |
+
896,
|
| 62 |
+
512,
|
| 63 |
+
768,
|
| 64 |
+
512,
|
| 65 |
+
384,
|
| 66 |
+
1024,
|
| 67 |
+
896,
|
| 68 |
+
896,
|
| 69 |
+
768,
|
| 70 |
+
128
|
| 71 |
+
],
|
| 72 |
+
[
|
| 73 |
+
768,
|
| 74 |
+
896,
|
| 75 |
+
896,
|
| 76 |
+
256,
|
| 77 |
+
896,
|
| 78 |
+
384,
|
| 79 |
+
768,
|
| 80 |
+
768,
|
| 81 |
+
1024,
|
| 82 |
+
384,
|
| 83 |
+
512,
|
| 84 |
+
1024,
|
| 85 |
+
768,
|
| 86 |
+
768,
|
| 87 |
+
1024,
|
| 88 |
+
640,
|
| 89 |
+
128,
|
| 90 |
+
768,
|
| 91 |
+
384,
|
| 92 |
+
768,
|
| 93 |
+
128,
|
| 94 |
+
256,
|
| 95 |
+
512,
|
| 96 |
+
384,
|
| 97 |
+
896,
|
| 98 |
+
256,
|
| 99 |
+
1024,
|
| 100 |
+
1024,
|
| 101 |
+
896,
|
| 102 |
+
1024,
|
| 103 |
+
896,
|
| 104 |
+
768,
|
| 105 |
+
256,
|
| 106 |
+
768,
|
| 107 |
+
128,
|
| 108 |
+
1024,
|
| 109 |
+
768,
|
| 110 |
+
768,
|
| 111 |
+
640,
|
| 112 |
+
384,
|
| 113 |
+
1024,
|
| 114 |
+
384,
|
| 115 |
+
640,
|
| 116 |
+
896,
|
| 117 |
+
1024,
|
| 118 |
+
512,
|
| 119 |
+
512,
|
| 120 |
+
512,
|
| 121 |
+
640
|
| 122 |
+
],
|
| 123 |
+
[
|
| 124 |
+
768,
|
| 125 |
+
1024,
|
| 126 |
+
512,
|
| 127 |
+
384,
|
| 128 |
+
384,
|
| 129 |
+
768,
|
| 130 |
+
256,
|
| 131 |
+
384,
|
| 132 |
+
1024,
|
| 133 |
+
896,
|
| 134 |
+
640,
|
| 135 |
+
256,
|
| 136 |
+
1024,
|
| 137 |
+
1024,
|
| 138 |
+
1024,
|
| 139 |
+
640,
|
| 140 |
+
1024,
|
| 141 |
+
896,
|
| 142 |
+
512,
|
| 143 |
+
768,
|
| 144 |
+
128,
|
| 145 |
+
768,
|
| 146 |
+
1024,
|
| 147 |
+
768,
|
| 148 |
+
256,
|
| 149 |
+
1024,
|
| 150 |
+
896,
|
| 151 |
+
1024,
|
| 152 |
+
896,
|
| 153 |
+
512,
|
| 154 |
+
640,
|
| 155 |
+
384,
|
| 156 |
+
896,
|
| 157 |
+
384,
|
| 158 |
+
512,
|
| 159 |
+
640,
|
| 160 |
+
256,
|
| 161 |
+
768,
|
| 162 |
+
256,
|
| 163 |
+
768,
|
| 164 |
+
768,
|
| 165 |
+
640,
|
| 166 |
+
1024,
|
| 167 |
+
640,
|
| 168 |
+
896,
|
| 169 |
+
768,
|
| 170 |
+
768,
|
| 171 |
+
256
|
| 172 |
+
],
|
| 173 |
+
[
|
| 174 |
+
640,
|
| 175 |
+
640,
|
| 176 |
+
896,
|
| 177 |
+
512,
|
| 178 |
+
640,
|
| 179 |
+
896,
|
| 180 |
+
896,
|
| 181 |
+
384,
|
| 182 |
+
896,
|
| 183 |
+
896,
|
| 184 |
+
384,
|
| 185 |
+
896,
|
| 186 |
+
640,
|
| 187 |
+
256,
|
| 188 |
+
640,
|
| 189 |
+
1024,
|
| 190 |
+
1024,
|
| 191 |
+
768,
|
| 192 |
+
1024,
|
| 193 |
+
896,
|
| 194 |
+
1024,
|
| 195 |
+
768,
|
| 196 |
+
896,
|
| 197 |
+
256,
|
| 198 |
+
512,
|
| 199 |
+
768,
|
| 200 |
+
1024,
|
| 201 |
+
256,
|
| 202 |
+
768,
|
| 203 |
+
512,
|
| 204 |
+
256,
|
| 205 |
+
640,
|
| 206 |
+
1024,
|
| 207 |
+
1024,
|
| 208 |
+
512,
|
| 209 |
+
1024,
|
| 210 |
+
768,
|
| 211 |
+
256,
|
| 212 |
+
1024,
|
| 213 |
+
384,
|
| 214 |
+
896,
|
| 215 |
+
1024,
|
| 216 |
+
896,
|
| 217 |
+
1024,
|
| 218 |
+
384
|
| 219 |
+
],
|
| 220 |
+
[
|
| 221 |
+
256,
|
| 222 |
+
640,
|
| 223 |
+
640,
|
| 224 |
+
896,
|
| 225 |
+
768,
|
| 226 |
+
896,
|
| 227 |
+
768,
|
| 228 |
+
768,
|
| 229 |
+
896,
|
| 230 |
+
896,
|
| 231 |
+
1024,
|
| 232 |
+
256,
|
| 233 |
+
512,
|
| 234 |
+
1024,
|
| 235 |
+
640,
|
| 236 |
+
896,
|
| 237 |
+
512,
|
| 238 |
+
512,
|
| 239 |
+
384,
|
| 240 |
+
384,
|
| 241 |
+
256,
|
| 242 |
+
384,
|
| 243 |
+
384,
|
| 244 |
+
896,
|
| 245 |
+
896,
|
| 246 |
+
768,
|
| 247 |
+
640,
|
| 248 |
+
896,
|
| 249 |
+
768,
|
| 250 |
+
1024,
|
| 251 |
+
512,
|
| 252 |
+
640,
|
| 253 |
+
512,
|
| 254 |
+
640,
|
| 255 |
+
896,
|
| 256 |
+
512,
|
| 257 |
+
512,
|
| 258 |
+
384,
|
| 259 |
+
640,
|
| 260 |
+
896,
|
| 261 |
+
896,
|
| 262 |
+
896,
|
| 263 |
+
1024,
|
| 264 |
+
640,
|
| 265 |
+
1024,
|
| 266 |
+
640,
|
| 267 |
+
1024
|
| 268 |
+
],
|
| 269 |
+
[
|
| 270 |
+
1024,
|
| 271 |
+
512,
|
| 272 |
+
1024,
|
| 273 |
+
1024,
|
| 274 |
+
640,
|
| 275 |
+
896,
|
| 276 |
+
640,
|
| 277 |
+
1024,
|
| 278 |
+
896,
|
| 279 |
+
384,
|
| 280 |
+
1024,
|
| 281 |
+
128,
|
| 282 |
+
896,
|
| 283 |
+
768,
|
| 284 |
+
1024,
|
| 285 |
+
768,
|
| 286 |
+
640,
|
| 287 |
+
896,
|
| 288 |
+
768,
|
| 289 |
+
640,
|
| 290 |
+
512,
|
| 291 |
+
896,
|
| 292 |
+
512,
|
| 293 |
+
640,
|
| 294 |
+
256,
|
| 295 |
+
768,
|
| 296 |
+
640,
|
| 297 |
+
768,
|
| 298 |
+
384,
|
| 299 |
+
896,
|
| 300 |
+
512,
|
| 301 |
+
512,
|
| 302 |
+
256,
|
| 303 |
+
512,
|
| 304 |
+
896,
|
| 305 |
+
256,
|
| 306 |
+
384,
|
| 307 |
+
640,
|
| 308 |
+
512,
|
| 309 |
+
640,
|
| 310 |
+
896,
|
| 311 |
+
512,
|
| 312 |
+
1024,
|
| 313 |
+
256,
|
| 314 |
+
768,
|
| 315 |
+
1024,
|
| 316 |
+
768,
|
| 317 |
+
256,
|
| 318 |
+
256
|
| 319 |
+
],
|
| 320 |
+
[
|
| 321 |
+
640,
|
| 322 |
+
896,
|
| 323 |
+
1024,
|
| 324 |
+
896,
|
| 325 |
+
1024,
|
| 326 |
+
1024,
|
| 327 |
+
1024,
|
| 328 |
+
512,
|
| 329 |
+
256,
|
| 330 |
+
256,
|
| 331 |
+
1024,
|
| 332 |
+
768,
|
| 333 |
+
512,
|
| 334 |
+
768,
|
| 335 |
+
1024,
|
| 336 |
+
1024,
|
| 337 |
+
1024,
|
| 338 |
+
384,
|
| 339 |
+
512,
|
| 340 |
+
1024,
|
| 341 |
+
512,
|
| 342 |
+
1024,
|
| 343 |
+
128,
|
| 344 |
+
640,
|
| 345 |
+
640,
|
| 346 |
+
896,
|
| 347 |
+
768,
|
| 348 |
+
128,
|
| 349 |
+
256,
|
| 350 |
+
256,
|
| 351 |
+
256,
|
| 352 |
+
256,
|
| 353 |
+
896,
|
| 354 |
+
1024,
|
| 355 |
+
1024,
|
| 356 |
+
384,
|
| 357 |
+
896,
|
| 358 |
+
256,
|
| 359 |
+
896,
|
| 360 |
+
640,
|
| 361 |
+
1024,
|
| 362 |
+
384,
|
| 363 |
+
640,
|
| 364 |
+
256,
|
| 365 |
+
1024,
|
| 366 |
+
1024,
|
| 367 |
+
1024
|
| 368 |
+
],
|
| 369 |
+
[
|
| 370 |
+
1024,
|
| 371 |
+
384,
|
| 372 |
+
1024,
|
| 373 |
+
1024,
|
| 374 |
+
256,
|
| 375 |
+
128,
|
| 376 |
+
256,
|
| 377 |
+
384,
|
| 378 |
+
256,
|
| 379 |
+
384,
|
| 380 |
+
896,
|
| 381 |
+
768,
|
| 382 |
+
896,
|
| 383 |
+
896,
|
| 384 |
+
512,
|
| 385 |
+
896,
|
| 386 |
+
640,
|
| 387 |
+
384,
|
| 388 |
+
384,
|
| 389 |
+
896,
|
| 390 |
+
768,
|
| 391 |
+
384,
|
| 392 |
+
896,
|
| 393 |
+
768,
|
| 394 |
+
768,
|
| 395 |
+
512,
|
| 396 |
+
896,
|
| 397 |
+
768,
|
| 398 |
+
768,
|
| 399 |
+
896,
|
| 400 |
+
128,
|
| 401 |
+
896,
|
| 402 |
+
512,
|
| 403 |
+
256,
|
| 404 |
+
768,
|
| 405 |
+
128,
|
| 406 |
+
384,
|
| 407 |
+
256,
|
| 408 |
+
896,
|
| 409 |
+
896,
|
| 410 |
+
384,
|
| 411 |
+
768,
|
| 412 |
+
512,
|
| 413 |
+
640,
|
| 414 |
+
256,
|
| 415 |
+
768,
|
| 416 |
+
640,
|
| 417 |
+
896,
|
| 418 |
+
384,
|
| 419 |
+
512,
|
| 420 |
+
1024,
|
| 421 |
+
768,
|
| 422 |
+
384
|
| 423 |
+
],
|
| 424 |
+
[
|
| 425 |
+
512,
|
| 426 |
+
768,
|
| 427 |
+
512,
|
| 428 |
+
256,
|
| 429 |
+
128,
|
| 430 |
+
640,
|
| 431 |
+
384,
|
| 432 |
+
640,
|
| 433 |
+
768,
|
| 434 |
+
896,
|
| 435 |
+
640,
|
| 436 |
+
768,
|
| 437 |
+
256,
|
| 438 |
+
384,
|
| 439 |
+
1024,
|
| 440 |
+
896,
|
| 441 |
+
256,
|
| 442 |
+
896,
|
| 443 |
+
512,
|
| 444 |
+
256,
|
| 445 |
+
896,
|
| 446 |
+
768,
|
| 447 |
+
256,
|
| 448 |
+
896,
|
| 449 |
+
896,
|
| 450 |
+
384,
|
| 451 |
+
896,
|
| 452 |
+
640,
|
| 453 |
+
768,
|
| 454 |
+
512,
|
| 455 |
+
768,
|
| 456 |
+
768,
|
| 457 |
+
1024,
|
| 458 |
+
768,
|
| 459 |
+
640,
|
| 460 |
+
768,
|
| 461 |
+
384,
|
| 462 |
+
256,
|
| 463 |
+
512,
|
| 464 |
+
896,
|
| 465 |
+
128,
|
| 466 |
+
384,
|
| 467 |
+
256,
|
| 468 |
+
768,
|
| 469 |
+
384,
|
| 470 |
+
256,
|
| 471 |
+
1024,
|
| 472 |
+
1024,
|
| 473 |
+
896,
|
| 474 |
+
256,
|
| 475 |
+
1024,
|
| 476 |
+
256,
|
| 477 |
+
128,
|
| 478 |
+
896
|
| 479 |
+
],
|
| 480 |
+
[
|
| 481 |
+
640,
|
| 482 |
+
640,
|
| 483 |
+
896,
|
| 484 |
+
256,
|
| 485 |
+
1024,
|
| 486 |
+
512,
|
| 487 |
+
1024,
|
| 488 |
+
768,
|
| 489 |
+
384,
|
| 490 |
+
512,
|
| 491 |
+
256,
|
| 492 |
+
768,
|
| 493 |
+
896,
|
| 494 |
+
768,
|
| 495 |
+
512,
|
| 496 |
+
768,
|
| 497 |
+
768,
|
| 498 |
+
640,
|
| 499 |
+
384,
|
| 500 |
+
768,
|
| 501 |
+
512,
|
| 502 |
+
768,
|
| 503 |
+
768,
|
| 504 |
+
512,
|
| 505 |
+
768,
|
| 506 |
+
128,
|
| 507 |
+
896,
|
| 508 |
+
512,
|
| 509 |
+
768,
|
| 510 |
+
1024,
|
| 511 |
+
128,
|
| 512 |
+
384,
|
| 513 |
+
768,
|
| 514 |
+
768,
|
| 515 |
+
768,
|
| 516 |
+
384,
|
| 517 |
+
512,
|
| 518 |
+
640,
|
| 519 |
+
768,
|
| 520 |
+
512,
|
| 521 |
+
768,
|
| 522 |
+
1024,
|
| 523 |
+
640,
|
| 524 |
+
896,
|
| 525 |
+
256,
|
| 526 |
+
1024,
|
| 527 |
+
384,
|
| 528 |
+
768,
|
| 529 |
+
768,
|
| 530 |
+
768
|
| 531 |
+
],
|
| 532 |
+
[
|
| 533 |
+
896,
|
| 534 |
+
512,
|
| 535 |
+
896,
|
| 536 |
+
768,
|
| 537 |
+
384,
|
| 538 |
+
384,
|
| 539 |
+
768,
|
| 540 |
+
512,
|
| 541 |
+
768,
|
| 542 |
+
512,
|
| 543 |
+
1024,
|
| 544 |
+
640,
|
| 545 |
+
896,
|
| 546 |
+
896,
|
| 547 |
+
256,
|
| 548 |
+
640,
|
| 549 |
+
1024,
|
| 550 |
+
256,
|
| 551 |
+
896,
|
| 552 |
+
128,
|
| 553 |
+
128,
|
| 554 |
+
128,
|
| 555 |
+
768,
|
| 556 |
+
896,
|
| 557 |
+
384,
|
| 558 |
+
896,
|
| 559 |
+
512,
|
| 560 |
+
896,
|
| 561 |
+
384,
|
| 562 |
+
256,
|
| 563 |
+
640,
|
| 564 |
+
640,
|
| 565 |
+
896,
|
| 566 |
+
768,
|
| 567 |
+
640,
|
| 568 |
+
256,
|
| 569 |
+
896,
|
| 570 |
+
896,
|
| 571 |
+
512,
|
| 572 |
+
128,
|
| 573 |
+
896,
|
| 574 |
+
256,
|
| 575 |
+
256,
|
| 576 |
+
640,
|
| 577 |
+
896,
|
| 578 |
+
896,
|
| 579 |
+
128,
|
| 580 |
+
1024,
|
| 581 |
+
256,
|
| 582 |
+
384,
|
| 583 |
+
1024,
|
| 584 |
+
640,
|
| 585 |
+
896
|
| 586 |
+
],
|
| 587 |
+
[
|
| 588 |
+
768,
|
| 589 |
+
384,
|
| 590 |
+
640,
|
| 591 |
+
896,
|
| 592 |
+
256,
|
| 593 |
+
128,
|
| 594 |
+
384,
|
| 595 |
+
896,
|
| 596 |
+
128,
|
| 597 |
+
128,
|
| 598 |
+
896,
|
| 599 |
+
256,
|
| 600 |
+
384,
|
| 601 |
+
896,
|
| 602 |
+
512,
|
| 603 |
+
768,
|
| 604 |
+
768,
|
| 605 |
+
512,
|
| 606 |
+
512,
|
| 607 |
+
768,
|
| 608 |
+
896,
|
| 609 |
+
640,
|
| 610 |
+
768,
|
| 611 |
+
896,
|
| 612 |
+
896,
|
| 613 |
+
896,
|
| 614 |
+
640,
|
| 615 |
+
896,
|
| 616 |
+
640,
|
| 617 |
+
512,
|
| 618 |
+
896,
|
| 619 |
+
256,
|
| 620 |
+
512,
|
| 621 |
+
128,
|
| 622 |
+
512,
|
| 623 |
+
384,
|
| 624 |
+
768,
|
| 625 |
+
768,
|
| 626 |
+
1024,
|
| 627 |
+
256,
|
| 628 |
+
768,
|
| 629 |
+
256,
|
| 630 |
+
768,
|
| 631 |
+
512,
|
| 632 |
+
640,
|
| 633 |
+
1024,
|
| 634 |
+
128,
|
| 635 |
+
896,
|
| 636 |
+
896,
|
| 637 |
+
896,
|
| 638 |
+
896,
|
| 639 |
+
1024
|
| 640 |
+
],
|
| 641 |
+
[
|
| 642 |
+
512,
|
| 643 |
+
384,
|
| 644 |
+
768,
|
| 645 |
+
640,
|
| 646 |
+
640,
|
| 647 |
+
768,
|
| 648 |
+
1024,
|
| 649 |
+
896,
|
| 650 |
+
512,
|
| 651 |
+
256,
|
| 652 |
+
640,
|
| 653 |
+
768,
|
| 654 |
+
640,
|
| 655 |
+
896,
|
| 656 |
+
128,
|
| 657 |
+
256,
|
| 658 |
+
896,
|
| 659 |
+
1024,
|
| 660 |
+
256,
|
| 661 |
+
640,
|
| 662 |
+
512,
|
| 663 |
+
256,
|
| 664 |
+
128,
|
| 665 |
+
512,
|
| 666 |
+
256,
|
| 667 |
+
640,
|
| 668 |
+
768,
|
| 669 |
+
768,
|
| 670 |
+
128,
|
| 671 |
+
128,
|
| 672 |
+
768,
|
| 673 |
+
640,
|
| 674 |
+
1024,
|
| 675 |
+
1024,
|
| 676 |
+
768,
|
| 677 |
+
512,
|
| 678 |
+
896,
|
| 679 |
+
768,
|
| 680 |
+
896,
|
| 681 |
+
1024,
|
| 682 |
+
896,
|
| 683 |
+
896,
|
| 684 |
+
512,
|
| 685 |
+
640,
|
| 686 |
+
1024,
|
| 687 |
+
512,
|
| 688 |
+
1024,
|
| 689 |
+
512,
|
| 690 |
+
512,
|
| 691 |
+
512,
|
| 692 |
+
768
|
| 693 |
+
],
|
| 694 |
+
[
|
| 695 |
+
896,
|
| 696 |
+
896,
|
| 697 |
+
1024,
|
| 698 |
+
1024,
|
| 699 |
+
896,
|
| 700 |
+
128,
|
| 701 |
+
768,
|
| 702 |
+
256,
|
| 703 |
+
1024,
|
| 704 |
+
256,
|
| 705 |
+
1024,
|
| 706 |
+
640,
|
| 707 |
+
384,
|
| 708 |
+
256,
|
| 709 |
+
256,
|
| 710 |
+
512,
|
| 711 |
+
768,
|
| 712 |
+
896,
|
| 713 |
+
512,
|
| 714 |
+
768,
|
| 715 |
+
384,
|
| 716 |
+
1024,
|
| 717 |
+
896,
|
| 718 |
+
896,
|
| 719 |
+
1024,
|
| 720 |
+
896,
|
| 721 |
+
768,
|
| 722 |
+
896,
|
| 723 |
+
640,
|
| 724 |
+
1024,
|
| 725 |
+
512,
|
| 726 |
+
896,
|
| 727 |
+
512,
|
| 728 |
+
1024,
|
| 729 |
+
512,
|
| 730 |
+
512,
|
| 731 |
+
256,
|
| 732 |
+
256,
|
| 733 |
+
256,
|
| 734 |
+
512,
|
| 735 |
+
768,
|
| 736 |
+
128,
|
| 737 |
+
384,
|
| 738 |
+
512,
|
| 739 |
+
896,
|
| 740 |
+
896,
|
| 741 |
+
1024,
|
| 742 |
+
256,
|
| 743 |
+
384,
|
| 744 |
+
640
|
| 745 |
+
],
|
| 746 |
+
[
|
| 747 |
+
896,
|
| 748 |
+
640,
|
| 749 |
+
384,
|
| 750 |
+
512,
|
| 751 |
+
256,
|
| 752 |
+
640,
|
| 753 |
+
1024,
|
| 754 |
+
384,
|
| 755 |
+
1024,
|
| 756 |
+
1024,
|
| 757 |
+
768,
|
| 758 |
+
256,
|
| 759 |
+
1024,
|
| 760 |
+
768,
|
| 761 |
+
512,
|
| 762 |
+
896,
|
| 763 |
+
256,
|
| 764 |
+
1024,
|
| 765 |
+
768,
|
| 766 |
+
768,
|
| 767 |
+
768,
|
| 768 |
+
384,
|
| 769 |
+
384,
|
| 770 |
+
256,
|
| 771 |
+
1024,
|
| 772 |
+
384,
|
| 773 |
+
384,
|
| 774 |
+
384,
|
| 775 |
+
896,
|
| 776 |
+
768,
|
| 777 |
+
640,
|
| 778 |
+
768,
|
| 779 |
+
512,
|
| 780 |
+
896,
|
| 781 |
+
896,
|
| 782 |
+
896,
|
| 783 |
+
896,
|
| 784 |
+
256,
|
| 785 |
+
384,
|
| 786 |
+
128,
|
| 787 |
+
1024,
|
| 788 |
+
896,
|
| 789 |
+
256,
|
| 790 |
+
256,
|
| 791 |
+
768,
|
| 792 |
+
640,
|
| 793 |
+
896,
|
| 794 |
+
384,
|
| 795 |
+
768,
|
| 796 |
+
512,
|
| 797 |
+
640
|
| 798 |
+
],
|
| 799 |
+
[
|
| 800 |
+
896,
|
| 801 |
+
1024,
|
| 802 |
+
768,
|
| 803 |
+
1024,
|
| 804 |
+
896,
|
| 805 |
+
256,
|
| 806 |
+
768,
|
| 807 |
+
128,
|
| 808 |
+
128,
|
| 809 |
+
768,
|
| 810 |
+
512,
|
| 811 |
+
896,
|
| 812 |
+
384,
|
| 813 |
+
768,
|
| 814 |
+
1024,
|
| 815 |
+
256,
|
| 816 |
+
768,
|
| 817 |
+
768,
|
| 818 |
+
256,
|
| 819 |
+
512,
|
| 820 |
+
512,
|
| 821 |
+
640,
|
| 822 |
+
512,
|
| 823 |
+
256,
|
| 824 |
+
768,
|
| 825 |
+
896,
|
| 826 |
+
384,
|
| 827 |
+
1024,
|
| 828 |
+
640,
|
| 829 |
+
1024,
|
| 830 |
+
512,
|
| 831 |
+
512,
|
| 832 |
+
384,
|
| 833 |
+
512,
|
| 834 |
+
512,
|
| 835 |
+
1024,
|
| 836 |
+
384,
|
| 837 |
+
896,
|
| 838 |
+
768,
|
| 839 |
+
384,
|
| 840 |
+
384,
|
| 841 |
+
128,
|
| 842 |
+
384,
|
| 843 |
+
1024,
|
| 844 |
+
896,
|
| 845 |
+
640,
|
| 846 |
+
768,
|
| 847 |
+
768,
|
| 848 |
+
256,
|
| 849 |
+
640,
|
| 850 |
+
512,
|
| 851 |
+
640,
|
| 852 |
+
384
|
| 853 |
+
]
|
| 854 |
+
],
|
| 855 |
+
"glean_metadata": {
|
| 856 |
+
"base_model": "allenai/OLMoE-1B-7B-0125-Instruct",
|
| 857 |
+
"block_size": 128,
|
| 858 |
+
"criterion": "reap",
|
| 859 |
+
"dead_experts": 217,
|
| 860 |
+
"keep_fraction": 0.5,
|
| 861 |
+
"min_width": 128,
|
| 862 |
+
"params": 3697491968,
|
| 863 |
+
"scores": "outputs/scores_0125inst_dolmino-math/scores.pt"
|
| 864 |
+
},
|
| 865 |
+
"hidden_act": "silu",
|
| 866 |
+
"hidden_size": 2048,
|
| 867 |
+
"initializer_range": 0.02,
|
| 868 |
+
"intermediate_size": 1024,
|
| 869 |
+
"max_position_embeddings": 4096,
|
| 870 |
+
"model_type": "pruned_olmoe",
|
| 871 |
+
"norm_topk_prob": false,
|
| 872 |
+
"num_attention_heads": 16,
|
| 873 |
+
"num_experts": 64,
|
| 874 |
+
"num_experts_per_tok": 8,
|
| 875 |
+
"num_hidden_layers": 16,
|
| 876 |
+
"num_key_value_heads": 16,
|
| 877 |
+
"output_router_logits": false,
|
| 878 |
+
"pad_token_id": 1,
|
| 879 |
+
"rms_norm_eps": 1e-05,
|
| 880 |
+
"rope_scaling": null,
|
| 881 |
+
"rope_theta": 10000.0,
|
| 882 |
+
"router_aux_loss_coef": 0.01,
|
| 883 |
+
"tie_word_embeddings": false,
|
| 884 |
+
"transformers_version": "4.57.6",
|
| 885 |
+
"use_cache": false,
|
| 886 |
+
"vocab_size": 50304
|
| 887 |
+
}
|
healed/liger_smoke/step0006/configuration_pruned_olmoe.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Configuration for GLEAN-pruned OLMoE: variable-width, variable-count experts.
|
| 2 |
+
"""
|
| 3 |
+
|
| 4 |
+
from transformers.models.olmoe.configuration_olmoe import OlmoeConfig
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
class PrunedOlmoeConfig(OlmoeConfig):
|
| 8 |
+
"""OlmoeConfig plus a per-(layer, expert) width table.
|
| 9 |
+
|
| 10 |
+
``expert_widths[l]`` lists the SwiGLU intermediate width of each surviving
|
| 11 |
+
expert in decoder layer ``l``, in expert order. Lists are ragged: layers
|
| 12 |
+
may keep different numbers of experts (deleted experts simply don't
|
| 13 |
+
appear — the router in layer ``l`` has ``len(expert_widths[l])`` rows),
|
| 14 |
+
and each width may differ (multiples of the GEMM block size, 128, for
|
| 15 |
+
variable-MegaBlocks execution). ``None`` means an unpruned model
|
| 16 |
+
(uniform ``num_experts`` × ``intermediate_size``).
|
| 17 |
+
|
| 18 |
+
The inherited ``num_experts`` / ``intermediate_size`` keep their ORIGINAL
|
| 19 |
+
(pre-pruning) values for provenance; the width table is authoritative for
|
| 20 |
+
the built architecture.
|
| 21 |
+
"""
|
| 22 |
+
|
| 23 |
+
model_type = "pruned_olmoe"
|
| 24 |
+
|
| 25 |
+
def __init__(self, expert_widths: list[list[int]] | None = None, **kwargs):
|
| 26 |
+
super().__init__(**kwargs)
|
| 27 |
+
self.expert_widths = expert_widths
|
healed/liger_smoke/step0006/generation_config.json
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_from_model_config": true,
|
| 3 |
+
"eos_token_id": 50279,
|
| 4 |
+
"pad_token_id": 1,
|
| 5 |
+
"transformers_version": "4.57.6"
|
| 6 |
+
}
|
healed/liger_smoke/step0006/model.safetensors.index.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
healed/liger_smoke/step0006/modeling_pruned_olmoe.py
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""GLEAN-pruned OLMoE: HF-loadable model with ragged (variable-width) experts.
|
| 2 |
+
|
| 3 |
+
Pattern follows hbfreed/variable-flex-olmo's PrunedFlexOlmoForCausalLM
|
| 4 |
+
(docs/recon/prior-work-hbfreed.md), generalized from one scalar width to a
|
| 5 |
+
per-(layer, expert) width table: ``super().__init__`` builds the uniform
|
| 6 |
+
architecture from the config, then every MoE block is rebuilt to its pruned
|
| 7 |
+
shape — surviving experts only, each at its own width, router sliced to
|
| 8 |
+
match — so the state dict aligns exactly with what
|
| 9 |
+
``glean.prune.prune_channels_global`` leaves behind.
|
| 10 |
+
|
| 11 |
+
Caveat: ``output_router_logits=True`` (the load-balancing aux loss) assumes a
|
| 12 |
+
uniform ``config.num_experts`` and is unsupported on ragged models.
|
| 13 |
+
"""
|
| 14 |
+
|
| 15 |
+
import torch.nn as nn
|
| 16 |
+
from transformers.activations import ACT2FN
|
| 17 |
+
from transformers.models.olmoe.modeling_olmoe import OlmoeForCausalLM
|
| 18 |
+
|
| 19 |
+
from .configuration_pruned_olmoe import PrunedOlmoeConfig
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
class RaggedOlmoeMLP(nn.Module):
|
| 23 |
+
"""OlmoeMLP with an explicit intermediate width (SwiGLU, no biases)."""
|
| 24 |
+
|
| 25 |
+
def __init__(self, hidden_size: int, intermediate_size: int, hidden_act: str):
|
| 26 |
+
super().__init__()
|
| 27 |
+
self.hidden_size = hidden_size
|
| 28 |
+
self.intermediate_size = intermediate_size
|
| 29 |
+
self.gate_proj = nn.Linear(hidden_size, intermediate_size, bias=False)
|
| 30 |
+
self.up_proj = nn.Linear(hidden_size, intermediate_size, bias=False)
|
| 31 |
+
self.down_proj = nn.Linear(intermediate_size, hidden_size, bias=False)
|
| 32 |
+
self.act_fn = ACT2FN[hidden_act]
|
| 33 |
+
|
| 34 |
+
def forward(self, x):
|
| 35 |
+
return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
class PrunedOlmoeForCausalLM(OlmoeForCausalLM):
|
| 39 |
+
"""OLMoE with per-layer surviving-expert lists at per-expert widths."""
|
| 40 |
+
|
| 41 |
+
config_class = PrunedOlmoeConfig
|
| 42 |
+
|
| 43 |
+
def __init__(self, config: PrunedOlmoeConfig):
|
| 44 |
+
super().__init__(config)
|
| 45 |
+
widths_table = getattr(config, "expert_widths", None)
|
| 46 |
+
if widths_table is None:
|
| 47 |
+
return # unpruned: plain OLMoE
|
| 48 |
+
if len(widths_table) != len(self.model.layers):
|
| 49 |
+
raise ValueError(
|
| 50 |
+
f"expert_widths has {len(widths_table)} rows but the model has "
|
| 51 |
+
f"{len(self.model.layers)} decoder layers"
|
| 52 |
+
)
|
| 53 |
+
for layer, widths in zip(self.model.layers, widths_table):
|
| 54 |
+
if any(w <= 0 for w in widths):
|
| 55 |
+
raise ValueError("expert_widths must list surviving experts only (>0)")
|
| 56 |
+
block = layer.mlp
|
| 57 |
+
if len(widths) < block.top_k:
|
| 58 |
+
raise ValueError(
|
| 59 |
+
f"a layer keeps {len(widths)} experts < top_k={block.top_k}"
|
| 60 |
+
)
|
| 61 |
+
block.num_experts = len(widths)
|
| 62 |
+
block.gate = nn.Linear(config.hidden_size, len(widths), bias=False)
|
| 63 |
+
block.experts = nn.ModuleList(
|
| 64 |
+
RaggedOlmoeMLP(config.hidden_size, w, config.hidden_act)
|
| 65 |
+
for w in widths
|
| 66 |
+
)
|
healed/liger_smoke/step0006/special_tokens_map.json
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"bos_token": {
|
| 3 |
+
"content": "|||IP_ADDRESS|||",
|
| 4 |
+
"lstrip": false,
|
| 5 |
+
"normalized": true,
|
| 6 |
+
"rstrip": false,
|
| 7 |
+
"single_word": false
|
| 8 |
+
},
|
| 9 |
+
"eos_token": {
|
| 10 |
+
"content": "|||IP_ADDRESS|||",
|
| 11 |
+
"lstrip": false,
|
| 12 |
+
"normalized": true,
|
| 13 |
+
"rstrip": false,
|
| 14 |
+
"single_word": false
|
| 15 |
+
},
|
| 16 |
+
"pad_token": {
|
| 17 |
+
"content": "<pad>",
|
| 18 |
+
"lstrip": false,
|
| 19 |
+
"normalized": false,
|
| 20 |
+
"rstrip": false,
|
| 21 |
+
"single_word": false
|
| 22 |
+
}
|
| 23 |
+
}
|
healed/liger_smoke/step0006/tokenizer.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
healed/liger_smoke/step0006/tokenizer_config.json
ADDED
|
@@ -0,0 +1,247 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"add_bos_token": false,
|
| 3 |
+
"add_eos_token": false,
|
| 4 |
+
"add_prefix_space": false,
|
| 5 |
+
"added_tokens_decoder": {
|
| 6 |
+
"0": {
|
| 7 |
+
"content": "<|endoftext|>",
|
| 8 |
+
"lstrip": false,
|
| 9 |
+
"normalized": false,
|
| 10 |
+
"rstrip": false,
|
| 11 |
+
"single_word": false,
|
| 12 |
+
"special": true
|
| 13 |
+
},
|
| 14 |
+
"1": {
|
| 15 |
+
"content": "<|padding|>",
|
| 16 |
+
"lstrip": false,
|
| 17 |
+
"normalized": false,
|
| 18 |
+
"rstrip": false,
|
| 19 |
+
"single_word": false,
|
| 20 |
+
"special": true
|
| 21 |
+
},
|
| 22 |
+
"50254": {
|
| 23 |
+
"content": " ",
|
| 24 |
+
"lstrip": false,
|
| 25 |
+
"normalized": true,
|
| 26 |
+
"rstrip": false,
|
| 27 |
+
"single_word": false,
|
| 28 |
+
"special": false
|
| 29 |
+
},
|
| 30 |
+
"50255": {
|
| 31 |
+
"content": " ",
|
| 32 |
+
"lstrip": false,
|
| 33 |
+
"normalized": true,
|
| 34 |
+
"rstrip": false,
|
| 35 |
+
"single_word": false,
|
| 36 |
+
"special": false
|
| 37 |
+
},
|
| 38 |
+
"50256": {
|
| 39 |
+
"content": " ",
|
| 40 |
+
"lstrip": false,
|
| 41 |
+
"normalized": true,
|
| 42 |
+
"rstrip": false,
|
| 43 |
+
"single_word": false,
|
| 44 |
+
"special": false
|
| 45 |
+
},
|
| 46 |
+
"50257": {
|
| 47 |
+
"content": " ",
|
| 48 |
+
"lstrip": false,
|
| 49 |
+
"normalized": true,
|
| 50 |
+
"rstrip": false,
|
| 51 |
+
"single_word": false,
|
| 52 |
+
"special": false
|
| 53 |
+
},
|
| 54 |
+
"50258": {
|
| 55 |
+
"content": " ",
|
| 56 |
+
"lstrip": false,
|
| 57 |
+
"normalized": true,
|
| 58 |
+
"rstrip": false,
|
| 59 |
+
"single_word": false,
|
| 60 |
+
"special": false
|
| 61 |
+
},
|
| 62 |
+
"50259": {
|
| 63 |
+
"content": " ",
|
| 64 |
+
"lstrip": false,
|
| 65 |
+
"normalized": true,
|
| 66 |
+
"rstrip": false,
|
| 67 |
+
"single_word": false,
|
| 68 |
+
"special": false
|
| 69 |
+
},
|
| 70 |
+
"50260": {
|
| 71 |
+
"content": " ",
|
| 72 |
+
"lstrip": false,
|
| 73 |
+
"normalized": true,
|
| 74 |
+
"rstrip": false,
|
| 75 |
+
"single_word": false,
|
| 76 |
+
"special": false
|
| 77 |
+
},
|
| 78 |
+
"50261": {
|
| 79 |
+
"content": " ",
|
| 80 |
+
"lstrip": false,
|
| 81 |
+
"normalized": true,
|
| 82 |
+
"rstrip": false,
|
| 83 |
+
"single_word": false,
|
| 84 |
+
"special": false
|
| 85 |
+
},
|
| 86 |
+
"50262": {
|
| 87 |
+
"content": " ",
|
| 88 |
+
"lstrip": false,
|
| 89 |
+
"normalized": true,
|
| 90 |
+
"rstrip": false,
|
| 91 |
+
"single_word": false,
|
| 92 |
+
"special": false
|
| 93 |
+
},
|
| 94 |
+
"50263": {
|
| 95 |
+
"content": " ",
|
| 96 |
+
"lstrip": false,
|
| 97 |
+
"normalized": true,
|
| 98 |
+
"rstrip": false,
|
| 99 |
+
"single_word": false,
|
| 100 |
+
"special": false
|
| 101 |
+
},
|
| 102 |
+
"50264": {
|
| 103 |
+
"content": " ",
|
| 104 |
+
"lstrip": false,
|
| 105 |
+
"normalized": true,
|
| 106 |
+
"rstrip": false,
|
| 107 |
+
"single_word": false,
|
| 108 |
+
"special": false
|
| 109 |
+
},
|
| 110 |
+
"50265": {
|
| 111 |
+
"content": " ",
|
| 112 |
+
"lstrip": false,
|
| 113 |
+
"normalized": true,
|
| 114 |
+
"rstrip": false,
|
| 115 |
+
"single_word": false,
|
| 116 |
+
"special": false
|
| 117 |
+
},
|
| 118 |
+
"50266": {
|
| 119 |
+
"content": " ",
|
| 120 |
+
"lstrip": false,
|
| 121 |
+
"normalized": true,
|
| 122 |
+
"rstrip": false,
|
| 123 |
+
"single_word": false,
|
| 124 |
+
"special": false
|
| 125 |
+
},
|
| 126 |
+
"50267": {
|
| 127 |
+
"content": " ",
|
| 128 |
+
"lstrip": false,
|
| 129 |
+
"normalized": true,
|
| 130 |
+
"rstrip": false,
|
| 131 |
+
"single_word": false,
|
| 132 |
+
"special": false
|
| 133 |
+
},
|
| 134 |
+
"50268": {
|
| 135 |
+
"content": " ",
|
| 136 |
+
"lstrip": false,
|
| 137 |
+
"normalized": true,
|
| 138 |
+
"rstrip": false,
|
| 139 |
+
"single_word": false,
|
| 140 |
+
"special": false
|
| 141 |
+
},
|
| 142 |
+
"50269": {
|
| 143 |
+
"content": " ",
|
| 144 |
+
"lstrip": false,
|
| 145 |
+
"normalized": true,
|
| 146 |
+
"rstrip": false,
|
| 147 |
+
"single_word": false,
|
| 148 |
+
"special": false
|
| 149 |
+
},
|
| 150 |
+
"50270": {
|
| 151 |
+
"content": " ",
|
| 152 |
+
"lstrip": false,
|
| 153 |
+
"normalized": true,
|
| 154 |
+
"rstrip": false,
|
| 155 |
+
"single_word": false,
|
| 156 |
+
"special": false
|
| 157 |
+
},
|
| 158 |
+
"50271": {
|
| 159 |
+
"content": " ",
|
| 160 |
+
"lstrip": false,
|
| 161 |
+
"normalized": true,
|
| 162 |
+
"rstrip": false,
|
| 163 |
+
"single_word": false,
|
| 164 |
+
"special": false
|
| 165 |
+
},
|
| 166 |
+
"50272": {
|
| 167 |
+
"content": " ",
|
| 168 |
+
"lstrip": false,
|
| 169 |
+
"normalized": true,
|
| 170 |
+
"rstrip": false,
|
| 171 |
+
"single_word": false,
|
| 172 |
+
"special": false
|
| 173 |
+
},
|
| 174 |
+
"50273": {
|
| 175 |
+
"content": " ",
|
| 176 |
+
"lstrip": false,
|
| 177 |
+
"normalized": true,
|
| 178 |
+
"rstrip": false,
|
| 179 |
+
"single_word": false,
|
| 180 |
+
"special": false
|
| 181 |
+
},
|
| 182 |
+
"50274": {
|
| 183 |
+
"content": " ",
|
| 184 |
+
"lstrip": false,
|
| 185 |
+
"normalized": true,
|
| 186 |
+
"rstrip": false,
|
| 187 |
+
"single_word": false,
|
| 188 |
+
"special": false
|
| 189 |
+
},
|
| 190 |
+
"50275": {
|
| 191 |
+
"content": " ",
|
| 192 |
+
"lstrip": false,
|
| 193 |
+
"normalized": true,
|
| 194 |
+
"rstrip": false,
|
| 195 |
+
"single_word": false,
|
| 196 |
+
"special": false
|
| 197 |
+
},
|
| 198 |
+
"50276": {
|
| 199 |
+
"content": " ",
|
| 200 |
+
"lstrip": false,
|
| 201 |
+
"normalized": true,
|
| 202 |
+
"rstrip": false,
|
| 203 |
+
"single_word": false,
|
| 204 |
+
"special": false
|
| 205 |
+
},
|
| 206 |
+
"50277": {
|
| 207 |
+
"content": "|||EMAIL_ADDRESS|||",
|
| 208 |
+
"lstrip": false,
|
| 209 |
+
"normalized": true,
|
| 210 |
+
"rstrip": false,
|
| 211 |
+
"single_word": false,
|
| 212 |
+
"special": false
|
| 213 |
+
},
|
| 214 |
+
"50278": {
|
| 215 |
+
"content": "|||PHONE_NUMBER|||",
|
| 216 |
+
"lstrip": false,
|
| 217 |
+
"normalized": true,
|
| 218 |
+
"rstrip": false,
|
| 219 |
+
"single_word": false,
|
| 220 |
+
"special": false
|
| 221 |
+
},
|
| 222 |
+
"50279": {
|
| 223 |
+
"content": "|||IP_ADDRESS|||",
|
| 224 |
+
"lstrip": false,
|
| 225 |
+
"normalized": true,
|
| 226 |
+
"rstrip": false,
|
| 227 |
+
"single_word": false,
|
| 228 |
+
"special": true
|
| 229 |
+
},
|
| 230 |
+
"50280": {
|
| 231 |
+
"content": "<pad>",
|
| 232 |
+
"lstrip": false,
|
| 233 |
+
"normalized": false,
|
| 234 |
+
"rstrip": false,
|
| 235 |
+
"single_word": false,
|
| 236 |
+
"special": true
|
| 237 |
+
}
|
| 238 |
+
},
|
| 239 |
+
"bos_token": "|||IP_ADDRESS|||",
|
| 240 |
+
"clean_up_tokenization_spaces": false,
|
| 241 |
+
"eos_token": "|||IP_ADDRESS|||",
|
| 242 |
+
"extra_special_tokens": {},
|
| 243 |
+
"model_max_length": 1000000000000000019884624838656,
|
| 244 |
+
"pad_token": "<pad>",
|
| 245 |
+
"tokenizer_class": "GPTNeoXTokenizer",
|
| 246 |
+
"unk_token": null
|
| 247 |
+
}
|
healed/mixonly_keep50/vllm_live/chat_template.jinja
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{{ bos_token }}{% for message in messages %}{% if message['role'] == 'system' %}{{ '<|system|>
|
| 2 |
+
' + message['content'] + '
|
| 3 |
+
' }}{% elif message['role'] == 'user' %}{{ '<|user|>
|
| 4 |
+
' + message['content'] + '
|
| 5 |
+
' }}{% elif message['role'] == 'assistant' %}{% if not loop.last %}{{ '<|assistant|>
|
| 6 |
+
' + message['content'] + eos_token + '
|
| 7 |
+
' }}{% else %}{{ '<|assistant|>
|
| 8 |
+
' + message['content'] + eos_token }}{% endif %}{% endif %}{% if loop.last and add_generation_prompt %}{{ '<|assistant|>
|
| 9 |
+
' }}{% endif %}{% endfor %}
|
healed/mixonly_keep50/vllm_live/config.json
ADDED
|
@@ -0,0 +1,887 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"architectures": [
|
| 3 |
+
"PrunedOlmoeForCausalLM"
|
| 4 |
+
],
|
| 5 |
+
"attention_bias": false,
|
| 6 |
+
"attention_dropout": 0.0,
|
| 7 |
+
"auto_map": {
|
| 8 |
+
"AutoConfig": "configuration_pruned_olmoe.PrunedOlmoeConfig",
|
| 9 |
+
"AutoModelForCausalLM": "modeling_pruned_olmoe.PrunedOlmoeForCausalLM"
|
| 10 |
+
},
|
| 11 |
+
"clip_qkv": null,
|
| 12 |
+
"dtype": "bfloat16",
|
| 13 |
+
"eos_token_id": 50279,
|
| 14 |
+
"expert_widths": [
|
| 15 |
+
[
|
| 16 |
+
1024,
|
| 17 |
+
384,
|
| 18 |
+
256,
|
| 19 |
+
256,
|
| 20 |
+
768,
|
| 21 |
+
1024,
|
| 22 |
+
128,
|
| 23 |
+
768,
|
| 24 |
+
384,
|
| 25 |
+
128,
|
| 26 |
+
1024,
|
| 27 |
+
384,
|
| 28 |
+
384,
|
| 29 |
+
128,
|
| 30 |
+
640,
|
| 31 |
+
896,
|
| 32 |
+
896,
|
| 33 |
+
256,
|
| 34 |
+
768,
|
| 35 |
+
128,
|
| 36 |
+
768,
|
| 37 |
+
768,
|
| 38 |
+
768,
|
| 39 |
+
768,
|
| 40 |
+
640,
|
| 41 |
+
896,
|
| 42 |
+
256,
|
| 43 |
+
128,
|
| 44 |
+
384,
|
| 45 |
+
896,
|
| 46 |
+
256,
|
| 47 |
+
768,
|
| 48 |
+
512,
|
| 49 |
+
640,
|
| 50 |
+
768,
|
| 51 |
+
896,
|
| 52 |
+
1024,
|
| 53 |
+
384,
|
| 54 |
+
384,
|
| 55 |
+
512,
|
| 56 |
+
384,
|
| 57 |
+
256,
|
| 58 |
+
768,
|
| 59 |
+
640,
|
| 60 |
+
896,
|
| 61 |
+
896,
|
| 62 |
+
512,
|
| 63 |
+
768,
|
| 64 |
+
512,
|
| 65 |
+
384,
|
| 66 |
+
1024,
|
| 67 |
+
896,
|
| 68 |
+
896,
|
| 69 |
+
768,
|
| 70 |
+
128
|
| 71 |
+
],
|
| 72 |
+
[
|
| 73 |
+
768,
|
| 74 |
+
896,
|
| 75 |
+
896,
|
| 76 |
+
256,
|
| 77 |
+
896,
|
| 78 |
+
384,
|
| 79 |
+
768,
|
| 80 |
+
768,
|
| 81 |
+
1024,
|
| 82 |
+
384,
|
| 83 |
+
512,
|
| 84 |
+
1024,
|
| 85 |
+
768,
|
| 86 |
+
768,
|
| 87 |
+
1024,
|
| 88 |
+
640,
|
| 89 |
+
128,
|
| 90 |
+
768,
|
| 91 |
+
384,
|
| 92 |
+
768,
|
| 93 |
+
128,
|
| 94 |
+
256,
|
| 95 |
+
512,
|
| 96 |
+
384,
|
| 97 |
+
896,
|
| 98 |
+
256,
|
| 99 |
+
1024,
|
| 100 |
+
1024,
|
| 101 |
+
896,
|
| 102 |
+
1024,
|
| 103 |
+
896,
|
| 104 |
+
768,
|
| 105 |
+
256,
|
| 106 |
+
768,
|
| 107 |
+
128,
|
| 108 |
+
1024,
|
| 109 |
+
768,
|
| 110 |
+
768,
|
| 111 |
+
640,
|
| 112 |
+
384,
|
| 113 |
+
1024,
|
| 114 |
+
384,
|
| 115 |
+
640,
|
| 116 |
+
896,
|
| 117 |
+
1024,
|
| 118 |
+
512,
|
| 119 |
+
512,
|
| 120 |
+
512,
|
| 121 |
+
640
|
| 122 |
+
],
|
| 123 |
+
[
|
| 124 |
+
768,
|
| 125 |
+
1024,
|
| 126 |
+
512,
|
| 127 |
+
384,
|
| 128 |
+
384,
|
| 129 |
+
768,
|
| 130 |
+
256,
|
| 131 |
+
384,
|
| 132 |
+
1024,
|
| 133 |
+
896,
|
| 134 |
+
640,
|
| 135 |
+
256,
|
| 136 |
+
1024,
|
| 137 |
+
1024,
|
| 138 |
+
1024,
|
| 139 |
+
640,
|
| 140 |
+
1024,
|
| 141 |
+
896,
|
| 142 |
+
512,
|
| 143 |
+
768,
|
| 144 |
+
128,
|
| 145 |
+
768,
|
| 146 |
+
1024,
|
| 147 |
+
768,
|
| 148 |
+
256,
|
| 149 |
+
1024,
|
| 150 |
+
896,
|
| 151 |
+
1024,
|
| 152 |
+
896,
|
| 153 |
+
512,
|
| 154 |
+
640,
|
| 155 |
+
384,
|
| 156 |
+
896,
|
| 157 |
+
384,
|
| 158 |
+
512,
|
| 159 |
+
640,
|
| 160 |
+
256,
|
| 161 |
+
768,
|
| 162 |
+
256,
|
| 163 |
+
768,
|
| 164 |
+
768,
|
| 165 |
+
640,
|
| 166 |
+
1024,
|
| 167 |
+
640,
|
| 168 |
+
896,
|
| 169 |
+
768,
|
| 170 |
+
768,
|
| 171 |
+
256
|
| 172 |
+
],
|
| 173 |
+
[
|
| 174 |
+
640,
|
| 175 |
+
640,
|
| 176 |
+
896,
|
| 177 |
+
512,
|
| 178 |
+
640,
|
| 179 |
+
896,
|
| 180 |
+
896,
|
| 181 |
+
384,
|
| 182 |
+
896,
|
| 183 |
+
896,
|
| 184 |
+
384,
|
| 185 |
+
896,
|
| 186 |
+
640,
|
| 187 |
+
256,
|
| 188 |
+
640,
|
| 189 |
+
1024,
|
| 190 |
+
1024,
|
| 191 |
+
768,
|
| 192 |
+
1024,
|
| 193 |
+
896,
|
| 194 |
+
1024,
|
| 195 |
+
768,
|
| 196 |
+
896,
|
| 197 |
+
256,
|
| 198 |
+
512,
|
| 199 |
+
768,
|
| 200 |
+
1024,
|
| 201 |
+
256,
|
| 202 |
+
768,
|
| 203 |
+
512,
|
| 204 |
+
256,
|
| 205 |
+
640,
|
| 206 |
+
1024,
|
| 207 |
+
1024,
|
| 208 |
+
512,
|
| 209 |
+
1024,
|
| 210 |
+
768,
|
| 211 |
+
256,
|
| 212 |
+
1024,
|
| 213 |
+
384,
|
| 214 |
+
896,
|
| 215 |
+
1024,
|
| 216 |
+
896,
|
| 217 |
+
1024,
|
| 218 |
+
384
|
| 219 |
+
],
|
| 220 |
+
[
|
| 221 |
+
256,
|
| 222 |
+
640,
|
| 223 |
+
640,
|
| 224 |
+
896,
|
| 225 |
+
768,
|
| 226 |
+
896,
|
| 227 |
+
768,
|
| 228 |
+
768,
|
| 229 |
+
896,
|
| 230 |
+
896,
|
| 231 |
+
1024,
|
| 232 |
+
256,
|
| 233 |
+
512,
|
| 234 |
+
1024,
|
| 235 |
+
640,
|
| 236 |
+
896,
|
| 237 |
+
512,
|
| 238 |
+
512,
|
| 239 |
+
384,
|
| 240 |
+
384,
|
| 241 |
+
256,
|
| 242 |
+
384,
|
| 243 |
+
384,
|
| 244 |
+
896,
|
| 245 |
+
896,
|
| 246 |
+
768,
|
| 247 |
+
640,
|
| 248 |
+
896,
|
| 249 |
+
768,
|
| 250 |
+
1024,
|
| 251 |
+
512,
|
| 252 |
+
640,
|
| 253 |
+
512,
|
| 254 |
+
640,
|
| 255 |
+
896,
|
| 256 |
+
512,
|
| 257 |
+
512,
|
| 258 |
+
384,
|
| 259 |
+
640,
|
| 260 |
+
896,
|
| 261 |
+
896,
|
| 262 |
+
896,
|
| 263 |
+
1024,
|
| 264 |
+
640,
|
| 265 |
+
1024,
|
| 266 |
+
640,
|
| 267 |
+
1024
|
| 268 |
+
],
|
| 269 |
+
[
|
| 270 |
+
1024,
|
| 271 |
+
512,
|
| 272 |
+
1024,
|
| 273 |
+
1024,
|
| 274 |
+
640,
|
| 275 |
+
896,
|
| 276 |
+
640,
|
| 277 |
+
1024,
|
| 278 |
+
896,
|
| 279 |
+
384,
|
| 280 |
+
1024,
|
| 281 |
+
128,
|
| 282 |
+
896,
|
| 283 |
+
768,
|
| 284 |
+
1024,
|
| 285 |
+
768,
|
| 286 |
+
640,
|
| 287 |
+
896,
|
| 288 |
+
768,
|
| 289 |
+
640,
|
| 290 |
+
512,
|
| 291 |
+
896,
|
| 292 |
+
512,
|
| 293 |
+
640,
|
| 294 |
+
256,
|
| 295 |
+
768,
|
| 296 |
+
640,
|
| 297 |
+
768,
|
| 298 |
+
384,
|
| 299 |
+
896,
|
| 300 |
+
512,
|
| 301 |
+
512,
|
| 302 |
+
256,
|
| 303 |
+
512,
|
| 304 |
+
896,
|
| 305 |
+
256,
|
| 306 |
+
384,
|
| 307 |
+
640,
|
| 308 |
+
512,
|
| 309 |
+
640,
|
| 310 |
+
896,
|
| 311 |
+
512,
|
| 312 |
+
1024,
|
| 313 |
+
256,
|
| 314 |
+
768,
|
| 315 |
+
1024,
|
| 316 |
+
768,
|
| 317 |
+
256,
|
| 318 |
+
256
|
| 319 |
+
],
|
| 320 |
+
[
|
| 321 |
+
640,
|
| 322 |
+
896,
|
| 323 |
+
1024,
|
| 324 |
+
896,
|
| 325 |
+
1024,
|
| 326 |
+
1024,
|
| 327 |
+
1024,
|
| 328 |
+
512,
|
| 329 |
+
256,
|
| 330 |
+
256,
|
| 331 |
+
1024,
|
| 332 |
+
768,
|
| 333 |
+
512,
|
| 334 |
+
768,
|
| 335 |
+
1024,
|
| 336 |
+
1024,
|
| 337 |
+
1024,
|
| 338 |
+
384,
|
| 339 |
+
512,
|
| 340 |
+
1024,
|
| 341 |
+
512,
|
| 342 |
+
1024,
|
| 343 |
+
128,
|
| 344 |
+
640,
|
| 345 |
+
640,
|
| 346 |
+
896,
|
| 347 |
+
768,
|
| 348 |
+
128,
|
| 349 |
+
256,
|
| 350 |
+
256,
|
| 351 |
+
256,
|
| 352 |
+
256,
|
| 353 |
+
896,
|
| 354 |
+
1024,
|
| 355 |
+
1024,
|
| 356 |
+
384,
|
| 357 |
+
896,
|
| 358 |
+
256,
|
| 359 |
+
896,
|
| 360 |
+
640,
|
| 361 |
+
1024,
|
| 362 |
+
384,
|
| 363 |
+
640,
|
| 364 |
+
256,
|
| 365 |
+
1024,
|
| 366 |
+
1024,
|
| 367 |
+
1024
|
| 368 |
+
],
|
| 369 |
+
[
|
| 370 |
+
1024,
|
| 371 |
+
384,
|
| 372 |
+
1024,
|
| 373 |
+
1024,
|
| 374 |
+
256,
|
| 375 |
+
128,
|
| 376 |
+
256,
|
| 377 |
+
384,
|
| 378 |
+
256,
|
| 379 |
+
384,
|
| 380 |
+
896,
|
| 381 |
+
768,
|
| 382 |
+
896,
|
| 383 |
+
896,
|
| 384 |
+
512,
|
| 385 |
+
896,
|
| 386 |
+
640,
|
| 387 |
+
384,
|
| 388 |
+
384,
|
| 389 |
+
896,
|
| 390 |
+
768,
|
| 391 |
+
384,
|
| 392 |
+
896,
|
| 393 |
+
768,
|
| 394 |
+
768,
|
| 395 |
+
512,
|
| 396 |
+
896,
|
| 397 |
+
768,
|
| 398 |
+
768,
|
| 399 |
+
896,
|
| 400 |
+
128,
|
| 401 |
+
896,
|
| 402 |
+
512,
|
| 403 |
+
256,
|
| 404 |
+
768,
|
| 405 |
+
128,
|
| 406 |
+
384,
|
| 407 |
+
256,
|
| 408 |
+
896,
|
| 409 |
+
896,
|
| 410 |
+
384,
|
| 411 |
+
768,
|
| 412 |
+
512,
|
| 413 |
+
640,
|
| 414 |
+
256,
|
| 415 |
+
768,
|
| 416 |
+
640,
|
| 417 |
+
896,
|
| 418 |
+
384,
|
| 419 |
+
512,
|
| 420 |
+
1024,
|
| 421 |
+
768,
|
| 422 |
+
384
|
| 423 |
+
],
|
| 424 |
+
[
|
| 425 |
+
512,
|
| 426 |
+
768,
|
| 427 |
+
512,
|
| 428 |
+
256,
|
| 429 |
+
128,
|
| 430 |
+
640,
|
| 431 |
+
384,
|
| 432 |
+
640,
|
| 433 |
+
768,
|
| 434 |
+
896,
|
| 435 |
+
640,
|
| 436 |
+
768,
|
| 437 |
+
256,
|
| 438 |
+
384,
|
| 439 |
+
1024,
|
| 440 |
+
896,
|
| 441 |
+
256,
|
| 442 |
+
896,
|
| 443 |
+
512,
|
| 444 |
+
256,
|
| 445 |
+
896,
|
| 446 |
+
768,
|
| 447 |
+
256,
|
| 448 |
+
896,
|
| 449 |
+
896,
|
| 450 |
+
384,
|
| 451 |
+
896,
|
| 452 |
+
640,
|
| 453 |
+
768,
|
| 454 |
+
512,
|
| 455 |
+
768,
|
| 456 |
+
768,
|
| 457 |
+
1024,
|
| 458 |
+
768,
|
| 459 |
+
640,
|
| 460 |
+
768,
|
| 461 |
+
384,
|
| 462 |
+
256,
|
| 463 |
+
512,
|
| 464 |
+
896,
|
| 465 |
+
128,
|
| 466 |
+
384,
|
| 467 |
+
256,
|
| 468 |
+
768,
|
| 469 |
+
384,
|
| 470 |
+
256,
|
| 471 |
+
1024,
|
| 472 |
+
1024,
|
| 473 |
+
896,
|
| 474 |
+
256,
|
| 475 |
+
1024,
|
| 476 |
+
256,
|
| 477 |
+
128,
|
| 478 |
+
896
|
| 479 |
+
],
|
| 480 |
+
[
|
| 481 |
+
640,
|
| 482 |
+
640,
|
| 483 |
+
896,
|
| 484 |
+
256,
|
| 485 |
+
1024,
|
| 486 |
+
512,
|
| 487 |
+
1024,
|
| 488 |
+
768,
|
| 489 |
+
384,
|
| 490 |
+
512,
|
| 491 |
+
256,
|
| 492 |
+
768,
|
| 493 |
+
896,
|
| 494 |
+
768,
|
| 495 |
+
512,
|
| 496 |
+
768,
|
| 497 |
+
768,
|
| 498 |
+
640,
|
| 499 |
+
384,
|
| 500 |
+
768,
|
| 501 |
+
512,
|
| 502 |
+
768,
|
| 503 |
+
768,
|
| 504 |
+
512,
|
| 505 |
+
768,
|
| 506 |
+
128,
|
| 507 |
+
896,
|
| 508 |
+
512,
|
| 509 |
+
768,
|
| 510 |
+
1024,
|
| 511 |
+
128,
|
| 512 |
+
384,
|
| 513 |
+
768,
|
| 514 |
+
768,
|
| 515 |
+
768,
|
| 516 |
+
384,
|
| 517 |
+
512,
|
| 518 |
+
640,
|
| 519 |
+
768,
|
| 520 |
+
512,
|
| 521 |
+
768,
|
| 522 |
+
1024,
|
| 523 |
+
640,
|
| 524 |
+
896,
|
| 525 |
+
256,
|
| 526 |
+
1024,
|
| 527 |
+
384,
|
| 528 |
+
768,
|
| 529 |
+
768,
|
| 530 |
+
768
|
| 531 |
+
],
|
| 532 |
+
[
|
| 533 |
+
896,
|
| 534 |
+
512,
|
| 535 |
+
896,
|
| 536 |
+
768,
|
| 537 |
+
384,
|
| 538 |
+
384,
|
| 539 |
+
768,
|
| 540 |
+
512,
|
| 541 |
+
768,
|
| 542 |
+
512,
|
| 543 |
+
1024,
|
| 544 |
+
640,
|
| 545 |
+
896,
|
| 546 |
+
896,
|
| 547 |
+
256,
|
| 548 |
+
640,
|
| 549 |
+
1024,
|
| 550 |
+
256,
|
| 551 |
+
896,
|
| 552 |
+
128,
|
| 553 |
+
128,
|
| 554 |
+
128,
|
| 555 |
+
768,
|
| 556 |
+
896,
|
| 557 |
+
384,
|
| 558 |
+
896,
|
| 559 |
+
512,
|
| 560 |
+
896,
|
| 561 |
+
384,
|
| 562 |
+
256,
|
| 563 |
+
640,
|
| 564 |
+
640,
|
| 565 |
+
896,
|
| 566 |
+
768,
|
| 567 |
+
640,
|
| 568 |
+
256,
|
| 569 |
+
896,
|
| 570 |
+
896,
|
| 571 |
+
512,
|
| 572 |
+
128,
|
| 573 |
+
896,
|
| 574 |
+
256,
|
| 575 |
+
256,
|
| 576 |
+
640,
|
| 577 |
+
896,
|
| 578 |
+
896,
|
| 579 |
+
128,
|
| 580 |
+
1024,
|
| 581 |
+
256,
|
| 582 |
+
384,
|
| 583 |
+
1024,
|
| 584 |
+
640,
|
| 585 |
+
896
|
| 586 |
+
],
|
| 587 |
+
[
|
| 588 |
+
768,
|
| 589 |
+
384,
|
| 590 |
+
640,
|
| 591 |
+
896,
|
| 592 |
+
256,
|
| 593 |
+
128,
|
| 594 |
+
384,
|
| 595 |
+
896,
|
| 596 |
+
128,
|
| 597 |
+
128,
|
| 598 |
+
896,
|
| 599 |
+
256,
|
| 600 |
+
384,
|
| 601 |
+
896,
|
| 602 |
+
512,
|
| 603 |
+
768,
|
| 604 |
+
768,
|
| 605 |
+
512,
|
| 606 |
+
512,
|
| 607 |
+
768,
|
| 608 |
+
896,
|
| 609 |
+
640,
|
| 610 |
+
768,
|
| 611 |
+
896,
|
| 612 |
+
896,
|
| 613 |
+
896,
|
| 614 |
+
640,
|
| 615 |
+
896,
|
| 616 |
+
640,
|
| 617 |
+
512,
|
| 618 |
+
896,
|
| 619 |
+
256,
|
| 620 |
+
512,
|
| 621 |
+
128,
|
| 622 |
+
512,
|
| 623 |
+
384,
|
| 624 |
+
768,
|
| 625 |
+
768,
|
| 626 |
+
1024,
|
| 627 |
+
256,
|
| 628 |
+
768,
|
| 629 |
+
256,
|
| 630 |
+
768,
|
| 631 |
+
512,
|
| 632 |
+
640,
|
| 633 |
+
1024,
|
| 634 |
+
128,
|
| 635 |
+
896,
|
| 636 |
+
896,
|
| 637 |
+
896,
|
| 638 |
+
896,
|
| 639 |
+
1024
|
| 640 |
+
],
|
| 641 |
+
[
|
| 642 |
+
512,
|
| 643 |
+
384,
|
| 644 |
+
768,
|
| 645 |
+
640,
|
| 646 |
+
640,
|
| 647 |
+
768,
|
| 648 |
+
1024,
|
| 649 |
+
896,
|
| 650 |
+
512,
|
| 651 |
+
256,
|
| 652 |
+
640,
|
| 653 |
+
768,
|
| 654 |
+
640,
|
| 655 |
+
896,
|
| 656 |
+
128,
|
| 657 |
+
256,
|
| 658 |
+
896,
|
| 659 |
+
1024,
|
| 660 |
+
256,
|
| 661 |
+
640,
|
| 662 |
+
512,
|
| 663 |
+
256,
|
| 664 |
+
128,
|
| 665 |
+
512,
|
| 666 |
+
256,
|
| 667 |
+
640,
|
| 668 |
+
768,
|
| 669 |
+
768,
|
| 670 |
+
128,
|
| 671 |
+
128,
|
| 672 |
+
768,
|
| 673 |
+
640,
|
| 674 |
+
1024,
|
| 675 |
+
1024,
|
| 676 |
+
768,
|
| 677 |
+
512,
|
| 678 |
+
896,
|
| 679 |
+
768,
|
| 680 |
+
896,
|
| 681 |
+
1024,
|
| 682 |
+
896,
|
| 683 |
+
896,
|
| 684 |
+
512,
|
| 685 |
+
640,
|
| 686 |
+
1024,
|
| 687 |
+
512,
|
| 688 |
+
1024,
|
| 689 |
+
512,
|
| 690 |
+
512,
|
| 691 |
+
512,
|
| 692 |
+
768
|
| 693 |
+
],
|
| 694 |
+
[
|
| 695 |
+
896,
|
| 696 |
+
896,
|
| 697 |
+
1024,
|
| 698 |
+
1024,
|
| 699 |
+
896,
|
| 700 |
+
128,
|
| 701 |
+
768,
|
| 702 |
+
256,
|
| 703 |
+
1024,
|
| 704 |
+
256,
|
| 705 |
+
1024,
|
| 706 |
+
640,
|
| 707 |
+
384,
|
| 708 |
+
256,
|
| 709 |
+
256,
|
| 710 |
+
512,
|
| 711 |
+
768,
|
| 712 |
+
896,
|
| 713 |
+
512,
|
| 714 |
+
768,
|
| 715 |
+
384,
|
| 716 |
+
1024,
|
| 717 |
+
896,
|
| 718 |
+
896,
|
| 719 |
+
1024,
|
| 720 |
+
896,
|
| 721 |
+
768,
|
| 722 |
+
896,
|
| 723 |
+
640,
|
| 724 |
+
1024,
|
| 725 |
+
512,
|
| 726 |
+
896,
|
| 727 |
+
512,
|
| 728 |
+
1024,
|
| 729 |
+
512,
|
| 730 |
+
512,
|
| 731 |
+
256,
|
| 732 |
+
256,
|
| 733 |
+
256,
|
| 734 |
+
512,
|
| 735 |
+
768,
|
| 736 |
+
128,
|
| 737 |
+
384,
|
| 738 |
+
512,
|
| 739 |
+
896,
|
| 740 |
+
896,
|
| 741 |
+
1024,
|
| 742 |
+
256,
|
| 743 |
+
384,
|
| 744 |
+
640
|
| 745 |
+
],
|
| 746 |
+
[
|
| 747 |
+
896,
|
| 748 |
+
640,
|
| 749 |
+
384,
|
| 750 |
+
512,
|
| 751 |
+
256,
|
| 752 |
+
640,
|
| 753 |
+
1024,
|
| 754 |
+
384,
|
| 755 |
+
1024,
|
| 756 |
+
1024,
|
| 757 |
+
768,
|
| 758 |
+
256,
|
| 759 |
+
1024,
|
| 760 |
+
768,
|
| 761 |
+
512,
|
| 762 |
+
896,
|
| 763 |
+
256,
|
| 764 |
+
1024,
|
| 765 |
+
768,
|
| 766 |
+
768,
|
| 767 |
+
768,
|
| 768 |
+
384,
|
| 769 |
+
384,
|
| 770 |
+
256,
|
| 771 |
+
1024,
|
| 772 |
+
384,
|
| 773 |
+
384,
|
| 774 |
+
384,
|
| 775 |
+
896,
|
| 776 |
+
768,
|
| 777 |
+
640,
|
| 778 |
+
768,
|
| 779 |
+
512,
|
| 780 |
+
896,
|
| 781 |
+
896,
|
| 782 |
+
896,
|
| 783 |
+
896,
|
| 784 |
+
256,
|
| 785 |
+
384,
|
| 786 |
+
128,
|
| 787 |
+
1024,
|
| 788 |
+
896,
|
| 789 |
+
256,
|
| 790 |
+
256,
|
| 791 |
+
768,
|
| 792 |
+
640,
|
| 793 |
+
896,
|
| 794 |
+
384,
|
| 795 |
+
768,
|
| 796 |
+
512,
|
| 797 |
+
640
|
| 798 |
+
],
|
| 799 |
+
[
|
| 800 |
+
896,
|
| 801 |
+
1024,
|
| 802 |
+
768,
|
| 803 |
+
1024,
|
| 804 |
+
896,
|
| 805 |
+
256,
|
| 806 |
+
768,
|
| 807 |
+
128,
|
| 808 |
+
128,
|
| 809 |
+
768,
|
| 810 |
+
512,
|
| 811 |
+
896,
|
| 812 |
+
384,
|
| 813 |
+
768,
|
| 814 |
+
1024,
|
| 815 |
+
256,
|
| 816 |
+
768,
|
| 817 |
+
768,
|
| 818 |
+
256,
|
| 819 |
+
512,
|
| 820 |
+
512,
|
| 821 |
+
640,
|
| 822 |
+
512,
|
| 823 |
+
256,
|
| 824 |
+
768,
|
| 825 |
+
896,
|
| 826 |
+
384,
|
| 827 |
+
1024,
|
| 828 |
+
640,
|
| 829 |
+
1024,
|
| 830 |
+
512,
|
| 831 |
+
512,
|
| 832 |
+
384,
|
| 833 |
+
512,
|
| 834 |
+
512,
|
| 835 |
+
1024,
|
| 836 |
+
384,
|
| 837 |
+
896,
|
| 838 |
+
768,
|
| 839 |
+
384,
|
| 840 |
+
384,
|
| 841 |
+
128,
|
| 842 |
+
384,
|
| 843 |
+
1024,
|
| 844 |
+
896,
|
| 845 |
+
640,
|
| 846 |
+
768,
|
| 847 |
+
768,
|
| 848 |
+
256,
|
| 849 |
+
640,
|
| 850 |
+
512,
|
| 851 |
+
640,
|
| 852 |
+
384
|
| 853 |
+
]
|
| 854 |
+
],
|
| 855 |
+
"glean_metadata": {
|
| 856 |
+
"base_model": "allenai/OLMoE-1B-7B-0125-Instruct",
|
| 857 |
+
"block_size": 128,
|
| 858 |
+
"criterion": "reap",
|
| 859 |
+
"dead_experts": 217,
|
| 860 |
+
"keep_fraction": 0.5,
|
| 861 |
+
"min_width": 128,
|
| 862 |
+
"params": 3697491968,
|
| 863 |
+
"scores": "outputs/scores_0125inst_dolmino-math/scores.pt"
|
| 864 |
+
},
|
| 865 |
+
"hidden_act": "silu",
|
| 866 |
+
"hidden_size": 2048,
|
| 867 |
+
"initializer_range": 0.02,
|
| 868 |
+
"intermediate_size": 1024,
|
| 869 |
+
"max_position_embeddings": 4096,
|
| 870 |
+
"model_type": "pruned_olmoe",
|
| 871 |
+
"norm_topk_prob": false,
|
| 872 |
+
"num_attention_heads": 16,
|
| 873 |
+
"num_experts": 64,
|
| 874 |
+
"num_experts_per_tok": 8,
|
| 875 |
+
"num_hidden_layers": 16,
|
| 876 |
+
"num_key_value_heads": 16,
|
| 877 |
+
"output_router_logits": false,
|
| 878 |
+
"pad_token_id": 1,
|
| 879 |
+
"rms_norm_eps": 1e-05,
|
| 880 |
+
"rope_scaling": null,
|
| 881 |
+
"rope_theta": 10000.0,
|
| 882 |
+
"router_aux_loss_coef": 0.01,
|
| 883 |
+
"tie_word_embeddings": false,
|
| 884 |
+
"transformers_version": "4.57.6",
|
| 885 |
+
"use_cache": false,
|
| 886 |
+
"vocab_size": 50304
|
| 887 |
+
}
|
healed/mixonly_keep50/vllm_live/configuration_pruned_olmoe.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Configuration for GLEAN-pruned OLMoE: variable-width, variable-count experts.
|
| 2 |
+
"""
|
| 3 |
+
|
| 4 |
+
from transformers.models.olmoe.configuration_olmoe import OlmoeConfig
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
class PrunedOlmoeConfig(OlmoeConfig):
|
| 8 |
+
"""OlmoeConfig plus a per-(layer, expert) width table.
|
| 9 |
+
|
| 10 |
+
``expert_widths[l]`` lists the SwiGLU intermediate width of each surviving
|
| 11 |
+
expert in decoder layer ``l``, in expert order. Lists are ragged: layers
|
| 12 |
+
may keep different numbers of experts (deleted experts simply don't
|
| 13 |
+
appear — the router in layer ``l`` has ``len(expert_widths[l])`` rows),
|
| 14 |
+
and each width may differ (multiples of the GEMM block size, 128, for
|
| 15 |
+
variable-MegaBlocks execution). ``None`` means an unpruned model
|
| 16 |
+
(uniform ``num_experts`` × ``intermediate_size``).
|
| 17 |
+
|
| 18 |
+
The inherited ``num_experts`` / ``intermediate_size`` keep their ORIGINAL
|
| 19 |
+
(pre-pruning) values for provenance; the width table is authoritative for
|
| 20 |
+
the built architecture.
|
| 21 |
+
"""
|
| 22 |
+
|
| 23 |
+
model_type = "pruned_olmoe"
|
| 24 |
+
|
| 25 |
+
def __init__(self, expert_widths: list[list[int]] | None = None, **kwargs):
|
| 26 |
+
super().__init__(**kwargs)
|
| 27 |
+
self.expert_widths = expert_widths
|
healed/mixonly_keep50/vllm_live/generation_config.json
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_from_model_config": true,
|
| 3 |
+
"eos_token_id": 50279,
|
| 4 |
+
"pad_token_id": 1,
|
| 5 |
+
"transformers_version": "4.57.6"
|
| 6 |
+
}
|
healed/mixonly_keep50/vllm_live/model.safetensors.index.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
healed/mixonly_keep50/vllm_live/modeling_pruned_olmoe.py
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""GLEAN-pruned OLMoE: HF-loadable model with ragged (variable-width) experts.
|
| 2 |
+
|
| 3 |
+
Pattern follows hbfreed/variable-flex-olmo's PrunedFlexOlmoForCausalLM
|
| 4 |
+
(docs/recon/prior-work-hbfreed.md), generalized from one scalar width to a
|
| 5 |
+
per-(layer, expert) width table: ``super().__init__`` builds the uniform
|
| 6 |
+
architecture from the config, then every MoE block is rebuilt to its pruned
|
| 7 |
+
shape — surviving experts only, each at its own width, router sliced to
|
| 8 |
+
match — so the state dict aligns exactly with what
|
| 9 |
+
``glean.prune.prune_channels_global`` leaves behind.
|
| 10 |
+
|
| 11 |
+
Caveat: ``output_router_logits=True`` (the load-balancing aux loss) assumes a
|
| 12 |
+
uniform ``config.num_experts`` and is unsupported on ragged models.
|
| 13 |
+
"""
|
| 14 |
+
|
| 15 |
+
import torch.nn as nn
|
| 16 |
+
from transformers.activations import ACT2FN
|
| 17 |
+
from transformers.models.olmoe.modeling_olmoe import OlmoeForCausalLM
|
| 18 |
+
|
| 19 |
+
from .configuration_pruned_olmoe import PrunedOlmoeConfig
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
class RaggedOlmoeMLP(nn.Module):
|
| 23 |
+
"""OlmoeMLP with an explicit intermediate width (SwiGLU, no biases)."""
|
| 24 |
+
|
| 25 |
+
def __init__(self, hidden_size: int, intermediate_size: int, hidden_act: str):
|
| 26 |
+
super().__init__()
|
| 27 |
+
self.hidden_size = hidden_size
|
| 28 |
+
self.intermediate_size = intermediate_size
|
| 29 |
+
self.gate_proj = nn.Linear(hidden_size, intermediate_size, bias=False)
|
| 30 |
+
self.up_proj = nn.Linear(hidden_size, intermediate_size, bias=False)
|
| 31 |
+
self.down_proj = nn.Linear(intermediate_size, hidden_size, bias=False)
|
| 32 |
+
self.act_fn = ACT2FN[hidden_act]
|
| 33 |
+
|
| 34 |
+
def forward(self, x):
|
| 35 |
+
return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
class PrunedOlmoeForCausalLM(OlmoeForCausalLM):
|
| 39 |
+
"""OLMoE with per-layer surviving-expert lists at per-expert widths."""
|
| 40 |
+
|
| 41 |
+
config_class = PrunedOlmoeConfig
|
| 42 |
+
|
| 43 |
+
def __init__(self, config: PrunedOlmoeConfig):
|
| 44 |
+
super().__init__(config)
|
| 45 |
+
widths_table = getattr(config, "expert_widths", None)
|
| 46 |
+
if widths_table is None:
|
| 47 |
+
return # unpruned: plain OLMoE
|
| 48 |
+
if len(widths_table) != len(self.model.layers):
|
| 49 |
+
raise ValueError(
|
| 50 |
+
f"expert_widths has {len(widths_table)} rows but the model has "
|
| 51 |
+
f"{len(self.model.layers)} decoder layers"
|
| 52 |
+
)
|
| 53 |
+
for layer, widths in zip(self.model.layers, widths_table):
|
| 54 |
+
if any(w <= 0 for w in widths):
|
| 55 |
+
raise ValueError("expert_widths must list surviving experts only (>0)")
|
| 56 |
+
block = layer.mlp
|
| 57 |
+
if len(widths) < block.top_k:
|
| 58 |
+
raise ValueError(
|
| 59 |
+
f"a layer keeps {len(widths)} experts < top_k={block.top_k}"
|
| 60 |
+
)
|
| 61 |
+
block.num_experts = len(widths)
|
| 62 |
+
block.gate = nn.Linear(config.hidden_size, len(widths), bias=False)
|
| 63 |
+
block.experts = nn.ModuleList(
|
| 64 |
+
RaggedOlmoeMLP(config.hidden_size, w, config.hidden_act)
|
| 65 |
+
for w in widths
|
| 66 |
+
)
|
healed/mixonly_keep50/vllm_live/special_tokens_map.json
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"bos_token": {
|
| 3 |
+
"content": "|||IP_ADDRESS|||",
|
| 4 |
+
"lstrip": false,
|
| 5 |
+
"normalized": true,
|
| 6 |
+
"rstrip": false,
|
| 7 |
+
"single_word": false
|
| 8 |
+
},
|
| 9 |
+
"eos_token": {
|
| 10 |
+
"content": "|||IP_ADDRESS|||",
|
| 11 |
+
"lstrip": false,
|
| 12 |
+
"normalized": true,
|
| 13 |
+
"rstrip": false,
|
| 14 |
+
"single_word": false
|
| 15 |
+
},
|
| 16 |
+
"pad_token": {
|
| 17 |
+
"content": "<pad>",
|
| 18 |
+
"lstrip": false,
|
| 19 |
+
"normalized": false,
|
| 20 |
+
"rstrip": false,
|
| 21 |
+
"single_word": false
|
| 22 |
+
}
|
| 23 |
+
}
|
healed/mixonly_keep50/vllm_live/tokenizer.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
healed/mixonly_keep50/vllm_live/tokenizer_config.json
ADDED
|
@@ -0,0 +1,247 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"add_bos_token": false,
|
| 3 |
+
"add_eos_token": false,
|
| 4 |
+
"add_prefix_space": false,
|
| 5 |
+
"added_tokens_decoder": {
|
| 6 |
+
"0": {
|
| 7 |
+
"content": "<|endoftext|>",
|
| 8 |
+
"lstrip": false,
|
| 9 |
+
"normalized": false,
|
| 10 |
+
"rstrip": false,
|
| 11 |
+
"single_word": false,
|
| 12 |
+
"special": true
|
| 13 |
+
},
|
| 14 |
+
"1": {
|
| 15 |
+
"content": "<|padding|>",
|
| 16 |
+
"lstrip": false,
|
| 17 |
+
"normalized": false,
|
| 18 |
+
"rstrip": false,
|
| 19 |
+
"single_word": false,
|
| 20 |
+
"special": true
|
| 21 |
+
},
|
| 22 |
+
"50254": {
|
| 23 |
+
"content": " ",
|
| 24 |
+
"lstrip": false,
|
| 25 |
+
"normalized": true,
|
| 26 |
+
"rstrip": false,
|
| 27 |
+
"single_word": false,
|
| 28 |
+
"special": false
|
| 29 |
+
},
|
| 30 |
+
"50255": {
|
| 31 |
+
"content": " ",
|
| 32 |
+
"lstrip": false,
|
| 33 |
+
"normalized": true,
|
| 34 |
+
"rstrip": false,
|
| 35 |
+
"single_word": false,
|
| 36 |
+
"special": false
|
| 37 |
+
},
|
| 38 |
+
"50256": {
|
| 39 |
+
"content": " ",
|
| 40 |
+
"lstrip": false,
|
| 41 |
+
"normalized": true,
|
| 42 |
+
"rstrip": false,
|
| 43 |
+
"single_word": false,
|
| 44 |
+
"special": false
|
| 45 |
+
},
|
| 46 |
+
"50257": {
|
| 47 |
+
"content": " ",
|
| 48 |
+
"lstrip": false,
|
| 49 |
+
"normalized": true,
|
| 50 |
+
"rstrip": false,
|
| 51 |
+
"single_word": false,
|
| 52 |
+
"special": false
|
| 53 |
+
},
|
| 54 |
+
"50258": {
|
| 55 |
+
"content": " ",
|
| 56 |
+
"lstrip": false,
|
| 57 |
+
"normalized": true,
|
| 58 |
+
"rstrip": false,
|
| 59 |
+
"single_word": false,
|
| 60 |
+
"special": false
|
| 61 |
+
},
|
| 62 |
+
"50259": {
|
| 63 |
+
"content": " ",
|
| 64 |
+
"lstrip": false,
|
| 65 |
+
"normalized": true,
|
| 66 |
+
"rstrip": false,
|
| 67 |
+
"single_word": false,
|
| 68 |
+
"special": false
|
| 69 |
+
},
|
| 70 |
+
"50260": {
|
| 71 |
+
"content": " ",
|
| 72 |
+
"lstrip": false,
|
| 73 |
+
"normalized": true,
|
| 74 |
+
"rstrip": false,
|
| 75 |
+
"single_word": false,
|
| 76 |
+
"special": false
|
| 77 |
+
},
|
| 78 |
+
"50261": {
|
| 79 |
+
"content": " ",
|
| 80 |
+
"lstrip": false,
|
| 81 |
+
"normalized": true,
|
| 82 |
+
"rstrip": false,
|
| 83 |
+
"single_word": false,
|
| 84 |
+
"special": false
|
| 85 |
+
},
|
| 86 |
+
"50262": {
|
| 87 |
+
"content": " ",
|
| 88 |
+
"lstrip": false,
|
| 89 |
+
"normalized": true,
|
| 90 |
+
"rstrip": false,
|
| 91 |
+
"single_word": false,
|
| 92 |
+
"special": false
|
| 93 |
+
},
|
| 94 |
+
"50263": {
|
| 95 |
+
"content": " ",
|
| 96 |
+
"lstrip": false,
|
| 97 |
+
"normalized": true,
|
| 98 |
+
"rstrip": false,
|
| 99 |
+
"single_word": false,
|
| 100 |
+
"special": false
|
| 101 |
+
},
|
| 102 |
+
"50264": {
|
| 103 |
+
"content": " ",
|
| 104 |
+
"lstrip": false,
|
| 105 |
+
"normalized": true,
|
| 106 |
+
"rstrip": false,
|
| 107 |
+
"single_word": false,
|
| 108 |
+
"special": false
|
| 109 |
+
},
|
| 110 |
+
"50265": {
|
| 111 |
+
"content": " ",
|
| 112 |
+
"lstrip": false,
|
| 113 |
+
"normalized": true,
|
| 114 |
+
"rstrip": false,
|
| 115 |
+
"single_word": false,
|
| 116 |
+
"special": false
|
| 117 |
+
},
|
| 118 |
+
"50266": {
|
| 119 |
+
"content": " ",
|
| 120 |
+
"lstrip": false,
|
| 121 |
+
"normalized": true,
|
| 122 |
+
"rstrip": false,
|
| 123 |
+
"single_word": false,
|
| 124 |
+
"special": false
|
| 125 |
+
},
|
| 126 |
+
"50267": {
|
| 127 |
+
"content": " ",
|
| 128 |
+
"lstrip": false,
|
| 129 |
+
"normalized": true,
|
| 130 |
+
"rstrip": false,
|
| 131 |
+
"single_word": false,
|
| 132 |
+
"special": false
|
| 133 |
+
},
|
| 134 |
+
"50268": {
|
| 135 |
+
"content": " ",
|
| 136 |
+
"lstrip": false,
|
| 137 |
+
"normalized": true,
|
| 138 |
+
"rstrip": false,
|
| 139 |
+
"single_word": false,
|
| 140 |
+
"special": false
|
| 141 |
+
},
|
| 142 |
+
"50269": {
|
| 143 |
+
"content": " ",
|
| 144 |
+
"lstrip": false,
|
| 145 |
+
"normalized": true,
|
| 146 |
+
"rstrip": false,
|
| 147 |
+
"single_word": false,
|
| 148 |
+
"special": false
|
| 149 |
+
},
|
| 150 |
+
"50270": {
|
| 151 |
+
"content": " ",
|
| 152 |
+
"lstrip": false,
|
| 153 |
+
"normalized": true,
|
| 154 |
+
"rstrip": false,
|
| 155 |
+
"single_word": false,
|
| 156 |
+
"special": false
|
| 157 |
+
},
|
| 158 |
+
"50271": {
|
| 159 |
+
"content": " ",
|
| 160 |
+
"lstrip": false,
|
| 161 |
+
"normalized": true,
|
| 162 |
+
"rstrip": false,
|
| 163 |
+
"single_word": false,
|
| 164 |
+
"special": false
|
| 165 |
+
},
|
| 166 |
+
"50272": {
|
| 167 |
+
"content": " ",
|
| 168 |
+
"lstrip": false,
|
| 169 |
+
"normalized": true,
|
| 170 |
+
"rstrip": false,
|
| 171 |
+
"single_word": false,
|
| 172 |
+
"special": false
|
| 173 |
+
},
|
| 174 |
+
"50273": {
|
| 175 |
+
"content": " ",
|
| 176 |
+
"lstrip": false,
|
| 177 |
+
"normalized": true,
|
| 178 |
+
"rstrip": false,
|
| 179 |
+
"single_word": false,
|
| 180 |
+
"special": false
|
| 181 |
+
},
|
| 182 |
+
"50274": {
|
| 183 |
+
"content": " ",
|
| 184 |
+
"lstrip": false,
|
| 185 |
+
"normalized": true,
|
| 186 |
+
"rstrip": false,
|
| 187 |
+
"single_word": false,
|
| 188 |
+
"special": false
|
| 189 |
+
},
|
| 190 |
+
"50275": {
|
| 191 |
+
"content": " ",
|
| 192 |
+
"lstrip": false,
|
| 193 |
+
"normalized": true,
|
| 194 |
+
"rstrip": false,
|
| 195 |
+
"single_word": false,
|
| 196 |
+
"special": false
|
| 197 |
+
},
|
| 198 |
+
"50276": {
|
| 199 |
+
"content": " ",
|
| 200 |
+
"lstrip": false,
|
| 201 |
+
"normalized": true,
|
| 202 |
+
"rstrip": false,
|
| 203 |
+
"single_word": false,
|
| 204 |
+
"special": false
|
| 205 |
+
},
|
| 206 |
+
"50277": {
|
| 207 |
+
"content": "|||EMAIL_ADDRESS|||",
|
| 208 |
+
"lstrip": false,
|
| 209 |
+
"normalized": true,
|
| 210 |
+
"rstrip": false,
|
| 211 |
+
"single_word": false,
|
| 212 |
+
"special": false
|
| 213 |
+
},
|
| 214 |
+
"50278": {
|
| 215 |
+
"content": "|||PHONE_NUMBER|||",
|
| 216 |
+
"lstrip": false,
|
| 217 |
+
"normalized": true,
|
| 218 |
+
"rstrip": false,
|
| 219 |
+
"single_word": false,
|
| 220 |
+
"special": false
|
| 221 |
+
},
|
| 222 |
+
"50279": {
|
| 223 |
+
"content": "|||IP_ADDRESS|||",
|
| 224 |
+
"lstrip": false,
|
| 225 |
+
"normalized": true,
|
| 226 |
+
"rstrip": false,
|
| 227 |
+
"single_word": false,
|
| 228 |
+
"special": true
|
| 229 |
+
},
|
| 230 |
+
"50280": {
|
| 231 |
+
"content": "<pad>",
|
| 232 |
+
"lstrip": false,
|
| 233 |
+
"normalized": false,
|
| 234 |
+
"rstrip": false,
|
| 235 |
+
"single_word": false,
|
| 236 |
+
"special": true
|
| 237 |
+
}
|
| 238 |
+
},
|
| 239 |
+
"bos_token": "|||IP_ADDRESS|||",
|
| 240 |
+
"clean_up_tokenization_spaces": false,
|
| 241 |
+
"eos_token": "|||IP_ADDRESS|||",
|
| 242 |
+
"extra_special_tokens": {},
|
| 243 |
+
"model_max_length": 1000000000000000019884624838656,
|
| 244 |
+
"pad_token": "<pad>",
|
| 245 |
+
"tokenizer_class": "GPTNeoXTokenizer",
|
| 246 |
+
"unk_token": null
|
| 247 |
+
}
|
healed/mixonly_keep50/wandb/debug-internal.log
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-07-31T01:15:14.143006306-07:00","level":"INFO","msg":"wandb-core"}
|
| 2 |
+
{"time":"2026-07-31T01:15:14.143036247-07:00","level":"INFO","msg":"stream: starting","core version":"0.28.0"}
|
| 3 |
+
{"time":"2026-07-31T01:15:14.255553325-07:00","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
|
| 4 |
+
{"time":"2026-07-31T01:15:14.255573436-07:00","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
|
| 5 |
+
{"time":"2026-07-31T01:15:14.255616137-07:00","level":"INFO","msg":"stream: created new stream","id":"gexxktae"}
|
| 6 |
+
{"time":"2026-07-31T01:15:14.255765612-07:00","level":"INFO","msg":"stream: started"}
|
| 7 |
+
{"time":"2026-07-31T01:15:14.255812423-07:00","level":"INFO","msg":"writer: started","stream_id":"gexxktae"}
|
| 8 |
+
{"time":"2026-07-31T01:15:14.255852534-07:00","level":"INFO","msg":"handler: started"}
|
| 9 |
+
{"time":"2026-07-31T01:15:14.255895105-07:00","level":"INFO","msg":"sender: started"}
|
| 10 |
+
{"time":"2026-07-31T01:15:14.271152628-07:00","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
|
| 11 |
+
{"time":"2026-07-31T01:15:14.271190299-07:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"}
|
healed/mixonly_keep50/wandb/debug.log
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-07-31 01:15:13,687 INFO MainThread:2043723 [wandb_setup.py:_flush():81] Current SDK version is 0.28.0
|
| 2 |
+
2026-07-31 01:15:13,687 INFO MainThread:2043723 [wandb_setup.py:_flush():81] Configure stats pid to 2043723
|
| 3 |
+
2026-07-31 01:15:13,687 INFO MainThread:2043723 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-07-31 01:15:13,687 INFO MainThread:2043723 [wandb_init.py:setup_run_log_directory():725] Logging user logs to outputs/healed/mixonly_keep50/wandb/offline-run-20260731_011513-gexxktae/logs/debug.log
|
| 5 |
+
2026-07-31 01:15:13,687 INFO MainThread:2043723 [wandb_init.py:setup_run_log_directory():726] Logging internal logs to outputs/healed/mixonly_keep50/wandb/offline-run-20260731_011513-gexxktae/logs/debug-internal.log
|
| 6 |
+
2026-07-31 01:15:13,687 INFO MainThread:2043723 [wandb_init.py:init():768] calling init triggers
|
| 7 |
+
2026-07-31 01:15:13,687 INFO MainThread:2043723 [wandb_init.py:init():773] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'student': 'outputs/pruned/glean-0125inst-math-keep50', 'teacher': 'allenai/OLMoE-1B-7B-0125-Instruct', 'training_mode': 'on-policy', 'kl_direction': 'reverse', 'dataset': 'allenai/Dolci-Instruct-RL', 'dataset_sources': None, 'max_difficulty': None, 'trajectories': 'outputs/teacher_trajectories/dolci_math_curated.jsonl', 'trajectory_dataset': 'allenai/Dolci-Instruct-RL', 'off_policy_frames': 'chat', 'off_policy_max_seq_len': 2048, 'topk_targets': None, 'max_loss_tokens': None, 'loss_tokens_per_step': None, 'teacher_device': 'cuda:0', 'student_device': 'cuda:1', 'lr': 3e-05, 'optimizer': 'adamw8bit', 'weight_decay': 0.1, 'epochs': 2, 'prompts_per_step': 256, 'group_size': 4, 'rollout_batch': 64, 'micro_batch': 4, 'max_new_tokens': 2048, 'max_prompt_len': 1024, 'warmup_steps': 10, 'max_grad_norm': 1.0, 'eval_every': 10, 'gsm8k_every': 20, 'gsm8k_n': 256, 'gsm8k_batch': 16, 'gsm8k_max_new_tokens': 1024, 'gsm8k_frames': 'chat', 'save_every': 1000, 'out_dir': 'outputs/healed/mixonly_keep50', 'sweep': 60, 'wandb': True, 'wandb_project': 'glean-heal', 'wandb_run_name': 'mixonly_keep50-s1223', 'wandb_run_id': None, 'wandb_resume': None, 'wandb_mode': 'offline', 'no_wandb_sync': False, 'debug': False, 'resume_from': None, 'start_step': 0, 'no_grad_checkpointing': False, 'seed': 1223, 'no_teacher_overlap': False, 'sync_checkpoints': False, 'rollout_engine': 'vllm', 'vllm_gpu': '2', 'vllm_port': 8377, 'vllm_refresh_every': 1, 'vllm_serve_bin': 'vllm-plugin/.venv/bin/python', 'vllm_gpu_mem_util': 0.85, 'liger_loss': True, 'gold_mix_lambda': 0.5, 'gold_topk_targets': 'outputs/teacher_trajectories/dolci_combined_top128', 'gold_mix_decay': 0.0, 'fast_teacher': True, 'reference_kl_beta': 0.0, 'drop_truncated_rollouts': False, 'vllm_max_model_len': None, 'vllm_refresh_mode': 'reload', 'vllm_live_dir': None, 'resolved_kl_direction': 'reverse', '_wandb': {}}
|
| 9 |
+
2026-07-31 01:15:13,687 INFO MainThread:2043723 [wandb_init.py:init():816] starting backend
|
| 10 |
+
2026-07-31 01:15:14,134 INFO MainThread:2043723 [wandb_init.py:init():831] sending inform_init request
|
| 11 |
+
2026-07-31 01:15:14,256 INFO MainThread:2043723 [wandb_init.py:init():836] backend started and connected
|
| 12 |
+
2026-07-31 01:15:14,258 INFO MainThread:2043723 [wandb_init.py:init():906] updated telemetry
|
| 13 |
+
2026-07-31 01:15:14,265 INFO MainThread:2043723 [wandb_init.py:init():929] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-07-31 01:15:14,273 INFO MainThread:2043723 [wandb_init.py:init():974] starting run threads in backend
|
| 15 |
+
2026-07-31 01:15:14,385 INFO MainThread:2043723 [wandb_run.py:_console_start():2523] atexit reg
|
| 16 |
+
2026-07-31 01:15:14,386 INFO MainThread:2043723 [wandb_run.py:_redirect():2373] redirect: wrap_raw
|
| 17 |
+
2026-07-31 01:15:14,386 INFO MainThread:2043723 [wandb_run.py:_redirect():2442] Wrapping output streams.
|
| 18 |
+
2026-07-31 01:15:14,386 INFO MainThread:2043723 [wandb_run.py:_redirect():2465] Redirects installed.
|
| 19 |
+
2026-07-31 01:15:14,387 INFO MainThread:2043723 [wandb_init.py:init():1012] run started, returning control to user process
|
healed/mixonly_keep50/wandb/offline-run-20260731_011513-gexxktae/files/requirements.txt
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
certifi==2026.6.17
|
| 2 |
+
fonttools==4.63.0
|
| 3 |
+
urllib3==2.7.0
|
| 4 |
+
requests==2.34.2
|
| 5 |
+
nvidia-cublas-cu12==12.6.4.1
|
| 6 |
+
packaging==26.2
|
| 7 |
+
regex==2026.6.28
|
| 8 |
+
portalocker==3.2.0
|
| 9 |
+
safetensors==0.8.0
|
| 10 |
+
datasets==5.0.0
|
| 11 |
+
sqlitedict==2.1.0
|
| 12 |
+
narwhals==2.23.0
|
| 13 |
+
latex2sympy2_extended==1.11.0
|
| 14 |
+
smmap==5.0.3
|
| 15 |
+
colorama==0.4.6
|
| 16 |
+
anyio==4.14.1
|
| 17 |
+
cycler==0.12.1
|
| 18 |
+
pytablewriter==1.2.1
|
| 19 |
+
pydantic_core==2.46.4
|
| 20 |
+
huggingface_hub==0.36.2
|
| 21 |
+
charset-normalizer==3.4.8
|
| 22 |
+
nvidia-cusolver-cu12==11.7.1.2
|
| 23 |
+
joblib==1.5.3
|
| 24 |
+
typing-inspection==0.4.2
|
| 25 |
+
wandb==0.28.0
|
| 26 |
+
bitsandbytes==0.49.2
|
| 27 |
+
kiwisolver==1.5.0
|
| 28 |
+
cuda-toolkit==12.6.3
|
| 29 |
+
propcache==0.5.2
|
| 30 |
+
Jinja2==3.1.6
|
| 31 |
+
cuda-bindings==12.9.7
|
| 32 |
+
pluggy==1.6.0
|
| 33 |
+
pytest==9.1.1
|
| 34 |
+
fsspec==2026.4.0
|
| 35 |
+
tqdm==4.68.3
|
| 36 |
+
psutil==7.2.2
|
| 37 |
+
pydantic==2.13.4
|
| 38 |
+
typepy==1.3.5
|
| 39 |
+
xxhash==3.8.1
|
| 40 |
+
DataProperty==1.1.1
|
| 41 |
+
aiosignal==1.4.0
|
| 42 |
+
annotated-types==0.7.0
|
| 43 |
+
Pygments==2.20.0
|
| 44 |
+
zstandard==0.25.0
|
| 45 |
+
threadpoolctl==3.6.0
|
| 46 |
+
liger_kernel==0.8.1
|
| 47 |
+
nvidia-cudnn-cu12==9.10.2.21
|
| 48 |
+
evaluate==0.4.6
|
| 49 |
+
pillow==12.3.0
|
| 50 |
+
GitPython==3.1.50
|
| 51 |
+
httpx==0.28.1
|
| 52 |
+
lm_eval==0.4.12
|
| 53 |
+
pytz==2026.2
|
| 54 |
+
mbstrdecoder==1.1.5
|
| 55 |
+
immutabledict==4.3.1
|
| 56 |
+
dill==0.4.1
|
| 57 |
+
mpmath==1.3.0
|
| 58 |
+
yarl==1.24.2
|
| 59 |
+
word2number==1.1
|
| 60 |
+
networkx==3.6.1
|
| 61 |
+
einops==0.8.2
|
| 62 |
+
tabulate==0.10.0
|
| 63 |
+
pathvalidate==3.3.1
|
| 64 |
+
tenacity==9.1.4
|
| 65 |
+
sentry-sdk==2.64.0
|
| 66 |
+
nvidia-nvshmem-cu12==3.4.5
|
| 67 |
+
hf-xet==1.5.1
|
| 68 |
+
six==1.17.0
|
| 69 |
+
pyarrow==24.0.0
|
| 70 |
+
nvidia-curand-cu12==10.3.7.77
|
| 71 |
+
cuda-pathfinder==1.5.6
|
| 72 |
+
sacrebleu==2.6.0
|
| 73 |
+
triton==3.7.1
|
| 74 |
+
frozenlist==1.8.0
|
| 75 |
+
antlr4-python3-runtime==4.11.0
|
| 76 |
+
tabledata==1.3.5
|
| 77 |
+
h11==0.16.0
|
| 78 |
+
nvidia-cusparselt-cu12==0.7.1
|
| 79 |
+
nvidia-nvjitlink-cu12==12.6.85
|
| 80 |
+
scipy==1.18.0
|
| 81 |
+
rouge_score==0.1.2
|
| 82 |
+
transformers==4.57.6
|
| 83 |
+
multiprocess==0.70.19
|
| 84 |
+
absl-py==2.5.0
|
| 85 |
+
accelerate==1.14.0
|
| 86 |
+
click==8.4.2
|
| 87 |
+
lxml==6.1.1
|
| 88 |
+
defusedxml==0.7.1
|
| 89 |
+
setuptools==81.0.0
|
| 90 |
+
nvidia-cuda-cupti-cu12==12.6.80
|
| 91 |
+
httpcore==1.0.9
|
| 92 |
+
langdetect==1.0.9
|
| 93 |
+
nltk==3.10.0
|
| 94 |
+
iniconfig==2.3.0
|
| 95 |
+
nvidia-cuda-runtime-cu12==12.6.77
|
| 96 |
+
nvidia-cusparse-cu12==12.5.4.2
|
| 97 |
+
matplotlib==3.11.0
|
| 98 |
+
protobuf==7.35.1
|
| 99 |
+
nvidia-cufft-cu12==11.3.0.4
|
| 100 |
+
pyparsing==3.3.2
|
| 101 |
+
chardet==6.0.0.post1
|
| 102 |
+
typing_extensions==4.16.0
|
| 103 |
+
stanford-stk==0.7.1
|
| 104 |
+
aiohappyeyeballs==2.7.1
|
| 105 |
+
MarkupSafe==3.0.3
|
| 106 |
+
nvidia-cufile-cu12==1.11.1.6
|
| 107 |
+
tcolorpy==0.1.7
|
| 108 |
+
scikit-learn==1.9.0
|
| 109 |
+
tokenizers==0.22.2
|
| 110 |
+
attrs==26.1.0
|
| 111 |
+
gitdb==4.0.12
|
| 112 |
+
aiohttp==3.14.1
|
| 113 |
+
glean==0.0.1
|
| 114 |
+
platformdirs==4.10.0
|
| 115 |
+
megablocks==0.11.0.dev0
|
| 116 |
+
nvidia-cuda-nvrtc-cu12==12.6.85
|
| 117 |
+
pandas==3.0.3
|
| 118 |
+
multidict==6.7.1
|
| 119 |
+
torch==2.12.1+cu126
|
| 120 |
+
contourpy==1.3.3
|
| 121 |
+
nvidia-nvtx-cu12==12.6.77
|
| 122 |
+
math-verify==0.9.0
|
| 123 |
+
filelock==3.29.5
|
| 124 |
+
idna==3.18
|
| 125 |
+
nvidia-nccl-cu12==2.29.3
|
| 126 |
+
python-dateutil==2.9.0.post0
|
| 127 |
+
more-itertools==11.1.0
|
| 128 |
+
numpy==2.0.2
|
| 129 |
+
sympy==1.14.0
|
| 130 |
+
PyYAML==6.0.3
|
healed/mixonly_keep50/wandb/offline-run-20260731_011513-gexxktae/logs/debug-core.log
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-07-31T01:15:13.959518652-07:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpbrwnypt3/port-2043723.txt","pid":2043723,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
|
| 2 |
+
{"time":"2026-07-31T01:15:13.960346877-07:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":2043723}
|
| 3 |
+
{"time":"2026-07-31T01:15:13.960308186-07:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-2043723-2043827-3299481953/socket","Net":"unix"}}
|
| 4 |
+
{"time":"2026-07-31T01:15:14.134409981-07:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
|
| 5 |
+
{"time":"2026-07-31T01:15:14.142865012-07:00","level":"INFO","msg":"handleInformInit: received","streamId":"gexxktae","id":"1(@)"}
|
| 6 |
+
{"time":"2026-07-31T01:15:14.255776822-07:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"gexxktae","id":"1(@)"}
|
healed/mixonly_keep50/wandb/offline-run-20260731_011513-gexxktae/logs/debug-internal.log
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-07-31T01:15:14.143006306-07:00","level":"INFO","msg":"wandb-core"}
|
| 2 |
+
{"time":"2026-07-31T01:15:14.143036247-07:00","level":"INFO","msg":"stream: starting","core version":"0.28.0"}
|
| 3 |
+
{"time":"2026-07-31T01:15:14.255553325-07:00","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
|
| 4 |
+
{"time":"2026-07-31T01:15:14.255573436-07:00","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
|
| 5 |
+
{"time":"2026-07-31T01:15:14.255616137-07:00","level":"INFO","msg":"stream: created new stream","id":"gexxktae"}
|
| 6 |
+
{"time":"2026-07-31T01:15:14.255765612-07:00","level":"INFO","msg":"stream: started"}
|
| 7 |
+
{"time":"2026-07-31T01:15:14.255812423-07:00","level":"INFO","msg":"writer: started","stream_id":"gexxktae"}
|
| 8 |
+
{"time":"2026-07-31T01:15:14.255852534-07:00","level":"INFO","msg":"handler: started"}
|
| 9 |
+
{"time":"2026-07-31T01:15:14.255895105-07:00","level":"INFO","msg":"sender: started"}
|
| 10 |
+
{"time":"2026-07-31T01:15:14.271152628-07:00","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
|
| 11 |
+
{"time":"2026-07-31T01:15:14.271190299-07:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"}
|
healed/mixonly_keep50/wandb/offline-run-20260731_011513-gexxktae/logs/debug.log
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-07-31 01:15:13,687 INFO MainThread:2043723 [wandb_setup.py:_flush():81] Current SDK version is 0.28.0
|
| 2 |
+
2026-07-31 01:15:13,687 INFO MainThread:2043723 [wandb_setup.py:_flush():81] Configure stats pid to 2043723
|
| 3 |
+
2026-07-31 01:15:13,687 INFO MainThread:2043723 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-07-31 01:15:13,687 INFO MainThread:2043723 [wandb_init.py:setup_run_log_directory():725] Logging user logs to outputs/healed/mixonly_keep50/wandb/offline-run-20260731_011513-gexxktae/logs/debug.log
|
| 5 |
+
2026-07-31 01:15:13,687 INFO MainThread:2043723 [wandb_init.py:setup_run_log_directory():726] Logging internal logs to outputs/healed/mixonly_keep50/wandb/offline-run-20260731_011513-gexxktae/logs/debug-internal.log
|
| 6 |
+
2026-07-31 01:15:13,687 INFO MainThread:2043723 [wandb_init.py:init():768] calling init triggers
|
| 7 |
+
2026-07-31 01:15:13,687 INFO MainThread:2043723 [wandb_init.py:init():773] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'student': 'outputs/pruned/glean-0125inst-math-keep50', 'teacher': 'allenai/OLMoE-1B-7B-0125-Instruct', 'training_mode': 'on-policy', 'kl_direction': 'reverse', 'dataset': 'allenai/Dolci-Instruct-RL', 'dataset_sources': None, 'max_difficulty': None, 'trajectories': 'outputs/teacher_trajectories/dolci_math_curated.jsonl', 'trajectory_dataset': 'allenai/Dolci-Instruct-RL', 'off_policy_frames': 'chat', 'off_policy_max_seq_len': 2048, 'topk_targets': None, 'max_loss_tokens': None, 'loss_tokens_per_step': None, 'teacher_device': 'cuda:0', 'student_device': 'cuda:1', 'lr': 3e-05, 'optimizer': 'adamw8bit', 'weight_decay': 0.1, 'epochs': 2, 'prompts_per_step': 256, 'group_size': 4, 'rollout_batch': 64, 'micro_batch': 4, 'max_new_tokens': 2048, 'max_prompt_len': 1024, 'warmup_steps': 10, 'max_grad_norm': 1.0, 'eval_every': 10, 'gsm8k_every': 20, 'gsm8k_n': 256, 'gsm8k_batch': 16, 'gsm8k_max_new_tokens': 1024, 'gsm8k_frames': 'chat', 'save_every': 1000, 'out_dir': 'outputs/healed/mixonly_keep50', 'sweep': 60, 'wandb': True, 'wandb_project': 'glean-heal', 'wandb_run_name': 'mixonly_keep50-s1223', 'wandb_run_id': None, 'wandb_resume': None, 'wandb_mode': 'offline', 'no_wandb_sync': False, 'debug': False, 'resume_from': None, 'start_step': 0, 'no_grad_checkpointing': False, 'seed': 1223, 'no_teacher_overlap': False, 'sync_checkpoints': False, 'rollout_engine': 'vllm', 'vllm_gpu': '2', 'vllm_port': 8377, 'vllm_refresh_every': 1, 'vllm_serve_bin': 'vllm-plugin/.venv/bin/python', 'vllm_gpu_mem_util': 0.85, 'liger_loss': True, 'gold_mix_lambda': 0.5, 'gold_topk_targets': 'outputs/teacher_trajectories/dolci_combined_top128', 'gold_mix_decay': 0.0, 'fast_teacher': True, 'reference_kl_beta': 0.0, 'drop_truncated_rollouts': False, 'vllm_max_model_len': None, 'vllm_refresh_mode': 'reload', 'vllm_live_dir': None, 'resolved_kl_direction': 'reverse', '_wandb': {}}
|
| 9 |
+
2026-07-31 01:15:13,687 INFO MainThread:2043723 [wandb_init.py:init():816] starting backend
|
| 10 |
+
2026-07-31 01:15:14,134 INFO MainThread:2043723 [wandb_init.py:init():831] sending inform_init request
|
| 11 |
+
2026-07-31 01:15:14,256 INFO MainThread:2043723 [wandb_init.py:init():836] backend started and connected
|
| 12 |
+
2026-07-31 01:15:14,258 INFO MainThread:2043723 [wandb_init.py:init():906] updated telemetry
|
| 13 |
+
2026-07-31 01:15:14,265 INFO MainThread:2043723 [wandb_init.py:init():929] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-07-31 01:15:14,273 INFO MainThread:2043723 [wandb_init.py:init():974] starting run threads in backend
|
| 15 |
+
2026-07-31 01:15:14,385 INFO MainThread:2043723 [wandb_run.py:_console_start():2523] atexit reg
|
| 16 |
+
2026-07-31 01:15:14,386 INFO MainThread:2043723 [wandb_run.py:_redirect():2373] redirect: wrap_raw
|
| 17 |
+
2026-07-31 01:15:14,386 INFO MainThread:2043723 [wandb_run.py:_redirect():2442] Wrapping output streams.
|
| 18 |
+
2026-07-31 01:15:14,386 INFO MainThread:2043723 [wandb_run.py:_redirect():2465] Redirects installed.
|
| 19 |
+
2026-07-31 01:15:14,387 INFO MainThread:2043723 [wandb_init.py:init():1012] run started, returning control to user process
|
healed/opd_warm_fixed_keep50/step0120/chat_template.jinja
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{{ bos_token }}{% for message in messages %}{% if message['role'] == 'system' %}{{ '<|system|>
|
| 2 |
+
' + message['content'] + '
|
| 3 |
+
' }}{% elif message['role'] == 'user' %}{{ '<|user|>
|
| 4 |
+
' + message['content'] + '
|
| 5 |
+
' }}{% elif message['role'] == 'assistant' %}{% if not loop.last %}{{ '<|assistant|>
|
| 6 |
+
' + message['content'] + eos_token + '
|
| 7 |
+
' }}{% else %}{{ '<|assistant|>
|
| 8 |
+
' + message['content'] + eos_token }}{% endif %}{% endif %}{% if loop.last and add_generation_prompt %}{{ '<|assistant|>
|
| 9 |
+
' }}{% endif %}{% endfor %}
|
healed/opd_warm_fixed_keep50/step0120/config.json
ADDED
|
@@ -0,0 +1,887 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"architectures": [
|
| 3 |
+
"PrunedOlmoeForCausalLM"
|
| 4 |
+
],
|
| 5 |
+
"attention_bias": false,
|
| 6 |
+
"attention_dropout": 0.0,
|
| 7 |
+
"auto_map": {
|
| 8 |
+
"AutoConfig": "configuration_pruned_olmoe.PrunedOlmoeConfig",
|
| 9 |
+
"AutoModelForCausalLM": "modeling_pruned_olmoe.PrunedOlmoeForCausalLM"
|
| 10 |
+
},
|
| 11 |
+
"clip_qkv": null,
|
| 12 |
+
"dtype": "bfloat16",
|
| 13 |
+
"eos_token_id": 50279,
|
| 14 |
+
"expert_widths": [
|
| 15 |
+
[
|
| 16 |
+
1024,
|
| 17 |
+
384,
|
| 18 |
+
256,
|
| 19 |
+
256,
|
| 20 |
+
768,
|
| 21 |
+
1024,
|
| 22 |
+
128,
|
| 23 |
+
768,
|
| 24 |
+
384,
|
| 25 |
+
128,
|
| 26 |
+
1024,
|
| 27 |
+
384,
|
| 28 |
+
384,
|
| 29 |
+
128,
|
| 30 |
+
640,
|
| 31 |
+
896,
|
| 32 |
+
896,
|
| 33 |
+
256,
|
| 34 |
+
768,
|
| 35 |
+
128,
|
| 36 |
+
768,
|
| 37 |
+
768,
|
| 38 |
+
768,
|
| 39 |
+
768,
|
| 40 |
+
640,
|
| 41 |
+
896,
|
| 42 |
+
256,
|
| 43 |
+
128,
|
| 44 |
+
384,
|
| 45 |
+
896,
|
| 46 |
+
256,
|
| 47 |
+
768,
|
| 48 |
+
512,
|
| 49 |
+
640,
|
| 50 |
+
768,
|
| 51 |
+
896,
|
| 52 |
+
1024,
|
| 53 |
+
384,
|
| 54 |
+
384,
|
| 55 |
+
512,
|
| 56 |
+
384,
|
| 57 |
+
256,
|
| 58 |
+
768,
|
| 59 |
+
640,
|
| 60 |
+
896,
|
| 61 |
+
896,
|
| 62 |
+
512,
|
| 63 |
+
768,
|
| 64 |
+
512,
|
| 65 |
+
384,
|
| 66 |
+
1024,
|
| 67 |
+
896,
|
| 68 |
+
896,
|
| 69 |
+
768,
|
| 70 |
+
128
|
| 71 |
+
],
|
| 72 |
+
[
|
| 73 |
+
768,
|
| 74 |
+
896,
|
| 75 |
+
896,
|
| 76 |
+
256,
|
| 77 |
+
896,
|
| 78 |
+
384,
|
| 79 |
+
768,
|
| 80 |
+
768,
|
| 81 |
+
1024,
|
| 82 |
+
384,
|
| 83 |
+
512,
|
| 84 |
+
1024,
|
| 85 |
+
768,
|
| 86 |
+
768,
|
| 87 |
+
1024,
|
| 88 |
+
640,
|
| 89 |
+
128,
|
| 90 |
+
768,
|
| 91 |
+
384,
|
| 92 |
+
768,
|
| 93 |
+
128,
|
| 94 |
+
256,
|
| 95 |
+
512,
|
| 96 |
+
384,
|
| 97 |
+
896,
|
| 98 |
+
256,
|
| 99 |
+
1024,
|
| 100 |
+
1024,
|
| 101 |
+
896,
|
| 102 |
+
1024,
|
| 103 |
+
896,
|
| 104 |
+
768,
|
| 105 |
+
256,
|
| 106 |
+
768,
|
| 107 |
+
128,
|
| 108 |
+
1024,
|
| 109 |
+
768,
|
| 110 |
+
768,
|
| 111 |
+
640,
|
| 112 |
+
384,
|
| 113 |
+
1024,
|
| 114 |
+
384,
|
| 115 |
+
640,
|
| 116 |
+
896,
|
| 117 |
+
1024,
|
| 118 |
+
512,
|
| 119 |
+
512,
|
| 120 |
+
512,
|
| 121 |
+
640
|
| 122 |
+
],
|
| 123 |
+
[
|
| 124 |
+
768,
|
| 125 |
+
1024,
|
| 126 |
+
512,
|
| 127 |
+
384,
|
| 128 |
+
384,
|
| 129 |
+
768,
|
| 130 |
+
256,
|
| 131 |
+
384,
|
| 132 |
+
1024,
|
| 133 |
+
896,
|
| 134 |
+
640,
|
| 135 |
+
256,
|
| 136 |
+
1024,
|
| 137 |
+
1024,
|
| 138 |
+
1024,
|
| 139 |
+
640,
|
| 140 |
+
1024,
|
| 141 |
+
896,
|
| 142 |
+
512,
|
| 143 |
+
768,
|
| 144 |
+
128,
|
| 145 |
+
768,
|
| 146 |
+
1024,
|
| 147 |
+
768,
|
| 148 |
+
256,
|
| 149 |
+
1024,
|
| 150 |
+
896,
|
| 151 |
+
1024,
|
| 152 |
+
896,
|
| 153 |
+
512,
|
| 154 |
+
640,
|
| 155 |
+
384,
|
| 156 |
+
896,
|
| 157 |
+
384,
|
| 158 |
+
512,
|
| 159 |
+
640,
|
| 160 |
+
256,
|
| 161 |
+
768,
|
| 162 |
+
256,
|
| 163 |
+
768,
|
| 164 |
+
768,
|
| 165 |
+
640,
|
| 166 |
+
1024,
|
| 167 |
+
640,
|
| 168 |
+
896,
|
| 169 |
+
768,
|
| 170 |
+
768,
|
| 171 |
+
256
|
| 172 |
+
],
|
| 173 |
+
[
|
| 174 |
+
640,
|
| 175 |
+
640,
|
| 176 |
+
896,
|
| 177 |
+
512,
|
| 178 |
+
640,
|
| 179 |
+
896,
|
| 180 |
+
896,
|
| 181 |
+
384,
|
| 182 |
+
896,
|
| 183 |
+
896,
|
| 184 |
+
384,
|
| 185 |
+
896,
|
| 186 |
+
640,
|
| 187 |
+
256,
|
| 188 |
+
640,
|
| 189 |
+
1024,
|
| 190 |
+
1024,
|
| 191 |
+
768,
|
| 192 |
+
1024,
|
| 193 |
+
896,
|
| 194 |
+
1024,
|
| 195 |
+
768,
|
| 196 |
+
896,
|
| 197 |
+
256,
|
| 198 |
+
512,
|
| 199 |
+
768,
|
| 200 |
+
1024,
|
| 201 |
+
256,
|
| 202 |
+
768,
|
| 203 |
+
512,
|
| 204 |
+
256,
|
| 205 |
+
640,
|
| 206 |
+
1024,
|
| 207 |
+
1024,
|
| 208 |
+
512,
|
| 209 |
+
1024,
|
| 210 |
+
768,
|
| 211 |
+
256,
|
| 212 |
+
1024,
|
| 213 |
+
384,
|
| 214 |
+
896,
|
| 215 |
+
1024,
|
| 216 |
+
896,
|
| 217 |
+
1024,
|
| 218 |
+
384
|
| 219 |
+
],
|
| 220 |
+
[
|
| 221 |
+
256,
|
| 222 |
+
640,
|
| 223 |
+
640,
|
| 224 |
+
896,
|
| 225 |
+
768,
|
| 226 |
+
896,
|
| 227 |
+
768,
|
| 228 |
+
768,
|
| 229 |
+
896,
|
| 230 |
+
896,
|
| 231 |
+
1024,
|
| 232 |
+
256,
|
| 233 |
+
512,
|
| 234 |
+
1024,
|
| 235 |
+
640,
|
| 236 |
+
896,
|
| 237 |
+
512,
|
| 238 |
+
512,
|
| 239 |
+
384,
|
| 240 |
+
384,
|
| 241 |
+
256,
|
| 242 |
+
384,
|
| 243 |
+
384,
|
| 244 |
+
896,
|
| 245 |
+
896,
|
| 246 |
+
768,
|
| 247 |
+
640,
|
| 248 |
+
896,
|
| 249 |
+
768,
|
| 250 |
+
1024,
|
| 251 |
+
512,
|
| 252 |
+
640,
|
| 253 |
+
512,
|
| 254 |
+
640,
|
| 255 |
+
896,
|
| 256 |
+
512,
|
| 257 |
+
512,
|
| 258 |
+
384,
|
| 259 |
+
640,
|
| 260 |
+
896,
|
| 261 |
+
896,
|
| 262 |
+
896,
|
| 263 |
+
1024,
|
| 264 |
+
640,
|
| 265 |
+
1024,
|
| 266 |
+
640,
|
| 267 |
+
1024
|
| 268 |
+
],
|
| 269 |
+
[
|
| 270 |
+
1024,
|
| 271 |
+
512,
|
| 272 |
+
1024,
|
| 273 |
+
1024,
|
| 274 |
+
640,
|
| 275 |
+
896,
|
| 276 |
+
640,
|
| 277 |
+
1024,
|
| 278 |
+
896,
|
| 279 |
+
384,
|
| 280 |
+
1024,
|
| 281 |
+
128,
|
| 282 |
+
896,
|
| 283 |
+
768,
|
| 284 |
+
1024,
|
| 285 |
+
768,
|
| 286 |
+
640,
|
| 287 |
+
896,
|
| 288 |
+
768,
|
| 289 |
+
640,
|
| 290 |
+
512,
|
| 291 |
+
896,
|
| 292 |
+
512,
|
| 293 |
+
640,
|
| 294 |
+
256,
|
| 295 |
+
768,
|
| 296 |
+
640,
|
| 297 |
+
768,
|
| 298 |
+
384,
|
| 299 |
+
896,
|
| 300 |
+
512,
|
| 301 |
+
512,
|
| 302 |
+
256,
|
| 303 |
+
512,
|
| 304 |
+
896,
|
| 305 |
+
256,
|
| 306 |
+
384,
|
| 307 |
+
640,
|
| 308 |
+
512,
|
| 309 |
+
640,
|
| 310 |
+
896,
|
| 311 |
+
512,
|
| 312 |
+
1024,
|
| 313 |
+
256,
|
| 314 |
+
768,
|
| 315 |
+
1024,
|
| 316 |
+
768,
|
| 317 |
+
256,
|
| 318 |
+
256
|
| 319 |
+
],
|
| 320 |
+
[
|
| 321 |
+
640,
|
| 322 |
+
896,
|
| 323 |
+
1024,
|
| 324 |
+
896,
|
| 325 |
+
1024,
|
| 326 |
+
1024,
|
| 327 |
+
1024,
|
| 328 |
+
512,
|
| 329 |
+
256,
|
| 330 |
+
256,
|
| 331 |
+
1024,
|
| 332 |
+
768,
|
| 333 |
+
512,
|
| 334 |
+
768,
|
| 335 |
+
1024,
|
| 336 |
+
1024,
|
| 337 |
+
1024,
|
| 338 |
+
384,
|
| 339 |
+
512,
|
| 340 |
+
1024,
|
| 341 |
+
512,
|
| 342 |
+
1024,
|
| 343 |
+
128,
|
| 344 |
+
640,
|
| 345 |
+
640,
|
| 346 |
+
896,
|
| 347 |
+
768,
|
| 348 |
+
128,
|
| 349 |
+
256,
|
| 350 |
+
256,
|
| 351 |
+
256,
|
| 352 |
+
256,
|
| 353 |
+
896,
|
| 354 |
+
1024,
|
| 355 |
+
1024,
|
| 356 |
+
384,
|
| 357 |
+
896,
|
| 358 |
+
256,
|
| 359 |
+
896,
|
| 360 |
+
640,
|
| 361 |
+
1024,
|
| 362 |
+
384,
|
| 363 |
+
640,
|
| 364 |
+
256,
|
| 365 |
+
1024,
|
| 366 |
+
1024,
|
| 367 |
+
1024
|
| 368 |
+
],
|
| 369 |
+
[
|
| 370 |
+
1024,
|
| 371 |
+
384,
|
| 372 |
+
1024,
|
| 373 |
+
1024,
|
| 374 |
+
256,
|
| 375 |
+
128,
|
| 376 |
+
256,
|
| 377 |
+
384,
|
| 378 |
+
256,
|
| 379 |
+
384,
|
| 380 |
+
896,
|
| 381 |
+
768,
|
| 382 |
+
896,
|
| 383 |
+
896,
|
| 384 |
+
512,
|
| 385 |
+
896,
|
| 386 |
+
640,
|
| 387 |
+
384,
|
| 388 |
+
384,
|
| 389 |
+
896,
|
| 390 |
+
768,
|
| 391 |
+
384,
|
| 392 |
+
896,
|
| 393 |
+
768,
|
| 394 |
+
768,
|
| 395 |
+
512,
|
| 396 |
+
896,
|
| 397 |
+
768,
|
| 398 |
+
768,
|
| 399 |
+
896,
|
| 400 |
+
128,
|
| 401 |
+
896,
|
| 402 |
+
512,
|
| 403 |
+
256,
|
| 404 |
+
768,
|
| 405 |
+
128,
|
| 406 |
+
384,
|
| 407 |
+
256,
|
| 408 |
+
896,
|
| 409 |
+
896,
|
| 410 |
+
384,
|
| 411 |
+
768,
|
| 412 |
+
512,
|
| 413 |
+
640,
|
| 414 |
+
256,
|
| 415 |
+
768,
|
| 416 |
+
640,
|
| 417 |
+
896,
|
| 418 |
+
384,
|
| 419 |
+
512,
|
| 420 |
+
1024,
|
| 421 |
+
768,
|
| 422 |
+
384
|
| 423 |
+
],
|
| 424 |
+
[
|
| 425 |
+
512,
|
| 426 |
+
768,
|
| 427 |
+
512,
|
| 428 |
+
256,
|
| 429 |
+
128,
|
| 430 |
+
640,
|
| 431 |
+
384,
|
| 432 |
+
640,
|
| 433 |
+
768,
|
| 434 |
+
896,
|
| 435 |
+
640,
|
| 436 |
+
768,
|
| 437 |
+
256,
|
| 438 |
+
384,
|
| 439 |
+
1024,
|
| 440 |
+
896,
|
| 441 |
+
256,
|
| 442 |
+
896,
|
| 443 |
+
512,
|
| 444 |
+
256,
|
| 445 |
+
896,
|
| 446 |
+
768,
|
| 447 |
+
256,
|
| 448 |
+
896,
|
| 449 |
+
896,
|
| 450 |
+
384,
|
| 451 |
+
896,
|
| 452 |
+
640,
|
| 453 |
+
768,
|
| 454 |
+
512,
|
| 455 |
+
768,
|
| 456 |
+
768,
|
| 457 |
+
1024,
|
| 458 |
+
768,
|
| 459 |
+
640,
|
| 460 |
+
768,
|
| 461 |
+
384,
|
| 462 |
+
256,
|
| 463 |
+
512,
|
| 464 |
+
896,
|
| 465 |
+
128,
|
| 466 |
+
384,
|
| 467 |
+
256,
|
| 468 |
+
768,
|
| 469 |
+
384,
|
| 470 |
+
256,
|
| 471 |
+
1024,
|
| 472 |
+
1024,
|
| 473 |
+
896,
|
| 474 |
+
256,
|
| 475 |
+
1024,
|
| 476 |
+
256,
|
| 477 |
+
128,
|
| 478 |
+
896
|
| 479 |
+
],
|
| 480 |
+
[
|
| 481 |
+
640,
|
| 482 |
+
640,
|
| 483 |
+
896,
|
| 484 |
+
256,
|
| 485 |
+
1024,
|
| 486 |
+
512,
|
| 487 |
+
1024,
|
| 488 |
+
768,
|
| 489 |
+
384,
|
| 490 |
+
512,
|
| 491 |
+
256,
|
| 492 |
+
768,
|
| 493 |
+
896,
|
| 494 |
+
768,
|
| 495 |
+
512,
|
| 496 |
+
768,
|
| 497 |
+
768,
|
| 498 |
+
640,
|
| 499 |
+
384,
|
| 500 |
+
768,
|
| 501 |
+
512,
|
| 502 |
+
768,
|
| 503 |
+
768,
|
| 504 |
+
512,
|
| 505 |
+
768,
|
| 506 |
+
128,
|
| 507 |
+
896,
|
| 508 |
+
512,
|
| 509 |
+
768,
|
| 510 |
+
1024,
|
| 511 |
+
128,
|
| 512 |
+
384,
|
| 513 |
+
768,
|
| 514 |
+
768,
|
| 515 |
+
768,
|
| 516 |
+
384,
|
| 517 |
+
512,
|
| 518 |
+
640,
|
| 519 |
+
768,
|
| 520 |
+
512,
|
| 521 |
+
768,
|
| 522 |
+
1024,
|
| 523 |
+
640,
|
| 524 |
+
896,
|
| 525 |
+
256,
|
| 526 |
+
1024,
|
| 527 |
+
384,
|
| 528 |
+
768,
|
| 529 |
+
768,
|
| 530 |
+
768
|
| 531 |
+
],
|
| 532 |
+
[
|
| 533 |
+
896,
|
| 534 |
+
512,
|
| 535 |
+
896,
|
| 536 |
+
768,
|
| 537 |
+
384,
|
| 538 |
+
384,
|
| 539 |
+
768,
|
| 540 |
+
512,
|
| 541 |
+
768,
|
| 542 |
+
512,
|
| 543 |
+
1024,
|
| 544 |
+
640,
|
| 545 |
+
896,
|
| 546 |
+
896,
|
| 547 |
+
256,
|
| 548 |
+
640,
|
| 549 |
+
1024,
|
| 550 |
+
256,
|
| 551 |
+
896,
|
| 552 |
+
128,
|
| 553 |
+
128,
|
| 554 |
+
128,
|
| 555 |
+
768,
|
| 556 |
+
896,
|
| 557 |
+
384,
|
| 558 |
+
896,
|
| 559 |
+
512,
|
| 560 |
+
896,
|
| 561 |
+
384,
|
| 562 |
+
256,
|
| 563 |
+
640,
|
| 564 |
+
640,
|
| 565 |
+
896,
|
| 566 |
+
768,
|
| 567 |
+
640,
|
| 568 |
+
256,
|
| 569 |
+
896,
|
| 570 |
+
896,
|
| 571 |
+
512,
|
| 572 |
+
128,
|
| 573 |
+
896,
|
| 574 |
+
256,
|
| 575 |
+
256,
|
| 576 |
+
640,
|
| 577 |
+
896,
|
| 578 |
+
896,
|
| 579 |
+
128,
|
| 580 |
+
1024,
|
| 581 |
+
256,
|
| 582 |
+
384,
|
| 583 |
+
1024,
|
| 584 |
+
640,
|
| 585 |
+
896
|
| 586 |
+
],
|
| 587 |
+
[
|
| 588 |
+
768,
|
| 589 |
+
384,
|
| 590 |
+
640,
|
| 591 |
+
896,
|
| 592 |
+
256,
|
| 593 |
+
128,
|
| 594 |
+
384,
|
| 595 |
+
896,
|
| 596 |
+
128,
|
| 597 |
+
128,
|
| 598 |
+
896,
|
| 599 |
+
256,
|
| 600 |
+
384,
|
| 601 |
+
896,
|
| 602 |
+
512,
|
| 603 |
+
768,
|
| 604 |
+
768,
|
| 605 |
+
512,
|
| 606 |
+
512,
|
| 607 |
+
768,
|
| 608 |
+
896,
|
| 609 |
+
640,
|
| 610 |
+
768,
|
| 611 |
+
896,
|
| 612 |
+
896,
|
| 613 |
+
896,
|
| 614 |
+
640,
|
| 615 |
+
896,
|
| 616 |
+
640,
|
| 617 |
+
512,
|
| 618 |
+
896,
|
| 619 |
+
256,
|
| 620 |
+
512,
|
| 621 |
+
128,
|
| 622 |
+
512,
|
| 623 |
+
384,
|
| 624 |
+
768,
|
| 625 |
+
768,
|
| 626 |
+
1024,
|
| 627 |
+
256,
|
| 628 |
+
768,
|
| 629 |
+
256,
|
| 630 |
+
768,
|
| 631 |
+
512,
|
| 632 |
+
640,
|
| 633 |
+
1024,
|
| 634 |
+
128,
|
| 635 |
+
896,
|
| 636 |
+
896,
|
| 637 |
+
896,
|
| 638 |
+
896,
|
| 639 |
+
1024
|
| 640 |
+
],
|
| 641 |
+
[
|
| 642 |
+
512,
|
| 643 |
+
384,
|
| 644 |
+
768,
|
| 645 |
+
640,
|
| 646 |
+
640,
|
| 647 |
+
768,
|
| 648 |
+
1024,
|
| 649 |
+
896,
|
| 650 |
+
512,
|
| 651 |
+
256,
|
| 652 |
+
640,
|
| 653 |
+
768,
|
| 654 |
+
640,
|
| 655 |
+
896,
|
| 656 |
+
128,
|
| 657 |
+
256,
|
| 658 |
+
896,
|
| 659 |
+
1024,
|
| 660 |
+
256,
|
| 661 |
+
640,
|
| 662 |
+
512,
|
| 663 |
+
256,
|
| 664 |
+
128,
|
| 665 |
+
512,
|
| 666 |
+
256,
|
| 667 |
+
640,
|
| 668 |
+
768,
|
| 669 |
+
768,
|
| 670 |
+
128,
|
| 671 |
+
128,
|
| 672 |
+
768,
|
| 673 |
+
640,
|
| 674 |
+
1024,
|
| 675 |
+
1024,
|
| 676 |
+
768,
|
| 677 |
+
512,
|
| 678 |
+
896,
|
| 679 |
+
768,
|
| 680 |
+
896,
|
| 681 |
+
1024,
|
| 682 |
+
896,
|
| 683 |
+
896,
|
| 684 |
+
512,
|
| 685 |
+
640,
|
| 686 |
+
1024,
|
| 687 |
+
512,
|
| 688 |
+
1024,
|
| 689 |
+
512,
|
| 690 |
+
512,
|
| 691 |
+
512,
|
| 692 |
+
768
|
| 693 |
+
],
|
| 694 |
+
[
|
| 695 |
+
896,
|
| 696 |
+
896,
|
| 697 |
+
1024,
|
| 698 |
+
1024,
|
| 699 |
+
896,
|
| 700 |
+
128,
|
| 701 |
+
768,
|
| 702 |
+
256,
|
| 703 |
+
1024,
|
| 704 |
+
256,
|
| 705 |
+
1024,
|
| 706 |
+
640,
|
| 707 |
+
384,
|
| 708 |
+
256,
|
| 709 |
+
256,
|
| 710 |
+
512,
|
| 711 |
+
768,
|
| 712 |
+
896,
|
| 713 |
+
512,
|
| 714 |
+
768,
|
| 715 |
+
384,
|
| 716 |
+
1024,
|
| 717 |
+
896,
|
| 718 |
+
896,
|
| 719 |
+
1024,
|
| 720 |
+
896,
|
| 721 |
+
768,
|
| 722 |
+
896,
|
| 723 |
+
640,
|
| 724 |
+
1024,
|
| 725 |
+
512,
|
| 726 |
+
896,
|
| 727 |
+
512,
|
| 728 |
+
1024,
|
| 729 |
+
512,
|
| 730 |
+
512,
|
| 731 |
+
256,
|
| 732 |
+
256,
|
| 733 |
+
256,
|
| 734 |
+
512,
|
| 735 |
+
768,
|
| 736 |
+
128,
|
| 737 |
+
384,
|
| 738 |
+
512,
|
| 739 |
+
896,
|
| 740 |
+
896,
|
| 741 |
+
1024,
|
| 742 |
+
256,
|
| 743 |
+
384,
|
| 744 |
+
640
|
| 745 |
+
],
|
| 746 |
+
[
|
| 747 |
+
896,
|
| 748 |
+
640,
|
| 749 |
+
384,
|
| 750 |
+
512,
|
| 751 |
+
256,
|
| 752 |
+
640,
|
| 753 |
+
1024,
|
| 754 |
+
384,
|
| 755 |
+
1024,
|
| 756 |
+
1024,
|
| 757 |
+
768,
|
| 758 |
+
256,
|
| 759 |
+
1024,
|
| 760 |
+
768,
|
| 761 |
+
512,
|
| 762 |
+
896,
|
| 763 |
+
256,
|
| 764 |
+
1024,
|
| 765 |
+
768,
|
| 766 |
+
768,
|
| 767 |
+
768,
|
| 768 |
+
384,
|
| 769 |
+
384,
|
| 770 |
+
256,
|
| 771 |
+
1024,
|
| 772 |
+
384,
|
| 773 |
+
384,
|
| 774 |
+
384,
|
| 775 |
+
896,
|
| 776 |
+
768,
|
| 777 |
+
640,
|
| 778 |
+
768,
|
| 779 |
+
512,
|
| 780 |
+
896,
|
| 781 |
+
896,
|
| 782 |
+
896,
|
| 783 |
+
896,
|
| 784 |
+
256,
|
| 785 |
+
384,
|
| 786 |
+
128,
|
| 787 |
+
1024,
|
| 788 |
+
896,
|
| 789 |
+
256,
|
| 790 |
+
256,
|
| 791 |
+
768,
|
| 792 |
+
640,
|
| 793 |
+
896,
|
| 794 |
+
384,
|
| 795 |
+
768,
|
| 796 |
+
512,
|
| 797 |
+
640
|
| 798 |
+
],
|
| 799 |
+
[
|
| 800 |
+
896,
|
| 801 |
+
1024,
|
| 802 |
+
768,
|
| 803 |
+
1024,
|
| 804 |
+
896,
|
| 805 |
+
256,
|
| 806 |
+
768,
|
| 807 |
+
128,
|
| 808 |
+
128,
|
| 809 |
+
768,
|
| 810 |
+
512,
|
| 811 |
+
896,
|
| 812 |
+
384,
|
| 813 |
+
768,
|
| 814 |
+
1024,
|
| 815 |
+
256,
|
| 816 |
+
768,
|
| 817 |
+
768,
|
| 818 |
+
256,
|
| 819 |
+
512,
|
| 820 |
+
512,
|
| 821 |
+
640,
|
| 822 |
+
512,
|
| 823 |
+
256,
|
| 824 |
+
768,
|
| 825 |
+
896,
|
| 826 |
+
384,
|
| 827 |
+
1024,
|
| 828 |
+
640,
|
| 829 |
+
1024,
|
| 830 |
+
512,
|
| 831 |
+
512,
|
| 832 |
+
384,
|
| 833 |
+
512,
|
| 834 |
+
512,
|
| 835 |
+
1024,
|
| 836 |
+
384,
|
| 837 |
+
896,
|
| 838 |
+
768,
|
| 839 |
+
384,
|
| 840 |
+
384,
|
| 841 |
+
128,
|
| 842 |
+
384,
|
| 843 |
+
1024,
|
| 844 |
+
896,
|
| 845 |
+
640,
|
| 846 |
+
768,
|
| 847 |
+
768,
|
| 848 |
+
256,
|
| 849 |
+
640,
|
| 850 |
+
512,
|
| 851 |
+
640,
|
| 852 |
+
384
|
| 853 |
+
]
|
| 854 |
+
],
|
| 855 |
+
"glean_metadata": {
|
| 856 |
+
"base_model": "allenai/OLMoE-1B-7B-0125-Instruct",
|
| 857 |
+
"block_size": 128,
|
| 858 |
+
"criterion": "reap",
|
| 859 |
+
"dead_experts": 217,
|
| 860 |
+
"keep_fraction": 0.5,
|
| 861 |
+
"min_width": 128,
|
| 862 |
+
"params": 3697491968,
|
| 863 |
+
"scores": "outputs/scores_0125inst_dolmino-math/scores.pt"
|
| 864 |
+
},
|
| 865 |
+
"hidden_act": "silu",
|
| 866 |
+
"hidden_size": 2048,
|
| 867 |
+
"initializer_range": 0.02,
|
| 868 |
+
"intermediate_size": 1024,
|
| 869 |
+
"max_position_embeddings": 4096,
|
| 870 |
+
"model_type": "pruned_olmoe",
|
| 871 |
+
"norm_topk_prob": false,
|
| 872 |
+
"num_attention_heads": 16,
|
| 873 |
+
"num_experts": 64,
|
| 874 |
+
"num_experts_per_tok": 8,
|
| 875 |
+
"num_hidden_layers": 16,
|
| 876 |
+
"num_key_value_heads": 16,
|
| 877 |
+
"output_router_logits": false,
|
| 878 |
+
"pad_token_id": 1,
|
| 879 |
+
"rms_norm_eps": 1e-05,
|
| 880 |
+
"rope_scaling": null,
|
| 881 |
+
"rope_theta": 10000.0,
|
| 882 |
+
"router_aux_loss_coef": 0.01,
|
| 883 |
+
"tie_word_embeddings": false,
|
| 884 |
+
"transformers_version": "4.57.6",
|
| 885 |
+
"use_cache": false,
|
| 886 |
+
"vocab_size": 50304
|
| 887 |
+
}
|
healed/opd_warm_fixed_keep50/step0120/configuration_pruned_olmoe.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Configuration for GLEAN-pruned OLMoE: variable-width, variable-count experts.
|
| 2 |
+
"""
|
| 3 |
+
|
| 4 |
+
from transformers.models.olmoe.configuration_olmoe import OlmoeConfig
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
class PrunedOlmoeConfig(OlmoeConfig):
|
| 8 |
+
"""OlmoeConfig plus a per-(layer, expert) width table.
|
| 9 |
+
|
| 10 |
+
``expert_widths[l]`` lists the SwiGLU intermediate width of each surviving
|
| 11 |
+
expert in decoder layer ``l``, in expert order. Lists are ragged: layers
|
| 12 |
+
may keep different numbers of experts (deleted experts simply don't
|
| 13 |
+
appear — the router in layer ``l`` has ``len(expert_widths[l])`` rows),
|
| 14 |
+
and each width may differ (multiples of the GEMM block size, 128, for
|
| 15 |
+
variable-MegaBlocks execution). ``None`` means an unpruned model
|
| 16 |
+
(uniform ``num_experts`` × ``intermediate_size``).
|
| 17 |
+
|
| 18 |
+
The inherited ``num_experts`` / ``intermediate_size`` keep their ORIGINAL
|
| 19 |
+
(pre-pruning) values for provenance; the width table is authoritative for
|
| 20 |
+
the built architecture.
|
| 21 |
+
"""
|
| 22 |
+
|
| 23 |
+
model_type = "pruned_olmoe"
|
| 24 |
+
|
| 25 |
+
def __init__(self, expert_widths: list[list[int]] | None = None, **kwargs):
|
| 26 |
+
super().__init__(**kwargs)
|
| 27 |
+
self.expert_widths = expert_widths
|
healed/opd_warm_fixed_keep50/step0120/generation_config.json
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_from_model_config": true,
|
| 3 |
+
"eos_token_id": 50279,
|
| 4 |
+
"pad_token_id": 1,
|
| 5 |
+
"transformers_version": "4.57.6"
|
| 6 |
+
}
|
healed/opd_warm_fixed_keep50/step0120/model.safetensors.index.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
healed/opd_warm_fixed_keep50/step0120/modeling_pruned_olmoe.py
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""GLEAN-pruned OLMoE: HF-loadable model with ragged (variable-width) experts.
|
| 2 |
+
|
| 3 |
+
Pattern follows hbfreed/variable-flex-olmo's PrunedFlexOlmoForCausalLM
|
| 4 |
+
(docs/recon/prior-work-hbfreed.md), generalized from one scalar width to a
|
| 5 |
+
per-(layer, expert) width table: ``super().__init__`` builds the uniform
|
| 6 |
+
architecture from the config, then every MoE block is rebuilt to its pruned
|
| 7 |
+
shape — surviving experts only, each at its own width, router sliced to
|
| 8 |
+
match — so the state dict aligns exactly with what
|
| 9 |
+
``glean.prune.prune_channels_global`` leaves behind.
|
| 10 |
+
|
| 11 |
+
Caveat: ``output_router_logits=True`` (the load-balancing aux loss) assumes a
|
| 12 |
+
uniform ``config.num_experts`` and is unsupported on ragged models.
|
| 13 |
+
"""
|
| 14 |
+
|
| 15 |
+
import torch.nn as nn
|
| 16 |
+
from transformers.activations import ACT2FN
|
| 17 |
+
from transformers.models.olmoe.modeling_olmoe import OlmoeForCausalLM
|
| 18 |
+
|
| 19 |
+
from .configuration_pruned_olmoe import PrunedOlmoeConfig
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
class RaggedOlmoeMLP(nn.Module):
|
| 23 |
+
"""OlmoeMLP with an explicit intermediate width (SwiGLU, no biases)."""
|
| 24 |
+
|
| 25 |
+
def __init__(self, hidden_size: int, intermediate_size: int, hidden_act: str):
|
| 26 |
+
super().__init__()
|
| 27 |
+
self.hidden_size = hidden_size
|
| 28 |
+
self.intermediate_size = intermediate_size
|
| 29 |
+
self.gate_proj = nn.Linear(hidden_size, intermediate_size, bias=False)
|
| 30 |
+
self.up_proj = nn.Linear(hidden_size, intermediate_size, bias=False)
|
| 31 |
+
self.down_proj = nn.Linear(intermediate_size, hidden_size, bias=False)
|
| 32 |
+
self.act_fn = ACT2FN[hidden_act]
|
| 33 |
+
|
| 34 |
+
def forward(self, x):
|
| 35 |
+
return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
class PrunedOlmoeForCausalLM(OlmoeForCausalLM):
|
| 39 |
+
"""OLMoE with per-layer surviving-expert lists at per-expert widths."""
|
| 40 |
+
|
| 41 |
+
config_class = PrunedOlmoeConfig
|
| 42 |
+
|
| 43 |
+
def __init__(self, config: PrunedOlmoeConfig):
|
| 44 |
+
super().__init__(config)
|
| 45 |
+
widths_table = getattr(config, "expert_widths", None)
|
| 46 |
+
if widths_table is None:
|
| 47 |
+
return # unpruned: plain OLMoE
|
| 48 |
+
if len(widths_table) != len(self.model.layers):
|
| 49 |
+
raise ValueError(
|
| 50 |
+
f"expert_widths has {len(widths_table)} rows but the model has "
|
| 51 |
+
f"{len(self.model.layers)} decoder layers"
|
| 52 |
+
)
|
| 53 |
+
for layer, widths in zip(self.model.layers, widths_table):
|
| 54 |
+
if any(w <= 0 for w in widths):
|
| 55 |
+
raise ValueError("expert_widths must list surviving experts only (>0)")
|
| 56 |
+
block = layer.mlp
|
| 57 |
+
if len(widths) < block.top_k:
|
| 58 |
+
raise ValueError(
|
| 59 |
+
f"a layer keeps {len(widths)} experts < top_k={block.top_k}"
|
| 60 |
+
)
|
| 61 |
+
block.num_experts = len(widths)
|
| 62 |
+
block.gate = nn.Linear(config.hidden_size, len(widths), bias=False)
|
| 63 |
+
block.experts = nn.ModuleList(
|
| 64 |
+
RaggedOlmoeMLP(config.hidden_size, w, config.hidden_act)
|
| 65 |
+
for w in widths
|
| 66 |
+
)
|
healed/opd_warm_fixed_keep50/step0120/special_tokens_map.json
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"bos_token": {
|
| 3 |
+
"content": "|||IP_ADDRESS|||",
|
| 4 |
+
"lstrip": false,
|
| 5 |
+
"normalized": true,
|
| 6 |
+
"rstrip": false,
|
| 7 |
+
"single_word": false
|
| 8 |
+
},
|
| 9 |
+
"eos_token": {
|
| 10 |
+
"content": "|||IP_ADDRESS|||",
|
| 11 |
+
"lstrip": false,
|
| 12 |
+
"normalized": true,
|
| 13 |
+
"rstrip": false,
|
| 14 |
+
"single_word": false
|
| 15 |
+
},
|
| 16 |
+
"pad_token": {
|
| 17 |
+
"content": "<pad>",
|
| 18 |
+
"lstrip": false,
|
| 19 |
+
"normalized": false,
|
| 20 |
+
"rstrip": false,
|
| 21 |
+
"single_word": false
|
| 22 |
+
}
|
| 23 |
+
}
|
healed/opd_warm_fixed_keep50/step0120/tokenizer.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
healed/opd_warm_fixed_keep50/step0120/tokenizer_config.json
ADDED
|
@@ -0,0 +1,247 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"add_bos_token": false,
|
| 3 |
+
"add_eos_token": false,
|
| 4 |
+
"add_prefix_space": false,
|
| 5 |
+
"added_tokens_decoder": {
|
| 6 |
+
"0": {
|
| 7 |
+
"content": "<|endoftext|>",
|
| 8 |
+
"lstrip": false,
|
| 9 |
+
"normalized": false,
|
| 10 |
+
"rstrip": false,
|
| 11 |
+
"single_word": false,
|
| 12 |
+
"special": true
|
| 13 |
+
},
|
| 14 |
+
"1": {
|
| 15 |
+
"content": "<|padding|>",
|
| 16 |
+
"lstrip": false,
|
| 17 |
+
"normalized": false,
|
| 18 |
+
"rstrip": false,
|
| 19 |
+
"single_word": false,
|
| 20 |
+
"special": true
|
| 21 |
+
},
|
| 22 |
+
"50254": {
|
| 23 |
+
"content": " ",
|
| 24 |
+
"lstrip": false,
|
| 25 |
+
"normalized": true,
|
| 26 |
+
"rstrip": false,
|
| 27 |
+
"single_word": false,
|
| 28 |
+
"special": false
|
| 29 |
+
},
|
| 30 |
+
"50255": {
|
| 31 |
+
"content": " ",
|
| 32 |
+
"lstrip": false,
|
| 33 |
+
"normalized": true,
|
| 34 |
+
"rstrip": false,
|
| 35 |
+
"single_word": false,
|
| 36 |
+
"special": false
|
| 37 |
+
},
|
| 38 |
+
"50256": {
|
| 39 |
+
"content": " ",
|
| 40 |
+
"lstrip": false,
|
| 41 |
+
"normalized": true,
|
| 42 |
+
"rstrip": false,
|
| 43 |
+
"single_word": false,
|
| 44 |
+
"special": false
|
| 45 |
+
},
|
| 46 |
+
"50257": {
|
| 47 |
+
"content": " ",
|
| 48 |
+
"lstrip": false,
|
| 49 |
+
"normalized": true,
|
| 50 |
+
"rstrip": false,
|
| 51 |
+
"single_word": false,
|
| 52 |
+
"special": false
|
| 53 |
+
},
|
| 54 |
+
"50258": {
|
| 55 |
+
"content": " ",
|
| 56 |
+
"lstrip": false,
|
| 57 |
+
"normalized": true,
|
| 58 |
+
"rstrip": false,
|
| 59 |
+
"single_word": false,
|
| 60 |
+
"special": false
|
| 61 |
+
},
|
| 62 |
+
"50259": {
|
| 63 |
+
"content": " ",
|
| 64 |
+
"lstrip": false,
|
| 65 |
+
"normalized": true,
|
| 66 |
+
"rstrip": false,
|
| 67 |
+
"single_word": false,
|
| 68 |
+
"special": false
|
| 69 |
+
},
|
| 70 |
+
"50260": {
|
| 71 |
+
"content": " ",
|
| 72 |
+
"lstrip": false,
|
| 73 |
+
"normalized": true,
|
| 74 |
+
"rstrip": false,
|
| 75 |
+
"single_word": false,
|
| 76 |
+
"special": false
|
| 77 |
+
},
|
| 78 |
+
"50261": {
|
| 79 |
+
"content": " ",
|
| 80 |
+
"lstrip": false,
|
| 81 |
+
"normalized": true,
|
| 82 |
+
"rstrip": false,
|
| 83 |
+
"single_word": false,
|
| 84 |
+
"special": false
|
| 85 |
+
},
|
| 86 |
+
"50262": {
|
| 87 |
+
"content": " ",
|
| 88 |
+
"lstrip": false,
|
| 89 |
+
"normalized": true,
|
| 90 |
+
"rstrip": false,
|
| 91 |
+
"single_word": false,
|
| 92 |
+
"special": false
|
| 93 |
+
},
|
| 94 |
+
"50263": {
|
| 95 |
+
"content": " ",
|
| 96 |
+
"lstrip": false,
|
| 97 |
+
"normalized": true,
|
| 98 |
+
"rstrip": false,
|
| 99 |
+
"single_word": false,
|
| 100 |
+
"special": false
|
| 101 |
+
},
|
| 102 |
+
"50264": {
|
| 103 |
+
"content": " ",
|
| 104 |
+
"lstrip": false,
|
| 105 |
+
"normalized": true,
|
| 106 |
+
"rstrip": false,
|
| 107 |
+
"single_word": false,
|
| 108 |
+
"special": false
|
| 109 |
+
},
|
| 110 |
+
"50265": {
|
| 111 |
+
"content": " ",
|
| 112 |
+
"lstrip": false,
|
| 113 |
+
"normalized": true,
|
| 114 |
+
"rstrip": false,
|
| 115 |
+
"single_word": false,
|
| 116 |
+
"special": false
|
| 117 |
+
},
|
| 118 |
+
"50266": {
|
| 119 |
+
"content": " ",
|
| 120 |
+
"lstrip": false,
|
| 121 |
+
"normalized": true,
|
| 122 |
+
"rstrip": false,
|
| 123 |
+
"single_word": false,
|
| 124 |
+
"special": false
|
| 125 |
+
},
|
| 126 |
+
"50267": {
|
| 127 |
+
"content": " ",
|
| 128 |
+
"lstrip": false,
|
| 129 |
+
"normalized": true,
|
| 130 |
+
"rstrip": false,
|
| 131 |
+
"single_word": false,
|
| 132 |
+
"special": false
|
| 133 |
+
},
|
| 134 |
+
"50268": {
|
| 135 |
+
"content": " ",
|
| 136 |
+
"lstrip": false,
|
| 137 |
+
"normalized": true,
|
| 138 |
+
"rstrip": false,
|
| 139 |
+
"single_word": false,
|
| 140 |
+
"special": false
|
| 141 |
+
},
|
| 142 |
+
"50269": {
|
| 143 |
+
"content": " ",
|
| 144 |
+
"lstrip": false,
|
| 145 |
+
"normalized": true,
|
| 146 |
+
"rstrip": false,
|
| 147 |
+
"single_word": false,
|
| 148 |
+
"special": false
|
| 149 |
+
},
|
| 150 |
+
"50270": {
|
| 151 |
+
"content": " ",
|
| 152 |
+
"lstrip": false,
|
| 153 |
+
"normalized": true,
|
| 154 |
+
"rstrip": false,
|
| 155 |
+
"single_word": false,
|
| 156 |
+
"special": false
|
| 157 |
+
},
|
| 158 |
+
"50271": {
|
| 159 |
+
"content": " ",
|
| 160 |
+
"lstrip": false,
|
| 161 |
+
"normalized": true,
|
| 162 |
+
"rstrip": false,
|
| 163 |
+
"single_word": false,
|
| 164 |
+
"special": false
|
| 165 |
+
},
|
| 166 |
+
"50272": {
|
| 167 |
+
"content": " ",
|
| 168 |
+
"lstrip": false,
|
| 169 |
+
"normalized": true,
|
| 170 |
+
"rstrip": false,
|
| 171 |
+
"single_word": false,
|
| 172 |
+
"special": false
|
| 173 |
+
},
|
| 174 |
+
"50273": {
|
| 175 |
+
"content": " ",
|
| 176 |
+
"lstrip": false,
|
| 177 |
+
"normalized": true,
|
| 178 |
+
"rstrip": false,
|
| 179 |
+
"single_word": false,
|
| 180 |
+
"special": false
|
| 181 |
+
},
|
| 182 |
+
"50274": {
|
| 183 |
+
"content": " ",
|
| 184 |
+
"lstrip": false,
|
| 185 |
+
"normalized": true,
|
| 186 |
+
"rstrip": false,
|
| 187 |
+
"single_word": false,
|
| 188 |
+
"special": false
|
| 189 |
+
},
|
| 190 |
+
"50275": {
|
| 191 |
+
"content": " ",
|
| 192 |
+
"lstrip": false,
|
| 193 |
+
"normalized": true,
|
| 194 |
+
"rstrip": false,
|
| 195 |
+
"single_word": false,
|
| 196 |
+
"special": false
|
| 197 |
+
},
|
| 198 |
+
"50276": {
|
| 199 |
+
"content": " ",
|
| 200 |
+
"lstrip": false,
|
| 201 |
+
"normalized": true,
|
| 202 |
+
"rstrip": false,
|
| 203 |
+
"single_word": false,
|
| 204 |
+
"special": false
|
| 205 |
+
},
|
| 206 |
+
"50277": {
|
| 207 |
+
"content": "|||EMAIL_ADDRESS|||",
|
| 208 |
+
"lstrip": false,
|
| 209 |
+
"normalized": true,
|
| 210 |
+
"rstrip": false,
|
| 211 |
+
"single_word": false,
|
| 212 |
+
"special": false
|
| 213 |
+
},
|
| 214 |
+
"50278": {
|
| 215 |
+
"content": "|||PHONE_NUMBER|||",
|
| 216 |
+
"lstrip": false,
|
| 217 |
+
"normalized": true,
|
| 218 |
+
"rstrip": false,
|
| 219 |
+
"single_word": false,
|
| 220 |
+
"special": false
|
| 221 |
+
},
|
| 222 |
+
"50279": {
|
| 223 |
+
"content": "|||IP_ADDRESS|||",
|
| 224 |
+
"lstrip": false,
|
| 225 |
+
"normalized": true,
|
| 226 |
+
"rstrip": false,
|
| 227 |
+
"single_word": false,
|
| 228 |
+
"special": true
|
| 229 |
+
},
|
| 230 |
+
"50280": {
|
| 231 |
+
"content": "<pad>",
|
| 232 |
+
"lstrip": false,
|
| 233 |
+
"normalized": false,
|
| 234 |
+
"rstrip": false,
|
| 235 |
+
"single_word": false,
|
| 236 |
+
"special": true
|
| 237 |
+
}
|
| 238 |
+
},
|
| 239 |
+
"bos_token": "|||IP_ADDRESS|||",
|
| 240 |
+
"clean_up_tokenization_spaces": false,
|
| 241 |
+
"eos_token": "|||IP_ADDRESS|||",
|
| 242 |
+
"extra_special_tokens": {},
|
| 243 |
+
"model_max_length": 1000000000000000019884624838656,
|
| 244 |
+
"pad_token": "<pad>",
|
| 245 |
+
"tokenizer_class": "GPTNeoXTokenizer",
|
| 246 |
+
"unk_token": null
|
| 247 |
+
}
|
healed/opd_warm_fixed_keep50/step0200/chat_template.jinja
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{{ bos_token }}{% for message in messages %}{% if message['role'] == 'system' %}{{ '<|system|>
|
| 2 |
+
' + message['content'] + '
|
| 3 |
+
' }}{% elif message['role'] == 'user' %}{{ '<|user|>
|
| 4 |
+
' + message['content'] + '
|
| 5 |
+
' }}{% elif message['role'] == 'assistant' %}{% if not loop.last %}{{ '<|assistant|>
|
| 6 |
+
' + message['content'] + eos_token + '
|
| 7 |
+
' }}{% else %}{{ '<|assistant|>
|
| 8 |
+
' + message['content'] + eos_token }}{% endif %}{% endif %}{% if loop.last and add_generation_prompt %}{{ '<|assistant|>
|
| 9 |
+
' }}{% endif %}{% endfor %}
|
healed/opd_warm_fixed_keep50/step0200/config.json
ADDED
|
@@ -0,0 +1,887 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"architectures": [
|
| 3 |
+
"PrunedOlmoeForCausalLM"
|
| 4 |
+
],
|
| 5 |
+
"attention_bias": false,
|
| 6 |
+
"attention_dropout": 0.0,
|
| 7 |
+
"auto_map": {
|
| 8 |
+
"AutoConfig": "configuration_pruned_olmoe.PrunedOlmoeConfig",
|
| 9 |
+
"AutoModelForCausalLM": "modeling_pruned_olmoe.PrunedOlmoeForCausalLM"
|
| 10 |
+
},
|
| 11 |
+
"clip_qkv": null,
|
| 12 |
+
"dtype": "bfloat16",
|
| 13 |
+
"eos_token_id": 50279,
|
| 14 |
+
"expert_widths": [
|
| 15 |
+
[
|
| 16 |
+
1024,
|
| 17 |
+
384,
|
| 18 |
+
256,
|
| 19 |
+
256,
|
| 20 |
+
768,
|
| 21 |
+
1024,
|
| 22 |
+
128,
|
| 23 |
+
768,
|
| 24 |
+
384,
|
| 25 |
+
128,
|
| 26 |
+
1024,
|
| 27 |
+
384,
|
| 28 |
+
384,
|
| 29 |
+
128,
|
| 30 |
+
640,
|
| 31 |
+
896,
|
| 32 |
+
896,
|
| 33 |
+
256,
|
| 34 |
+
768,
|
| 35 |
+
128,
|
| 36 |
+
768,
|
| 37 |
+
768,
|
| 38 |
+
768,
|
| 39 |
+
768,
|
| 40 |
+
640,
|
| 41 |
+
896,
|
| 42 |
+
256,
|
| 43 |
+
128,
|
| 44 |
+
384,
|
| 45 |
+
896,
|
| 46 |
+
256,
|
| 47 |
+
768,
|
| 48 |
+
512,
|
| 49 |
+
640,
|
| 50 |
+
768,
|
| 51 |
+
896,
|
| 52 |
+
1024,
|
| 53 |
+
384,
|
| 54 |
+
384,
|
| 55 |
+
512,
|
| 56 |
+
384,
|
| 57 |
+
256,
|
| 58 |
+
768,
|
| 59 |
+
640,
|
| 60 |
+
896,
|
| 61 |
+
896,
|
| 62 |
+
512,
|
| 63 |
+
768,
|
| 64 |
+
512,
|
| 65 |
+
384,
|
| 66 |
+
1024,
|
| 67 |
+
896,
|
| 68 |
+
896,
|
| 69 |
+
768,
|
| 70 |
+
128
|
| 71 |
+
],
|
| 72 |
+
[
|
| 73 |
+
768,
|
| 74 |
+
896,
|
| 75 |
+
896,
|
| 76 |
+
256,
|
| 77 |
+
896,
|
| 78 |
+
384,
|
| 79 |
+
768,
|
| 80 |
+
768,
|
| 81 |
+
1024,
|
| 82 |
+
384,
|
| 83 |
+
512,
|
| 84 |
+
1024,
|
| 85 |
+
768,
|
| 86 |
+
768,
|
| 87 |
+
1024,
|
| 88 |
+
640,
|
| 89 |
+
128,
|
| 90 |
+
768,
|
| 91 |
+
384,
|
| 92 |
+
768,
|
| 93 |
+
128,
|
| 94 |
+
256,
|
| 95 |
+
512,
|
| 96 |
+
384,
|
| 97 |
+
896,
|
| 98 |
+
256,
|
| 99 |
+
1024,
|
| 100 |
+
1024,
|
| 101 |
+
896,
|
| 102 |
+
1024,
|
| 103 |
+
896,
|
| 104 |
+
768,
|
| 105 |
+
256,
|
| 106 |
+
768,
|
| 107 |
+
128,
|
| 108 |
+
1024,
|
| 109 |
+
768,
|
| 110 |
+
768,
|
| 111 |
+
640,
|
| 112 |
+
384,
|
| 113 |
+
1024,
|
| 114 |
+
384,
|
| 115 |
+
640,
|
| 116 |
+
896,
|
| 117 |
+
1024,
|
| 118 |
+
512,
|
| 119 |
+
512,
|
| 120 |
+
512,
|
| 121 |
+
640
|
| 122 |
+
],
|
| 123 |
+
[
|
| 124 |
+
768,
|
| 125 |
+
1024,
|
| 126 |
+
512,
|
| 127 |
+
384,
|
| 128 |
+
384,
|
| 129 |
+
768,
|
| 130 |
+
256,
|
| 131 |
+
384,
|
| 132 |
+
1024,
|
| 133 |
+
896,
|
| 134 |
+
640,
|
| 135 |
+
256,
|
| 136 |
+
1024,
|
| 137 |
+
1024,
|
| 138 |
+
1024,
|
| 139 |
+
640,
|
| 140 |
+
1024,
|
| 141 |
+
896,
|
| 142 |
+
512,
|
| 143 |
+
768,
|
| 144 |
+
128,
|
| 145 |
+
768,
|
| 146 |
+
1024,
|
| 147 |
+
768,
|
| 148 |
+
256,
|
| 149 |
+
1024,
|
| 150 |
+
896,
|
| 151 |
+
1024,
|
| 152 |
+
896,
|
| 153 |
+
512,
|
| 154 |
+
640,
|
| 155 |
+
384,
|
| 156 |
+
896,
|
| 157 |
+
384,
|
| 158 |
+
512,
|
| 159 |
+
640,
|
| 160 |
+
256,
|
| 161 |
+
768,
|
| 162 |
+
256,
|
| 163 |
+
768,
|
| 164 |
+
768,
|
| 165 |
+
640,
|
| 166 |
+
1024,
|
| 167 |
+
640,
|
| 168 |
+
896,
|
| 169 |
+
768,
|
| 170 |
+
768,
|
| 171 |
+
256
|
| 172 |
+
],
|
| 173 |
+
[
|
| 174 |
+
640,
|
| 175 |
+
640,
|
| 176 |
+
896,
|
| 177 |
+
512,
|
| 178 |
+
640,
|
| 179 |
+
896,
|
| 180 |
+
896,
|
| 181 |
+
384,
|
| 182 |
+
896,
|
| 183 |
+
896,
|
| 184 |
+
384,
|
| 185 |
+
896,
|
| 186 |
+
640,
|
| 187 |
+
256,
|
| 188 |
+
640,
|
| 189 |
+
1024,
|
| 190 |
+
1024,
|
| 191 |
+
768,
|
| 192 |
+
1024,
|
| 193 |
+
896,
|
| 194 |
+
1024,
|
| 195 |
+
768,
|
| 196 |
+
896,
|
| 197 |
+
256,
|
| 198 |
+
512,
|
| 199 |
+
768,
|
| 200 |
+
1024,
|
| 201 |
+
256,
|
| 202 |
+
768,
|
| 203 |
+
512,
|
| 204 |
+
256,
|
| 205 |
+
640,
|
| 206 |
+
1024,
|
| 207 |
+
1024,
|
| 208 |
+
512,
|
| 209 |
+
1024,
|
| 210 |
+
768,
|
| 211 |
+
256,
|
| 212 |
+
1024,
|
| 213 |
+
384,
|
| 214 |
+
896,
|
| 215 |
+
1024,
|
| 216 |
+
896,
|
| 217 |
+
1024,
|
| 218 |
+
384
|
| 219 |
+
],
|
| 220 |
+
[
|
| 221 |
+
256,
|
| 222 |
+
640,
|
| 223 |
+
640,
|
| 224 |
+
896,
|
| 225 |
+
768,
|
| 226 |
+
896,
|
| 227 |
+
768,
|
| 228 |
+
768,
|
| 229 |
+
896,
|
| 230 |
+
896,
|
| 231 |
+
1024,
|
| 232 |
+
256,
|
| 233 |
+
512,
|
| 234 |
+
1024,
|
| 235 |
+
640,
|
| 236 |
+
896,
|
| 237 |
+
512,
|
| 238 |
+
512,
|
| 239 |
+
384,
|
| 240 |
+
384,
|
| 241 |
+
256,
|
| 242 |
+
384,
|
| 243 |
+
384,
|
| 244 |
+
896,
|
| 245 |
+
896,
|
| 246 |
+
768,
|
| 247 |
+
640,
|
| 248 |
+
896,
|
| 249 |
+
768,
|
| 250 |
+
1024,
|
| 251 |
+
512,
|
| 252 |
+
640,
|
| 253 |
+
512,
|
| 254 |
+
640,
|
| 255 |
+
896,
|
| 256 |
+
512,
|
| 257 |
+
512,
|
| 258 |
+
384,
|
| 259 |
+
640,
|
| 260 |
+
896,
|
| 261 |
+
896,
|
| 262 |
+
896,
|
| 263 |
+
1024,
|
| 264 |
+
640,
|
| 265 |
+
1024,
|
| 266 |
+
640,
|
| 267 |
+
1024
|
| 268 |
+
],
|
| 269 |
+
[
|
| 270 |
+
1024,
|
| 271 |
+
512,
|
| 272 |
+
1024,
|
| 273 |
+
1024,
|
| 274 |
+
640,
|
| 275 |
+
896,
|
| 276 |
+
640,
|
| 277 |
+
1024,
|
| 278 |
+
896,
|
| 279 |
+
384,
|
| 280 |
+
1024,
|
| 281 |
+
128,
|
| 282 |
+
896,
|
| 283 |
+
768,
|
| 284 |
+
1024,
|
| 285 |
+
768,
|
| 286 |
+
640,
|
| 287 |
+
896,
|
| 288 |
+
768,
|
| 289 |
+
640,
|
| 290 |
+
512,
|
| 291 |
+
896,
|
| 292 |
+
512,
|
| 293 |
+
640,
|
| 294 |
+
256,
|
| 295 |
+
768,
|
| 296 |
+
640,
|
| 297 |
+
768,
|
| 298 |
+
384,
|
| 299 |
+
896,
|
| 300 |
+
512,
|
| 301 |
+
512,
|
| 302 |
+
256,
|
| 303 |
+
512,
|
| 304 |
+
896,
|
| 305 |
+
256,
|
| 306 |
+
384,
|
| 307 |
+
640,
|
| 308 |
+
512,
|
| 309 |
+
640,
|
| 310 |
+
896,
|
| 311 |
+
512,
|
| 312 |
+
1024,
|
| 313 |
+
256,
|
| 314 |
+
768,
|
| 315 |
+
1024,
|
| 316 |
+
768,
|
| 317 |
+
256,
|
| 318 |
+
256
|
| 319 |
+
],
|
| 320 |
+
[
|
| 321 |
+
640,
|
| 322 |
+
896,
|
| 323 |
+
1024,
|
| 324 |
+
896,
|
| 325 |
+
1024,
|
| 326 |
+
1024,
|
| 327 |
+
1024,
|
| 328 |
+
512,
|
| 329 |
+
256,
|
| 330 |
+
256,
|
| 331 |
+
1024,
|
| 332 |
+
768,
|
| 333 |
+
512,
|
| 334 |
+
768,
|
| 335 |
+
1024,
|
| 336 |
+
1024,
|
| 337 |
+
1024,
|
| 338 |
+
384,
|
| 339 |
+
512,
|
| 340 |
+
1024,
|
| 341 |
+
512,
|
| 342 |
+
1024,
|
| 343 |
+
128,
|
| 344 |
+
640,
|
| 345 |
+
640,
|
| 346 |
+
896,
|
| 347 |
+
768,
|
| 348 |
+
128,
|
| 349 |
+
256,
|
| 350 |
+
256,
|
| 351 |
+
256,
|
| 352 |
+
256,
|
| 353 |
+
896,
|
| 354 |
+
1024,
|
| 355 |
+
1024,
|
| 356 |
+
384,
|
| 357 |
+
896,
|
| 358 |
+
256,
|
| 359 |
+
896,
|
| 360 |
+
640,
|
| 361 |
+
1024,
|
| 362 |
+
384,
|
| 363 |
+
640,
|
| 364 |
+
256,
|
| 365 |
+
1024,
|
| 366 |
+
1024,
|
| 367 |
+
1024
|
| 368 |
+
],
|
| 369 |
+
[
|
| 370 |
+
1024,
|
| 371 |
+
384,
|
| 372 |
+
1024,
|
| 373 |
+
1024,
|
| 374 |
+
256,
|
| 375 |
+
128,
|
| 376 |
+
256,
|
| 377 |
+
384,
|
| 378 |
+
256,
|
| 379 |
+
384,
|
| 380 |
+
896,
|
| 381 |
+
768,
|
| 382 |
+
896,
|
| 383 |
+
896,
|
| 384 |
+
512,
|
| 385 |
+
896,
|
| 386 |
+
640,
|
| 387 |
+
384,
|
| 388 |
+
384,
|
| 389 |
+
896,
|
| 390 |
+
768,
|
| 391 |
+
384,
|
| 392 |
+
896,
|
| 393 |
+
768,
|
| 394 |
+
768,
|
| 395 |
+
512,
|
| 396 |
+
896,
|
| 397 |
+
768,
|
| 398 |
+
768,
|
| 399 |
+
896,
|
| 400 |
+
128,
|
| 401 |
+
896,
|
| 402 |
+
512,
|
| 403 |
+
256,
|
| 404 |
+
768,
|
| 405 |
+
128,
|
| 406 |
+
384,
|
| 407 |
+
256,
|
| 408 |
+
896,
|
| 409 |
+
896,
|
| 410 |
+
384,
|
| 411 |
+
768,
|
| 412 |
+
512,
|
| 413 |
+
640,
|
| 414 |
+
256,
|
| 415 |
+
768,
|
| 416 |
+
640,
|
| 417 |
+
896,
|
| 418 |
+
384,
|
| 419 |
+
512,
|
| 420 |
+
1024,
|
| 421 |
+
768,
|
| 422 |
+
384
|
| 423 |
+
],
|
| 424 |
+
[
|
| 425 |
+
512,
|
| 426 |
+
768,
|
| 427 |
+
512,
|
| 428 |
+
256,
|
| 429 |
+
128,
|
| 430 |
+
640,
|
| 431 |
+
384,
|
| 432 |
+
640,
|
| 433 |
+
768,
|
| 434 |
+
896,
|
| 435 |
+
640,
|
| 436 |
+
768,
|
| 437 |
+
256,
|
| 438 |
+
384,
|
| 439 |
+
1024,
|
| 440 |
+
896,
|
| 441 |
+
256,
|
| 442 |
+
896,
|
| 443 |
+
512,
|
| 444 |
+
256,
|
| 445 |
+
896,
|
| 446 |
+
768,
|
| 447 |
+
256,
|
| 448 |
+
896,
|
| 449 |
+
896,
|
| 450 |
+
384,
|
| 451 |
+
896,
|
| 452 |
+
640,
|
| 453 |
+
768,
|
| 454 |
+
512,
|
| 455 |
+
768,
|
| 456 |
+
768,
|
| 457 |
+
1024,
|
| 458 |
+
768,
|
| 459 |
+
640,
|
| 460 |
+
768,
|
| 461 |
+
384,
|
| 462 |
+
256,
|
| 463 |
+
512,
|
| 464 |
+
896,
|
| 465 |
+
128,
|
| 466 |
+
384,
|
| 467 |
+
256,
|
| 468 |
+
768,
|
| 469 |
+
384,
|
| 470 |
+
256,
|
| 471 |
+
1024,
|
| 472 |
+
1024,
|
| 473 |
+
896,
|
| 474 |
+
256,
|
| 475 |
+
1024,
|
| 476 |
+
256,
|
| 477 |
+
128,
|
| 478 |
+
896
|
| 479 |
+
],
|
| 480 |
+
[
|
| 481 |
+
640,
|
| 482 |
+
640,
|
| 483 |
+
896,
|
| 484 |
+
256,
|
| 485 |
+
1024,
|
| 486 |
+
512,
|
| 487 |
+
1024,
|
| 488 |
+
768,
|
| 489 |
+
384,
|
| 490 |
+
512,
|
| 491 |
+
256,
|
| 492 |
+
768,
|
| 493 |
+
896,
|
| 494 |
+
768,
|
| 495 |
+
512,
|
| 496 |
+
768,
|
| 497 |
+
768,
|
| 498 |
+
640,
|
| 499 |
+
384,
|
| 500 |
+
768,
|
| 501 |
+
512,
|
| 502 |
+
768,
|
| 503 |
+
768,
|
| 504 |
+
512,
|
| 505 |
+
768,
|
| 506 |
+
128,
|
| 507 |
+
896,
|
| 508 |
+
512,
|
| 509 |
+
768,
|
| 510 |
+
1024,
|
| 511 |
+
128,
|
| 512 |
+
384,
|
| 513 |
+
768,
|
| 514 |
+
768,
|
| 515 |
+
768,
|
| 516 |
+
384,
|
| 517 |
+
512,
|
| 518 |
+
640,
|
| 519 |
+
768,
|
| 520 |
+
512,
|
| 521 |
+
768,
|
| 522 |
+
1024,
|
| 523 |
+
640,
|
| 524 |
+
896,
|
| 525 |
+
256,
|
| 526 |
+
1024,
|
| 527 |
+
384,
|
| 528 |
+
768,
|
| 529 |
+
768,
|
| 530 |
+
768
|
| 531 |
+
],
|
| 532 |
+
[
|
| 533 |
+
896,
|
| 534 |
+
512,
|
| 535 |
+
896,
|
| 536 |
+
768,
|
| 537 |
+
384,
|
| 538 |
+
384,
|
| 539 |
+
768,
|
| 540 |
+
512,
|
| 541 |
+
768,
|
| 542 |
+
512,
|
| 543 |
+
1024,
|
| 544 |
+
640,
|
| 545 |
+
896,
|
| 546 |
+
896,
|
| 547 |
+
256,
|
| 548 |
+
640,
|
| 549 |
+
1024,
|
| 550 |
+
256,
|
| 551 |
+
896,
|
| 552 |
+
128,
|
| 553 |
+
128,
|
| 554 |
+
128,
|
| 555 |
+
768,
|
| 556 |
+
896,
|
| 557 |
+
384,
|
| 558 |
+
896,
|
| 559 |
+
512,
|
| 560 |
+
896,
|
| 561 |
+
384,
|
| 562 |
+
256,
|
| 563 |
+
640,
|
| 564 |
+
640,
|
| 565 |
+
896,
|
| 566 |
+
768,
|
| 567 |
+
640,
|
| 568 |
+
256,
|
| 569 |
+
896,
|
| 570 |
+
896,
|
| 571 |
+
512,
|
| 572 |
+
128,
|
| 573 |
+
896,
|
| 574 |
+
256,
|
| 575 |
+
256,
|
| 576 |
+
640,
|
| 577 |
+
896,
|
| 578 |
+
896,
|
| 579 |
+
128,
|
| 580 |
+
1024,
|
| 581 |
+
256,
|
| 582 |
+
384,
|
| 583 |
+
1024,
|
| 584 |
+
640,
|
| 585 |
+
896
|
| 586 |
+
],
|
| 587 |
+
[
|
| 588 |
+
768,
|
| 589 |
+
384,
|
| 590 |
+
640,
|
| 591 |
+
896,
|
| 592 |
+
256,
|
| 593 |
+
128,
|
| 594 |
+
384,
|
| 595 |
+
896,
|
| 596 |
+
128,
|
| 597 |
+
128,
|
| 598 |
+
896,
|
| 599 |
+
256,
|
| 600 |
+
384,
|
| 601 |
+
896,
|
| 602 |
+
512,
|
| 603 |
+
768,
|
| 604 |
+
768,
|
| 605 |
+
512,
|
| 606 |
+
512,
|
| 607 |
+
768,
|
| 608 |
+
896,
|
| 609 |
+
640,
|
| 610 |
+
768,
|
| 611 |
+
896,
|
| 612 |
+
896,
|
| 613 |
+
896,
|
| 614 |
+
640,
|
| 615 |
+
896,
|
| 616 |
+
640,
|
| 617 |
+
512,
|
| 618 |
+
896,
|
| 619 |
+
256,
|
| 620 |
+
512,
|
| 621 |
+
128,
|
| 622 |
+
512,
|
| 623 |
+
384,
|
| 624 |
+
768,
|
| 625 |
+
768,
|
| 626 |
+
1024,
|
| 627 |
+
256,
|
| 628 |
+
768,
|
| 629 |
+
256,
|
| 630 |
+
768,
|
| 631 |
+
512,
|
| 632 |
+
640,
|
| 633 |
+
1024,
|
| 634 |
+
128,
|
| 635 |
+
896,
|
| 636 |
+
896,
|
| 637 |
+
896,
|
| 638 |
+
896,
|
| 639 |
+
1024
|
| 640 |
+
],
|
| 641 |
+
[
|
| 642 |
+
512,
|
| 643 |
+
384,
|
| 644 |
+
768,
|
| 645 |
+
640,
|
| 646 |
+
640,
|
| 647 |
+
768,
|
| 648 |
+
1024,
|
| 649 |
+
896,
|
| 650 |
+
512,
|
| 651 |
+
256,
|
| 652 |
+
640,
|
| 653 |
+
768,
|
| 654 |
+
640,
|
| 655 |
+
896,
|
| 656 |
+
128,
|
| 657 |
+
256,
|
| 658 |
+
896,
|
| 659 |
+
1024,
|
| 660 |
+
256,
|
| 661 |
+
640,
|
| 662 |
+
512,
|
| 663 |
+
256,
|
| 664 |
+
128,
|
| 665 |
+
512,
|
| 666 |
+
256,
|
| 667 |
+
640,
|
| 668 |
+
768,
|
| 669 |
+
768,
|
| 670 |
+
128,
|
| 671 |
+
128,
|
| 672 |
+
768,
|
| 673 |
+
640,
|
| 674 |
+
1024,
|
| 675 |
+
1024,
|
| 676 |
+
768,
|
| 677 |
+
512,
|
| 678 |
+
896,
|
| 679 |
+
768,
|
| 680 |
+
896,
|
| 681 |
+
1024,
|
| 682 |
+
896,
|
| 683 |
+
896,
|
| 684 |
+
512,
|
| 685 |
+
640,
|
| 686 |
+
1024,
|
| 687 |
+
512,
|
| 688 |
+
1024,
|
| 689 |
+
512,
|
| 690 |
+
512,
|
| 691 |
+
512,
|
| 692 |
+
768
|
| 693 |
+
],
|
| 694 |
+
[
|
| 695 |
+
896,
|
| 696 |
+
896,
|
| 697 |
+
1024,
|
| 698 |
+
1024,
|
| 699 |
+
896,
|
| 700 |
+
128,
|
| 701 |
+
768,
|
| 702 |
+
256,
|
| 703 |
+
1024,
|
| 704 |
+
256,
|
| 705 |
+
1024,
|
| 706 |
+
640,
|
| 707 |
+
384,
|
| 708 |
+
256,
|
| 709 |
+
256,
|
| 710 |
+
512,
|
| 711 |
+
768,
|
| 712 |
+
896,
|
| 713 |
+
512,
|
| 714 |
+
768,
|
| 715 |
+
384,
|
| 716 |
+
1024,
|
| 717 |
+
896,
|
| 718 |
+
896,
|
| 719 |
+
1024,
|
| 720 |
+
896,
|
| 721 |
+
768,
|
| 722 |
+
896,
|
| 723 |
+
640,
|
| 724 |
+
1024,
|
| 725 |
+
512,
|
| 726 |
+
896,
|
| 727 |
+
512,
|
| 728 |
+
1024,
|
| 729 |
+
512,
|
| 730 |
+
512,
|
| 731 |
+
256,
|
| 732 |
+
256,
|
| 733 |
+
256,
|
| 734 |
+
512,
|
| 735 |
+
768,
|
| 736 |
+
128,
|
| 737 |
+
384,
|
| 738 |
+
512,
|
| 739 |
+
896,
|
| 740 |
+
896,
|
| 741 |
+
1024,
|
| 742 |
+
256,
|
| 743 |
+
384,
|
| 744 |
+
640
|
| 745 |
+
],
|
| 746 |
+
[
|
| 747 |
+
896,
|
| 748 |
+
640,
|
| 749 |
+
384,
|
| 750 |
+
512,
|
| 751 |
+
256,
|
| 752 |
+
640,
|
| 753 |
+
1024,
|
| 754 |
+
384,
|
| 755 |
+
1024,
|
| 756 |
+
1024,
|
| 757 |
+
768,
|
| 758 |
+
256,
|
| 759 |
+
1024,
|
| 760 |
+
768,
|
| 761 |
+
512,
|
| 762 |
+
896,
|
| 763 |
+
256,
|
| 764 |
+
1024,
|
| 765 |
+
768,
|
| 766 |
+
768,
|
| 767 |
+
768,
|
| 768 |
+
384,
|
| 769 |
+
384,
|
| 770 |
+
256,
|
| 771 |
+
1024,
|
| 772 |
+
384,
|
| 773 |
+
384,
|
| 774 |
+
384,
|
| 775 |
+
896,
|
| 776 |
+
768,
|
| 777 |
+
640,
|
| 778 |
+
768,
|
| 779 |
+
512,
|
| 780 |
+
896,
|
| 781 |
+
896,
|
| 782 |
+
896,
|
| 783 |
+
896,
|
| 784 |
+
256,
|
| 785 |
+
384,
|
| 786 |
+
128,
|
| 787 |
+
1024,
|
| 788 |
+
896,
|
| 789 |
+
256,
|
| 790 |
+
256,
|
| 791 |
+
768,
|
| 792 |
+
640,
|
| 793 |
+
896,
|
| 794 |
+
384,
|
| 795 |
+
768,
|
| 796 |
+
512,
|
| 797 |
+
640
|
| 798 |
+
],
|
| 799 |
+
[
|
| 800 |
+
896,
|
| 801 |
+
1024,
|
| 802 |
+
768,
|
| 803 |
+
1024,
|
| 804 |
+
896,
|
| 805 |
+
256,
|
| 806 |
+
768,
|
| 807 |
+
128,
|
| 808 |
+
128,
|
| 809 |
+
768,
|
| 810 |
+
512,
|
| 811 |
+
896,
|
| 812 |
+
384,
|
| 813 |
+
768,
|
| 814 |
+
1024,
|
| 815 |
+
256,
|
| 816 |
+
768,
|
| 817 |
+
768,
|
| 818 |
+
256,
|
| 819 |
+
512,
|
| 820 |
+
512,
|
| 821 |
+
640,
|
| 822 |
+
512,
|
| 823 |
+
256,
|
| 824 |
+
768,
|
| 825 |
+
896,
|
| 826 |
+
384,
|
| 827 |
+
1024,
|
| 828 |
+
640,
|
| 829 |
+
1024,
|
| 830 |
+
512,
|
| 831 |
+
512,
|
| 832 |
+
384,
|
| 833 |
+
512,
|
| 834 |
+
512,
|
| 835 |
+
1024,
|
| 836 |
+
384,
|
| 837 |
+
896,
|
| 838 |
+
768,
|
| 839 |
+
384,
|
| 840 |
+
384,
|
| 841 |
+
128,
|
| 842 |
+
384,
|
| 843 |
+
1024,
|
| 844 |
+
896,
|
| 845 |
+
640,
|
| 846 |
+
768,
|
| 847 |
+
768,
|
| 848 |
+
256,
|
| 849 |
+
640,
|
| 850 |
+
512,
|
| 851 |
+
640,
|
| 852 |
+
384
|
| 853 |
+
]
|
| 854 |
+
],
|
| 855 |
+
"glean_metadata": {
|
| 856 |
+
"base_model": "allenai/OLMoE-1B-7B-0125-Instruct",
|
| 857 |
+
"block_size": 128,
|
| 858 |
+
"criterion": "reap",
|
| 859 |
+
"dead_experts": 217,
|
| 860 |
+
"keep_fraction": 0.5,
|
| 861 |
+
"min_width": 128,
|
| 862 |
+
"params": 3697491968,
|
| 863 |
+
"scores": "outputs/scores_0125inst_dolmino-math/scores.pt"
|
| 864 |
+
},
|
| 865 |
+
"hidden_act": "silu",
|
| 866 |
+
"hidden_size": 2048,
|
| 867 |
+
"initializer_range": 0.02,
|
| 868 |
+
"intermediate_size": 1024,
|
| 869 |
+
"max_position_embeddings": 4096,
|
| 870 |
+
"model_type": "pruned_olmoe",
|
| 871 |
+
"norm_topk_prob": false,
|
| 872 |
+
"num_attention_heads": 16,
|
| 873 |
+
"num_experts": 64,
|
| 874 |
+
"num_experts_per_tok": 8,
|
| 875 |
+
"num_hidden_layers": 16,
|
| 876 |
+
"num_key_value_heads": 16,
|
| 877 |
+
"output_router_logits": false,
|
| 878 |
+
"pad_token_id": 1,
|
| 879 |
+
"rms_norm_eps": 1e-05,
|
| 880 |
+
"rope_scaling": null,
|
| 881 |
+
"rope_theta": 10000.0,
|
| 882 |
+
"router_aux_loss_coef": 0.01,
|
| 883 |
+
"tie_word_embeddings": false,
|
| 884 |
+
"transformers_version": "4.57.6",
|
| 885 |
+
"use_cache": false,
|
| 886 |
+
"vocab_size": 50304
|
| 887 |
+
}
|
healed/opd_warm_fixed_keep50/step0200/configuration_pruned_olmoe.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Configuration for GLEAN-pruned OLMoE: variable-width, variable-count experts.
|
| 2 |
+
"""
|
| 3 |
+
|
| 4 |
+
from transformers.models.olmoe.configuration_olmoe import OlmoeConfig
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
class PrunedOlmoeConfig(OlmoeConfig):
|
| 8 |
+
"""OlmoeConfig plus a per-(layer, expert) width table.
|
| 9 |
+
|
| 10 |
+
``expert_widths[l]`` lists the SwiGLU intermediate width of each surviving
|
| 11 |
+
expert in decoder layer ``l``, in expert order. Lists are ragged: layers
|
| 12 |
+
may keep different numbers of experts (deleted experts simply don't
|
| 13 |
+
appear — the router in layer ``l`` has ``len(expert_widths[l])`` rows),
|
| 14 |
+
and each width may differ (multiples of the GEMM block size, 128, for
|
| 15 |
+
variable-MegaBlocks execution). ``None`` means an unpruned model
|
| 16 |
+
(uniform ``num_experts`` × ``intermediate_size``).
|
| 17 |
+
|
| 18 |
+
The inherited ``num_experts`` / ``intermediate_size`` keep their ORIGINAL
|
| 19 |
+
(pre-pruning) values for provenance; the width table is authoritative for
|
| 20 |
+
the built architecture.
|
| 21 |
+
"""
|
| 22 |
+
|
| 23 |
+
model_type = "pruned_olmoe"
|
| 24 |
+
|
| 25 |
+
def __init__(self, expert_widths: list[list[int]] | None = None, **kwargs):
|
| 26 |
+
super().__init__(**kwargs)
|
| 27 |
+
self.expert_widths = expert_widths
|