Add files using upload-large-folder tool

healed/calib/glean_keep25_s1224.eval.log

@@ -0,0 +1,81 @@
+2026-07-17T21:06:49-07:00 serving outputs/healed/calib/glean_keep25_s1224/step0100 on GPU 0 port 8430
+2026-07-17T21:06:49-07:00 waiting for server /health ...
+2026-07-17T21:07:14-07:00 server up; chat pass [gsm8k_cot_zeroshot]
+2026-07-17:21:07:15 WARNING  [config.evaluate_config:287] --limit SHOULD ONLY BE USED FOR TESTING. REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.
+2026-07-17:21:07:22 INFO     [_cli.run:388] Selected Tasks: ['gsm8k_cot_zeroshot']
+2026-07-17:21:07:23 INFO     [evaluator:214] Setting random seed to 0 | Setting numpy seed to 1234 | Setting torch manual seed to 1234 | Setting fewshot manual seed to 1234
+2026-07-17:21:07:23 WARNING  [evaluator:226] generation_kwargs: {'max_gen_toks': 1280} specified through cli, these settings will update set parameters in yaml tasks. Ensure 'do_sample=True' for non-greedy decoding!
+2026-07-17:21:07:23 INFO     [evaluator:239] Initializing local-chat-completions model, with arguments: {'model': 'student', 'base_url': 'http://127.0.0.1:8430/v1/chat/completions', 'num_concurrent': 48, 'tokenized_requests': False, 'max_retries': 3}
+2026-07-17:21:07:23 INFO     [models.api_models:179] Using max length 2048 - 1
+2026-07-17:21:07:23 INFO     [models.api_models:200] Using tokenizer None
+2026-07-17:21:07:25 INFO     [evaluator_utils:446] Selected tasks:
+2026-07-17:21:07:25 INFO     [evaluator_utils:480] Task: gsm8k_cot_zeroshot (gsm8k/gsm8k-cot-zeroshot.yaml)
+2026-07-17:21:07:25 INFO     [evaluator:314] gsm8k_cot_zeroshot: Using gen_kwargs: {'until': ['Q:', '</s>', '<|im_end|>'], 'do_sample': False, 'max_gen_toks': 1280}
+2026-07-17:21:07:25 INFO     [api.task:312] Building contexts for gsm8k_cot_zeroshot on rank 0...
+
  0%|          | 0/500 [00:00<?, ?it/s]
 38%|███▊      | 189/500 [00:00<00:00, 1882.41it/s]
 77%|███████▋  | 386/500 [00:00<00:00, 1932.60it/s]
+2026-07-17:21:07:25 INFO     [evaluator:585] Running generate_until requests
+2026-07-17:21:07:25 INFO     [models.api_models:747] Tokenized requests are disabled. Context + generation length is not checked.
+
+2026-07-17:21:08:50 INFO     [loggers.evaluation_tracker:247] Saving results aggregated
+2026-07-17:21:08:50 INFO     [loggers.evaluation_tracker:119] Saving per-task samples to outputs/evals/general_suite/calib/keep25_step100/student/*.jsonl
+local-chat-completions ({'model': 'student', 'base_url': 'http://127.0.0.1:8430/v1/chat/completions', 'num_concurrent': 48, 'tokenized_requests': False, 'max_retries': 3}), gen_kwargs: ({'max_gen_toks': 1280}), limit: 500.0, num_fewshot: None, batch_size: 1
+|      Tasks       |Version|     Filter     |n-shot|  Metric   |   |Value|   |Stderr|
+|------------------|------:|----------------|-----:|-----------|---|----:|---|-----:|
+|gsm8k_cot_zeroshot|      3|flexible-extract|     0|exact_match|↑  |0.182|±  |0.0173|
+|                  |       |strict-match    |     0|exact_match|↑  |0.000|±  |0.0000|
+
+2026-07-17T21:08:51-07:00 lm_eval exit=0 -> outputs/evals/general_suite/calib/keep25_step100
+2026-07-17T22:31:43-07:00 serving outputs/healed/calib/glean_keep25_s1224/step0200 on GPU 0 port 8430
+2026-07-17T22:31:43-07:00 waiting for server /health ...
+2026-07-17T22:32:08-07:00 server up; chat pass [gsm8k_cot_zeroshot]
+2026-07-17:22:32:08 WARNING  [config.evaluate_config:287] --limit SHOULD ONLY BE USED FOR TESTING. REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.
+2026-07-17:22:32:15 INFO     [_cli.run:388] Selected Tasks: ['gsm8k_cot_zeroshot']
+2026-07-17:22:32:16 INFO     [evaluator:214] Setting random seed to 0 | Setting numpy seed to 1234 | Setting torch manual seed to 1234 | Setting fewshot manual seed to 1234
+2026-07-17:22:32:16 WARNING  [evaluator:226] generation_kwargs: {'max_gen_toks': 1280} specified through cli, these settings will update set parameters in yaml tasks. Ensure 'do_sample=True' for non-greedy decoding!
+2026-07-17:22:32:16 INFO     [evaluator:239] Initializing local-chat-completions model, with arguments: {'model': 'student', 'base_url': 'http://127.0.0.1:8430/v1/chat/completions', 'num_concurrent': 48, 'tokenized_requests': False, 'max_retries': 3}
+2026-07-17:22:32:16 INFO     [models.api_models:179] Using max length 2048 - 1
+2026-07-17:22:32:16 INFO     [models.api_models:200] Using tokenizer None
+2026-07-17:22:32:18 INFO     [evaluator_utils:446] Selected tasks:
+2026-07-17:22:32:18 INFO     [evaluator_utils:480] Task: gsm8k_cot_zeroshot (gsm8k/gsm8k-cot-zeroshot.yaml)
+2026-07-17:22:32:18 INFO     [evaluator:314] gsm8k_cot_zeroshot: Using gen_kwargs: {'until': ['Q:', '</s>', '<|im_end|>'], 'do_sample': False, 'max_gen_toks': 1280}
+2026-07-17:22:32:18 INFO     [api.task:312] Building contexts for gsm8k_cot_zeroshot on rank 0...
+
  0%|          | 0/500 [00:00<?, ?it/s]
 37%|███▋      | 185/500 [00:00<00:00, 1848.56it/s]
 76%|███████▌  | 380/500 [00:00<00:00, 1903.49it/s]
+2026-07-17:22:32:18 INFO     [evaluator:585] Running generate_until requests
+2026-07-17:22:32:18 INFO     [models.api_models:747] Tokenized requests are disabled. Context + generation length is not checked.
+
+2026-07-17:22:33:31 INFO     [loggers.evaluation_tracker:247] Saving results aggregated
+2026-07-17:22:33:31 INFO     [loggers.evaluation_tracker:119] Saving per-task samples to outputs/evals/general_suite/calib/keep25_step200/student/*.jsonl
+local-chat-completions ({'model': 'student', 'base_url': 'http://127.0.0.1:8430/v1/chat/completions', 'num_concurrent': 48, 'tokenized_requests': False, 'max_retries': 3}), gen_kwargs: ({'max_gen_toks': 1280}), limit: 500.0, num_fewshot: None, batch_size: 1
+|      Tasks       |Version|     Filter     |n-shot|  Metric   |   |Value|   |Stderr|
+|------------------|------:|----------------|-----:|-----------|---|----:|---|-----:|
+|gsm8k_cot_zeroshot|      3|flexible-extract|     0|exact_match|↑  |0.238|±  |0.0191|
+|                  |       |strict-match    |     0|exact_match|↑  |0.000|±  |0.0000|
+
+2026-07-17T22:33:32-07:00 lm_eval exit=0 -> outputs/evals/general_suite/calib/keep25_step200
+2026-07-17T23:55:53-07:00 serving outputs/healed/calib/glean_keep25_s1224/step0300 on GPU 0 port 8430
+2026-07-17T23:55:53-07:00 waiting for server /health ...
+2026-07-17T23:56:18-07:00 server up; chat pass [gsm8k_cot_zeroshot]
+2026-07-17:23:56:18 WARNING  [config.evaluate_config:287] --limit SHOULD ONLY BE USED FOR TESTING. REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.
+2026-07-17:23:56:25 INFO     [_cli.run:388] Selected Tasks: ['gsm8k_cot_zeroshot']
+2026-07-17:23:56:26 INFO     [evaluator:214] Setting random seed to 0 | Setting numpy seed to 1234 | Setting torch manual seed to 1234 | Setting fewshot manual seed to 1234
+2026-07-17:23:56:26 WARNING  [evaluator:226] generation_kwargs: {'max_gen_toks': 1280} specified through cli, these settings will update set parameters in yaml tasks. Ensure 'do_sample=True' for non-greedy decoding!
+2026-07-17:23:56:26 INFO     [evaluator:239] Initializing local-chat-completions model, with arguments: {'model': 'student', 'base_url': 'http://127.0.0.1:8430/v1/chat/completions', 'num_concurrent': 48, 'tokenized_requests': False, 'max_retries': 3}
+2026-07-17:23:56:26 INFO     [models.api_models:179] Using max length 2048 - 1
+2026-07-17:23:56:26 INFO     [models.api_models:200] Using tokenizer None
+2026-07-17:23:56:28 INFO     [evaluator_utils:446] Selected tasks:
+2026-07-17:23:56:28 INFO     [evaluator_utils:480] Task: gsm8k_cot_zeroshot (gsm8k/gsm8k-cot-zeroshot.yaml)
+2026-07-17:23:56:28 INFO     [evaluator:314] gsm8k_cot_zeroshot: Using gen_kwargs: {'until': ['Q:', '</s>', '<|im_end|>'], 'do_sample': False, 'max_gen_toks': 1280}
+2026-07-17:23:56:28 INFO     [api.task:312] Building contexts for gsm8k_cot_zeroshot on rank 0...
+
  0%|          | 0/500 [00:00<?, ?it/s]
 37%|███▋      | 187/500 [00:00<00:00, 1862.95it/s]
 76%|███████▋  | 382/500 [00:00<00:00, 1910.76it/s]
+2026-07-17:23:56:28 INFO     [evaluator:585] Running generate_until requests
+2026-07-17:23:56:28 INFO     [models.api_models:747] Tokenized requests are disabled. Context + generation length is not checked.
+
+2026-07-17:23:57:31 INFO     [loggers.evaluation_tracker:247] Saving results aggregated
+2026-07-17:23:57:31 INFO     [loggers.evaluation_tracker:119] Saving per-task samples to outputs/evals/general_suite/calib/keep25_step300/student/*.jsonl
+local-chat-completions ({'model': 'student', 'base_url': 'http://127.0.0.1:8430/v1/chat/completions', 'num_concurrent': 48, 'tokenized_requests': False, 'max_retries': 3}), gen_kwargs: ({'max_gen_toks': 1280}), limit: 500.0, num_fewshot: None, batch_size: 1
+|      Tasks       |Version|     Filter     |n-shot|  Metric   |   |Value|   |Stderr|
+|------------------|------:|----------------|-----:|-----------|---|----:|---|-----:|
+|gsm8k_cot_zeroshot|      3|flexible-extract|     0|exact_match|↑  |0.244|±  |0.0192|
+|                  |       |strict-match    |     0|exact_match|↑  |0.004|±  |0.0028|
+
+2026-07-17T23:57:32-07:00 lm_eval exit=0 -> outputs/evals/general_suite/calib/keep25_step300

healed/calib/glean_keep25_s1224/wandb/run-20260717_210858-ypxqb0r8/files/config.yaml

@@ -0,0 +1,246 @@
+_wandb:
+    value:
+        cli_version: 0.28.0
+        e:
+            toks52dbikg8amgtxadijyojntrgsti4:
+                args:
+                    - --training-mode
+                    - off-policy
+                    - --kl-direction
+                    - forward
+                    - --topk-targets
+                    - outputs/teacher_trajectories/dolci_combined_top128
+                    - --off-policy-frames
+                    - chat
+                    - --student-device
+                    - cuda:0
+                    - --student
+                    - outputs/pruned/glean-0125inst-general-keep25
+                    - --resume-from
+                    - outputs/healed/calib/glean_keep25_s1224/step0100
+                    - --lr
+                    - "3e-5"
+                    - --optimizer
+                    - adamw8bit
+                    - --epochs
+                    - "9"
+                    - --sweep
+                    - "200"
+                    - --micro-batch
+                    - "3"
+                    - --loss-tokens-per-step
+                    - "120000"
+                    - --gsm8k-every
+                    - "0"
+                    - --save-every
+                    - "200"
+                    - --seed
+                    - "1224"
+                    - --out-dir
+                    - outputs/healed/calib/glean_keep25_s1224
+                    - --wandb
+                    - --wandb-mode
+                    - online
+                    - --wandb-project
+                    - glean-calib
+                    - --wandb-run-name
+                    - calib_keep25_200
+                codePath: scripts/11_distill_on_policy.py
+                codePathLocal: scripts/11_distill_on_policy.py
+                cpu_count: 24
+                cpu_count_logical: 48
+                cudaVersion: "13.0"
+                disk:
+                    /:
+                        total: "1958315118592"
+                        used: "1632623116288"
+                email: hbfreed@protonmail.com
+                executable: /home/henry/Documents/PythonProjects/variable-reap/.venv/bin/python3
+                git:
+                    commit: 247c7f0113d7a058b1db8044deddfd4456cc998d
+                    remote: https://github.com/hbfreed/variable-reap.git
+                gpu: NVIDIA GeForce RTX 3090
+                gpu_count: 3
+                gpu_nvidia:
+                    - architecture: Ampere
+                      cudaCores: 10496
+                      memoryTotal: "25769803776"
+                      name: NVIDIA GeForce RTX 3090
+                      uuid: GPU-8ca70870-ddf2-d274-bcc5-182c2075bced
+                    - architecture: Ampere
+                      cudaCores: 10496
+                      memoryTotal: "25769803776"
+                      name: NVIDIA GeForce RTX 3090
+                      uuid: GPU-a6acf07f-31f5-618f-a5d0-c0017e7e2e27
+                    - architecture: Ampere
+                      cudaCores: 10496
+                      memoryTotal: "25769803776"
+                      name: NVIDIA GeForce RTX 3090
+                      uuid: GPU-864c54df-0130-7780-e271-8a5551d1733f
+                host: pop-os
+                memory:
+                    total: "134900756480"
+                os: Linux-7.0.11-76070011-generic-x86_64-with-glibc2.35
+                program: /home/henry/Documents/PythonProjects/variable-reap/scripts/11_distill_on_policy.py
+                python: CPython 3.12.12
+                root: outputs/healed/calib/glean_keep25_s1224
+                startedAt: "2026-07-18T04:08:58.716590Z"
+                writerId: toks52dbikg8amgtxadijyojntrgsti4
+        m:
+            - "1": step
+              "6":
+                - 3
+              "7": []
+            - "2": '*'
+              "5": 1
+              "6":
+                - 1
+              "7": []
+        python_version: 3.12.12
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 71
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 71
+            "3":
+                - 2
+                - 7
+                - 13
+                - 16
+            "4": 3.12.12
+            "5": 0.28.0
+            "6": 4.57.6
+            "12": 0.28.0
+            "13": linux-x86_64
+dataset:
+    value: allenai/RLVR-MATH
+dataset_sources:
+    value: null
+debug:
+    value: false
+epochs:
+    value: 9
+eval_every:
+    value: 10
+group_size:
+    value: 1
+gsm8k_batch:
+    value: 16
+gsm8k_every:
+    value: 0
+gsm8k_frames:
+    value: chat
+gsm8k_max_new_tokens:
+    value: 512
+gsm8k_n:
+    value: 256
+kl_direction:
+    value: forward
+loss_tokens_per_step:
+    value: 120000
+lr:
+    value: 3e-05
+max_difficulty:
+    value: null
+max_grad_norm:
+    value: 1
+max_loss_tokens:
+    value: null
+max_new_tokens:
+    value: 256
+max_prompt_len:
+    value: 1024
+micro_batch:
+    value: 3
+no_grad_checkpointing:
+    value: false
+no_teacher_overlap:
+    value: false
+no_wandb_sync:
+    value: false
+off_policy_frames:
+    value: chat
+off_policy_max_seq_len:
+    value: 2048
+optimizer:
+    value: adamw8bit
+out_dir:
+    value: outputs/healed/calib/glean_keep25_s1224
+prompts_per_step:
+    value: 256
+resolved_kl_direction:
+    value: forward
+resume_from:
+    value: outputs/healed/calib/glean_keep25_s1224/step0100
+rollout_batch:
+    value: 64
+rollout_engine:
+    value: hf
+save_every:
+    value: 200
+seed:
+    value: 1224
+start_step:
+    value: 0
+student:
+    value: outputs/pruned/glean-0125inst-general-keep25
+student_device:
+    value: cuda:0
+sweep:
+    value: 200
+sync_checkpoints:
+    value: false
+teacher:
+    value: allenai/OLMoE-1B-7B-0125-Instruct
+teacher_device:
+    value: cuda:0
+topk_targets:
+    value: outputs/teacher_trajectories/dolci_combined_top128
+training_mode:
+    value: off-policy
+trajectories:
+    value: outputs/teacher_trajectories/dolci_math_curated.jsonl
+trajectory_dataset:
+    value: allenai/Dolci-Instruct-RL
+vllm_gpu:
+    value: null
+vllm_gpu_mem_util:
+    value: 0.85
+vllm_live_dir:
+    value: null
+vllm_port:
+    value: 8377
+vllm_refresh_every:
+    value: 5
+vllm_refresh_mode:
+    value: reload
+vllm_serve_bin:
+    value: vllm-plugin/.venv/bin/python
+wandb:
+    value: true
+wandb_mode:
+    value: online
+wandb_project:
+    value: glean-calib
+wandb_resume:
+    value: null
+wandb_run_id:
+    value: null
+wandb_run_name:
+    value: calib_keep25_200
+warmup_steps:
+    value: 10
+weight_decay:
+    value: 0.1

healed/calib/glean_keep25_s1224/wandb/run-20260717_210858-ypxqb0r8/files/output.log

@@ -0,0 +1,115 @@
+resumed student weights from outputs/healed/calib/glean_keep25_s1224/step0100 (fresh optimizer, step counter at 0)
+58360 cached top-128 chat trajectories / 22,295,631 unique tokens | 185 steps/epoch | 200 total steps | student params 2.09B | teacher overlap=False
+restored optimizer/scheduler state from step 100; rebuilt 228 paged buffers
+{"step": 101, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.6404966282300651, "tokens": 120000, "cumulative_loss_tokens": 12120000, "grad_norm": 0.6953125, "lr": 3e-05, "finish_rate": 0.994, "comp_len": 343.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 49.9, "frames": {"chat": 349}, "mem_gb": 9.61}
+{"step": 102, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.6106020681867376, "tokens": 120000, "cumulative_loss_tokens": 12240000, "grad_norm": 0.71484375, "lr": 3e-05, "finish_rate": 0.968, "comp_len": 384.6, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.1, "frames": {"chat": 312}, "mem_gb": 9.73}
+{"step": 103, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.22530150232814875, "tokens": 120000, "cumulative_loss_tokens": 12360000, "grad_norm": 0.4921875, "lr": 3e-05, "finish_rate": 0.817, "comp_len": 524.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 42.0, "frames": {"chat": 229}, "mem_gb": 10.0}
+{"step": 104, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.5713284558854997, "tokens": 120000, "cumulative_loss_tokens": 12480000, "grad_norm": 0.66796875, "lr": 3e-05, "finish_rate": 0.938, "comp_len": 416.7, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 48.1, "frames": {"chat": 288}, "mem_gb": 10.0}
+{"step": 105, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.7972744106804331, "tokens": 120000, "cumulative_loss_tokens": 12600000, "grad_norm": 0.8828125, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 341.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 54.7, "frames": {"chat": 351}, "mem_gb": 9.66}
+{"step": 106, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.3168164167162031, "tokens": 120000, "cumulative_loss_tokens": 12720000, "grad_norm": 0.578125, "lr": 3e-05, "finish_rate": 0.812, "comp_len": 538.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 36.9, "frames": {"chat": 223}, "mem_gb": 10.02}
+{"step": 107, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.15695743769599746, "tokens": 120000, "cumulative_loss_tokens": 12840000, "grad_norm": 0.455078125, "lr": 3e-05, "finish_rate": 0.793, "comp_len": 563.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 36.4, "frames": {"chat": 213}, "mem_gb": 9.99}
+{"step": 108, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.7303095779831211, "tokens": 120000, "cumulative_loss_tokens": 12960000, "grad_norm": 0.77734375, "lr": 3e-05, "finish_rate": 0.995, "comp_len": 315.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 54.1, "frames": {"chat": 380}, "mem_gb": 9.84}
+{"step": 109, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.4183019400515283, "tokens": 120000, "cumulative_loss_tokens": 13080000, "grad_norm": 0.609375, "lr": 3e-05, "finish_rate": 0.893, "comp_len": 442.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 39.9, "frames": {"chat": 271}, "mem_gb": 10.02}
+{"step": 110, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.7819241975242893, "tokens": 120000, "cumulative_loss_tokens": 13200000, "grad_norm": 0.73828125, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 352.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 49.6, "frames": {"chat": 340}, "mem_gb": 9.91}
+The attention mask is not set and cannot be inferred from input because pad token is same as eos token. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.
+[eval step 110] sample: 'To determine the rank of a matrix, we need to find the maximum number of linearly independent rows or columns in the matrix. \n\nGiven the matrix:\n\\[\n\\begin{bmatrix}\n12 & -16 & 4 & 16 \\\\\n-9 & 11 & -1 & '
+{"step": 111, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.8300092002927947, "tokens": 120000, "cumulative_loss_tokens": 13320000, "grad_norm": 0.8125, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 327.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 57.0, "frames": {"chat": 366}, "mem_gb": 9.61}
+{"step": 112, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.2994033165920526, "tokens": 120000, "cumulative_loss_tokens": 13440000, "grad_norm": 0.5, "lr": 3e-05, "finish_rate": 0.854, "comp_len": 515.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 39.5, "frames": {"chat": 233}, "mem_gb": 10.0}
+{"step": 113, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.5316771011674156, "tokens": 120000, "cumulative_loss_tokens": 13560000, "grad_norm": 0.671875, "lr": 3e-05, "finish_rate": 0.968, "comp_len": 384.6, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 48.1, "frames": {"chat": 312}, "mem_gb": 9.86}
+{"step": 114, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.7487247935026263, "tokens": 120000, "cumulative_loss_tokens": 13680000, "grad_norm": 0.7734375, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 330.6, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 52.6, "frames": {"chat": 363}, "mem_gb": 9.58}
+{"step": 115, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.7562611924018711, "tokens": 120000, "cumulative_loss_tokens": 13800000, "grad_norm": 0.7578125, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 371.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 49.1, "frames": {"chat": 323}, "mem_gb": 9.85}
+{"step": 116, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.8215424215157827, "tokens": 120000, "cumulative_loss_tokens": 13920000, "grad_norm": 0.80859375, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 346.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 52.4, "frames": {"chat": 346}, "mem_gb": 9.59}
+{"step": 117, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.7370786963778237, "tokens": 120000, "cumulative_loss_tokens": 14040000, "grad_norm": 1.0234375, "lr": 3e-05, "finish_rate": 0.975, "comp_len": 377.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 47.3, "frames": {"chat": 318}, "mem_gb": 9.74}
+{"step": 118, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.5495684528866162, "tokens": 120000, "cumulative_loss_tokens": 14160000, "grad_norm": 0.64453125, "lr": 3e-05, "finish_rate": 0.935, "comp_len": 436.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 48.6, "frames": {"chat": 275}, "mem_gb": 9.93}
+{"step": 119, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.6848975222120062, "tokens": 120000, "cumulative_loss_tokens": 14280000, "grad_norm": 0.76953125, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 347.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 52.9, "frames": {"chat": 345}, "mem_gb": 9.67}
+{"step": 120, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.6866140180771549, "tokens": 120000, "cumulative_loss_tokens": 14400000, "grad_norm": 0.80078125, "lr": 3e-05, "finish_rate": 0.992, "comp_len": 318.3, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 53.8, "frames": {"chat": 377}, "mem_gb": 9.82}
+[eval step 120] sample: 'To find the rank of a matrix, we need to determine the maximum number of linearly independent rows or columns in the matrix. \n\nGiven the matrix:\n\\[\n\\begin{bmatrix}\n12 & -16 & 4 & 16 \\\\\n-9 & 11 & -1 & '
+{"step": 121, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.5571080892120799, "tokens": 120000, "cumulative_loss_tokens": 14520000, "grad_norm": 0.66796875, "lr": 3e-05, "finish_rate": 0.956, "comp_len": 408.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 46.2, "frames": {"chat": 294}, "mem_gb": 10.04}
+{"step": 122, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.7723195617940277, "tokens": 120000, "cumulative_loss_tokens": 14640000, "grad_norm": 0.7578125, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 355.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 50.7, "frames": {"chat": 338}, "mem_gb": 9.73}
+{"step": 123, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.7758043260567511, "tokens": 120000, "cumulative_loss_tokens": 14760000, "grad_norm": 0.76953125, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 340.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 53.1, "frames": {"chat": 352}, "mem_gb": 9.63}
+{"step": 124, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.7300908756038795, "tokens": 120000, "cumulative_loss_tokens": 14880000, "grad_norm": 0.765625, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 345.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 55.2, "frames": {"chat": 347}, "mem_gb": 9.68}
+{"step": 125, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.44740089606537176, "tokens": 120000, "cumulative_loss_tokens": 15000000, "grad_norm": 0.65625, "lr": 3e-05, "finish_rate": 0.968, "comp_len": 387.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 46.6, "frames": {"chat": 310}, "mem_gb": 9.66}
+{"step": 126, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.8197008320604762, "tokens": 120000, "cumulative_loss_tokens": 15120000, "grad_norm": 0.84375, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 335.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 53.7, "frames": {"chat": 358}, "mem_gb": 9.69}
+{"step": 127, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.47266409537059567, "tokens": 120000, "cumulative_loss_tokens": 15240000, "grad_norm": 0.68359375, "lr": 3e-05, "finish_rate": 0.947, "comp_len": 376.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 49.7, "frames": {"chat": 319}, "mem_gb": 9.97}
+{"step": 128, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.31353171376734973, "tokens": 120000, "cumulative_loss_tokens": 15360000, "grad_norm": 0.53515625, "lr": 3e-05, "finish_rate": 0.9, "comp_len": 446.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 46.2, "frames": {"chat": 269}, "mem_gb": 10.0}
+{"step": 129, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.6114998117436965, "tokens": 120000, "cumulative_loss_tokens": 15480000, "grad_norm": 0.796875, "lr": 3e-05, "finish_rate": 0.974, "comp_len": 387.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 54.0, "frames": {"chat": 310}, "mem_gb": 9.85}
+{"step": 130, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.39959967873397595, "tokens": 120000, "cumulative_loss_tokens": 15600000, "grad_norm": 0.62890625, "lr": 3e-05, "finish_rate": 0.834, "comp_len": 463.3, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 47.2, "frames": {"chat": 259}, "mem_gb": 10.03}
+[eval step 130] sample: 'To find the rank of a matrix, we need to determine the maximum number of linearly independent rows or columns in the matrix. A matrix is said to be of rank \\( r \\) if it has \\( r \\) linearly independe'
+{"step": 131, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.3943806648887073, "tokens": 120000, "cumulative_loss_tokens": 15720000, "grad_norm": 0.6015625, "lr": 3e-05, "finish_rate": 0.868, "comp_len": 452.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 42.2, "frames": {"chat": 265}, "mem_gb": 10.0}
+{"step": 132, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.7735084446456283, "tokens": 120000, "cumulative_loss_tokens": 15840000, "grad_norm": 0.82421875, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 356.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 55.7, "frames": {"chat": 337}, "mem_gb": 9.55}
+{"step": 133, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.43104476007341097, "tokens": 120000, "cumulative_loss_tokens": 15960000, "grad_norm": 0.625, "lr": 3e-05, "finish_rate": 0.972, "comp_len": 413.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 42.8, "frames": {"chat": 290}, "mem_gb": 9.81}
+{"step": 134, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.6807688963243738, "tokens": 120000, "cumulative_loss_tokens": 16080000, "grad_norm": 0.7578125, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 343.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 48.4, "frames": {"chat": 349}, "mem_gb": 9.75}
+{"step": 135, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.43767833087344965, "tokens": 120000, "cumulative_loss_tokens": 16200000, "grad_norm": 0.609375, "lr": 3e-05, "finish_rate": 0.928, "comp_len": 412.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 41.8, "frames": {"chat": 291}, "mem_gb": 9.84}
+{"step": 136, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.6971611228440577, "tokens": 120000, "cumulative_loss_tokens": 16320000, "grad_norm": 0.7734375, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 350.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 47.9, "frames": {"chat": 342}, "mem_gb": 9.71}
+{"step": 137, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.8007253739355132, "tokens": 120000, "cumulative_loss_tokens": 16440000, "grad_norm": 0.77734375, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 355.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 46.8, "frames": {"chat": 338}, "mem_gb": 9.66}
+{"step": 138, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.7883249476203074, "tokens": 120000, "cumulative_loss_tokens": 16560000, "grad_norm": 0.78515625, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 337.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 51.6, "frames": {"chat": 356}, "mem_gb": 9.62}
+{"step": 139, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.36883119474363824, "tokens": 120000, "cumulative_loss_tokens": 16680000, "grad_norm": 0.63671875, "lr": 3e-05, "finish_rate": 0.86, "comp_len": 452.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.2, "frames": {"chat": 265}, "mem_gb": 10.0}
+{"step": 140, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.6382841675106437, "tokens": 120000, "cumulative_loss_tokens": 16800000, "grad_norm": 0.765625, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 339.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 52.4, "frames": {"chat": 353}, "mem_gb": 9.71}
+[eval step 140] sample: 'To compute the rank of a matrix, we need to determine the maximum number of linearly independent rows or columns in the matrix. \n\nGiven the matrix:\n\\[\n\\begin{bmatrix}\n12 & -16 & 4 & 16 \\\\\n-9 & 11 & -1'
+{"step": 141, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.6029005106360341, "tokens": 120000, "cumulative_loss_tokens": 16920000, "grad_norm": 0.703125, "lr": 3e-05, "finish_rate": 0.949, "comp_len": 379.7, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 50.6, "frames": {"chat": 316}, "mem_gb": 9.9}
+{"step": 142, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.5026192011914216, "tokens": 120000, "cumulative_loss_tokens": 17040000, "grad_norm": 0.66796875, "lr": 3e-05, "finish_rate": 0.945, "comp_len": 415.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 46.3, "frames": {"chat": 289}, "mem_gb": 10.05}
+{"step": 143, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.5401858362942934, "tokens": 120000, "cumulative_loss_tokens": 17160000, "grad_norm": 0.8046875, "lr": 3e-05, "finish_rate": 0.982, "comp_len": 362.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 49.9, "frames": {"chat": 331}, "mem_gb": 9.69}
+{"step": 144, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.5338050864121566, "tokens": 120000, "cumulative_loss_tokens": 17280000, "grad_norm": 0.69921875, "lr": 3e-05, "finish_rate": 0.972, "comp_len": 375.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 46.6, "frames": {"chat": 320}, "mem_gb": 10.0}
+{"step": 145, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.48790957024022935, "tokens": 120000, "cumulative_loss_tokens": 17400000, "grad_norm": 0.71875, "lr": 3e-05, "finish_rate": 0.941, "comp_len": 441.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 40.0, "frames": {"chat": 272}, "mem_gb": 9.74}
+{"step": 146, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.8456092888927087, "tokens": 120000, "cumulative_loss_tokens": 17520000, "grad_norm": 0.83984375, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 326.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 54.8, "frames": {"chat": 368}, "mem_gb": 9.66}
+{"step": 147, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.6982750026626512, "tokens": 120000, "cumulative_loss_tokens": 17640000, "grad_norm": 0.76953125, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 365.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 53.0, "frames": {"chat": 328}, "mem_gb": 9.68}
+{"step": 148, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.6740354536671191, "tokens": 120000, "cumulative_loss_tokens": 17760000, "grad_norm": 0.75390625, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 368.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 49.7, "frames": {"chat": 326}, "mem_gb": 9.9}
+{"step": 149, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.7793807467692221, "tokens": 120000, "cumulative_loss_tokens": 17880000, "grad_norm": 0.78125, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 329.7, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 55.5, "frames": {"chat": 364}, "mem_gb": 9.84}
+{"step": 150, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.4921399115661159, "tokens": 120000, "cumulative_loss_tokens": 18000000, "grad_norm": 0.62109375, "lr": 3e-05, "finish_rate": 0.97, "comp_len": 405.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.5, "frames": {"chat": 296}, "mem_gb": 9.88}
+[eval step 150] sample: "To compute the rank of a matrix, we need to determine the maximum number of linearly independent rows or columns in the matrix. \n\nLet's start by writing the given 4x4 matrix:\n\n\\[\n\\begin{bmatrix}\n12 & "
+{"step": 151, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.751563901235722, "tokens": 120000, "cumulative_loss_tokens": 18120000, "grad_norm": 0.76953125, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 317.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 54.7, "frames": {"chat": 378}, "mem_gb": 9.74}
+{"step": 152, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.8484429204796752, "tokens": 120000, "cumulative_loss_tokens": 18240000, "grad_norm": 0.8515625, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 326.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 57.0, "frames": {"chat": 368}, "mem_gb": 9.83}
+{"step": 153, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.385568957734108, "tokens": 120000, "cumulative_loss_tokens": 18360000, "grad_norm": 0.62109375, "lr": 3e-05, "finish_rate": 0.898, "comp_len": 468.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.8, "frames": {"chat": 256}, "mem_gb": 9.94}
+{"step": 154, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.42377624020079774, "tokens": 120000, "cumulative_loss_tokens": 18480000, "grad_norm": 0.62109375, "lr": 3e-05, "finish_rate": 0.93, "comp_len": 381.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.1, "frames": {"chat": 315}, "mem_gb": 9.89}
+{"step": 155, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.40477768267107506, "tokens": 120000, "cumulative_loss_tokens": 18600000, "grad_norm": 0.58984375, "lr": 3e-05, "finish_rate": 0.974, "comp_len": 384.6, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 48.4, "frames": {"chat": 312}, "mem_gb": 9.81}
+{"step": 156, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.5997266300438593, "tokens": 120000, "cumulative_loss_tokens": 18720000, "grad_norm": 0.69921875, "lr": 3e-05, "finish_rate": 0.97, "comp_len": 363.6, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 51.5, "frames": {"chat": 330}, "mem_gb": 10.0}
+{"step": 157, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.5225513759051139, "tokens": 120000, "cumulative_loss_tokens": 18840000, "grad_norm": 0.6796875, "lr": 3e-05, "finish_rate": 0.954, "comp_len": 367.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 52.3, "frames": {"chat": 327}, "mem_gb": 10.0}
+{"step": 158, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.6792106815474729, "tokens": 120000, "cumulative_loss_tokens": 18960000, "grad_norm": 0.734375, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 341.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 53.4, "frames": {"chat": 351}, "mem_gb": 9.66}
+{"step": 159, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.4116748679873844, "tokens": 120000, "cumulative_loss_tokens": 19080000, "grad_norm": 0.56640625, "lr": 3e-05, "finish_rate": 0.973, "comp_len": 357.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 48.9, "frames": {"chat": 336}, "mem_gb": 9.71}
+{"step": 160, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.7196924281466132, "tokens": 120000, "cumulative_loss_tokens": 19200000, "grad_norm": 0.79296875, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 332.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 57.5, "frames": {"chat": 361}, "mem_gb": 9.68}
+[eval step 160] sample: 'To compute the rank of a matrix, we need to determine the maximum number of linearly independent rows or columns in the matrix. \n\nGiven the matrix:\n\\[\n\\begin{bmatrix}\n12 & -16 & 4 & 16 \\\\\n-9 & 11 & -1'
+{"step": 161, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.7385822186128547, "tokens": 120000, "cumulative_loss_tokens": 19320000, "grad_norm": 0.75390625, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 328.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 57.3, "frames": {"chat": 365}, "mem_gb": 9.63}
+{"step": 162, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.6650009809102863, "tokens": 120000, "cumulative_loss_tokens": 19440000, "grad_norm": 0.72265625, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 364.7, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 47.8, "frames": {"chat": 329}, "mem_gb": 9.68}
+{"step": 163, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.4450254848840957, "tokens": 120000, "cumulative_loss_tokens": 19560000, "grad_norm": 0.58203125, "lr": 3e-05, "finish_rate": 0.931, "comp_len": 416.7, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.4, "frames": {"chat": 288}, "mem_gb": 9.89}
+{"step": 164, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.3387074120278781, "tokens": 120000, "cumulative_loss_tokens": 19680000, "grad_norm": 0.59765625, "lr": 3e-05, "finish_rate": 0.868, "comp_len": 451.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.9, "frames": {"chat": 266}, "mem_gb": 10.04}
+{"step": 165, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.622777245139579, "tokens": 120000, "cumulative_loss_tokens": 19800000, "grad_norm": 0.671875, "lr": 3e-05, "finish_rate": 0.988, "comp_len": 372.7, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 48.9, "frames": {"chat": 322}, "mem_gb": 9.96}
+{"step": 166, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.6013612735873709, "tokens": 120000, "cumulative_loss_tokens": 19920000, "grad_norm": 0.6875, "lr": 3e-05, "finish_rate": 0.989, "comp_len": 334.3, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 54.0, "frames": {"chat": 359}, "mem_gb": 9.66}
+{"step": 167, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.5200983359214539, "tokens": 120000, "cumulative_loss_tokens": 20040000, "grad_norm": 0.61328125, "lr": 3e-05, "finish_rate": 0.965, "comp_len": 385.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 51.3, "frames": {"chat": 311}, "mem_gb": 9.78}
+{"step": 168, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.4737990202021475, "tokens": 120000, "cumulative_loss_tokens": 20160000, "grad_norm": 0.60546875, "lr": 3e-05, "finish_rate": 0.95, "comp_len": 396.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 47.8, "frames": {"chat": 303}, "mem_gb": 9.84}
+{"step": 169, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.7251967468782018, "tokens": 120000, "cumulative_loss_tokens": 20280000, "grad_norm": 0.78125, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 351.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 55.9, "frames": {"chat": 341}, "mem_gb": 9.85}
+{"step": 170, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.18425790825231622, "tokens": 120000, "cumulative_loss_tokens": 20400000, "grad_norm": 0.45703125, "lr": 3e-05, "finish_rate": 0.843, "comp_len": 524.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 39.5, "frames": {"chat": 229}, "mem_gb": 9.91}
+[eval step 170] sample: "To compute the rank of a matrix, we need to determine the maximum number of linearly independent rows or columns in the matrix. \n\nLet's start by writing the given 4x4 matrix:\n\n\\[\n\\begin{bmatrix}\n12 & "
+{"step": 171, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.4877417909882963, "tokens": 120000, "cumulative_loss_tokens": 20520000, "grad_norm": 0.62109375, "lr": 3e-05, "finish_rate": 0.964, "comp_len": 389.6, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 47.5, "frames": {"chat": 308}, "mem_gb": 10.06}
+{"step": 172, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.6497918719161302, "tokens": 120000, "cumulative_loss_tokens": 20640000, "grad_norm": 0.734375, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 331.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 54.7, "frames": {"chat": 362}, "mem_gb": 9.56}
+{"step": 173, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.6920267764383927, "tokens": 120000, "cumulative_loss_tokens": 20760000, "grad_norm": 0.7265625, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 315.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 58.4, "frames": {"chat": 380}, "mem_gb": 9.81}
+{"step": 174, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.6489797173403203, "tokens": 120000, "cumulative_loss_tokens": 20880000, "grad_norm": 0.7578125, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 336.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 59.3, "frames": {"chat": 357}, "mem_gb": 9.59}
+{"step": 175, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.3671841130634149, "tokens": 120000, "cumulative_loss_tokens": 21000000, "grad_norm": 0.6015625, "lr": 3e-05, "finish_rate": 0.835, "comp_len": 472.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 41.8, "frames": {"chat": 254}, "mem_gb": 10.06}
+{"step": 176, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.3038933877159531, "tokens": 120000, "cumulative_loss_tokens": 21120000, "grad_norm": 0.5703125, "lr": 3e-05, "finish_rate": 0.874, "comp_len": 459.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 41.3, "frames": {"chat": 261}, "mem_gb": 10.03}
+{"step": 177, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.6068330939687168, "tokens": 120000, "cumulative_loss_tokens": 21240000, "grad_norm": 0.7109375, "lr": 3e-05, "finish_rate": 0.982, "comp_len": 431.7, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.2, "frames": {"chat": 278}, "mem_gb": 9.82}
+{"step": 178, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.6536359203963541, "tokens": 120000, "cumulative_loss_tokens": 21360000, "grad_norm": 0.67578125, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 360.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 51.0, "frames": {"chat": 333}, "mem_gb": 9.96}
+{"step": 179, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.3293319945723439, "tokens": 120000, "cumulative_loss_tokens": 21480000, "grad_norm": 0.53515625, "lr": 3e-05, "finish_rate": 0.888, "comp_len": 480.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 42.9, "frames": {"chat": 250}, "mem_gb": 9.92}
+{"step": 180, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.7798017734301587, "tokens": 120000, "cumulative_loss_tokens": 21600000, "grad_norm": 0.8671875, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 336.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 54.4, "frames": {"chat": 357}, "mem_gb": 9.7}
+[eval step 180] sample: 'To compute the rank of a matrix, we need to determine the maximum number of linearly independent rows or columns in the matrix. \n\nGiven the matrix:\n\\[\n\\begin{bmatrix}\n12 & -16 & 4 & 16 \\\\\n-9 & 11 & -1'
+{"step": 181, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.8047170679829704, "tokens": 120000, "cumulative_loss_tokens": 21720000, "grad_norm": 0.7421875, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 363.6, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 47.5, "frames": {"chat": 330}, "mem_gb": 9.67}
+{"step": 182, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.7163969376386454, "tokens": 120000, "cumulative_loss_tokens": 21840000, "grad_norm": 0.76171875, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 341.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 53.8, "frames": {"chat": 351}, "mem_gb": 9.71}
+{"step": 183, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.4812941745320335, "tokens": 120000, "cumulative_loss_tokens": 21960000, "grad_norm": 0.6484375, "lr": 3e-05, "finish_rate": 0.959, "comp_len": 413.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 46.0, "frames": {"chat": 290}, "mem_gb": 9.84}
+{"step": 184, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.567710873114597, "tokens": 120000, "cumulative_loss_tokens": 22080000, "grad_norm": 0.671875, "lr": 3e-05, "finish_rate": 0.969, "comp_len": 372.7, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 49.5, "frames": {"chat": 322}, "mem_gb": 9.78}
+{"step": 185, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.32963835290049515, "tokens": 120000, "cumulative_loss_tokens": 22200000, "grad_norm": 0.58203125, "lr": 3e-05, "finish_rate": 0.827, "comp_len": 451.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.1, "frames": {"chat": 266}, "mem_gb": 10.01}
+{"step": 186, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.6460849656254053, "tokens": 120000, "cumulative_loss_tokens": 22320000, "grad_norm": 0.92578125, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 362.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 49.8, "frames": {"chat": 331}, "mem_gb": 9.86}
+{"step": 187, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.5878346645558874, "tokens": 120000, "cumulative_loss_tokens": 22440000, "grad_norm": 0.74609375, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 350.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 51.1, "frames": {"chat": 342}, "mem_gb": 9.62}
+{"step": 188, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.7007336048837751, "tokens": 120000, "cumulative_loss_tokens": 22560000, "grad_norm": 0.73046875, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 335.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 51.9, "frames": {"chat": 358}, "mem_gb": 9.66}
+{"step": 189, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.5796603640264522, "tokens": 120000, "cumulative_loss_tokens": 22680000, "grad_norm": 0.7890625, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 348.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 54.6, "frames": {"chat": 344}, "mem_gb": 9.75}
+{"step": 190, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.37503966700950014, "tokens": 120000, "cumulative_loss_tokens": 22800000, "grad_norm": 0.640625, "lr": 3e-05, "finish_rate": 0.861, "comp_len": 449.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.4, "frames": {"chat": 267}, "mem_gb": 10.06}
+[eval step 190] sample: 'To find the rank of a matrix, we need to determine the maximum number of linearly independent rows or columns in the matrix. A matrix is said to be of rank \\( r \\) if it has \\( r \\) linearly independe'
+{"step": 191, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.16379121993631124, "tokens": 120000, "cumulative_loss_tokens": 22920000, "grad_norm": 0.46875, "lr": 3e-05, "finish_rate": 0.835, "comp_len": 483.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 37.6, "frames": {"chat": 248}, "mem_gb": 9.89}
+{"step": 192, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.5387236011125147, "tokens": 120000, "cumulative_loss_tokens": 23040000, "grad_norm": 0.62890625, "lr": 3e-05, "finish_rate": 0.979, "comp_len": 364.7, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 51.5, "frames": {"chat": 329}, "mem_gb": 9.89}
+{"step": 193, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.5913820276208221, "tokens": 120000, "cumulative_loss_tokens": 23160000, "grad_norm": 0.640625, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 354.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 57.5, "frames": {"chat": 339}, "mem_gb": 9.85}
+{"step": 194, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.7116979550040637, "tokens": 120000, "cumulative_loss_tokens": 23280000, "grad_norm": 0.7265625, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 339.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 57.4, "frames": {"chat": 354}, "mem_gb": 9.84}
+{"step": 195, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.3713194830263654, "tokens": 120000, "cumulative_loss_tokens": 23400000, "grad_norm": 0.58203125, "lr": 3e-05, "finish_rate": 0.943, "comp_len": 430.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 46.1, "frames": {"chat": 279}, "mem_gb": 9.98}
+{"step": 196, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.4681690845252946, "tokens": 120000, "cumulative_loss_tokens": 23520000, "grad_norm": 0.62109375, "lr": 3e-05, "finish_rate": 0.959, "comp_len": 377.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 53.0, "frames": {"chat": 318}, "mem_gb": 9.91}
+{"step": 197, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.3116183115092417, "tokens": 120000, "cumulative_loss_tokens": 23640000, "grad_norm": 0.54296875, "lr": 3e-05, "finish_rate": 0.875, "comp_len": 452.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.0, "frames": {"chat": 265}, "mem_gb": 9.99}
+{"step": 198, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.2913039409104114, "tokens": 120000, "cumulative_loss_tokens": 23760000, "grad_norm": 0.4921875, "lr": 3e-05, "finish_rate": 0.85, "comp_len": 472.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 40.9, "frames": {"chat": 254}, "mem_gb": 9.94}
+{"step": 199, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.4605057141344063, "tokens": 120000, "cumulative_loss_tokens": 23880000, "grad_norm": 0.6015625, "lr": 3e-05, "finish_rate": 0.962, "comp_len": 381.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 49.6, "frames": {"chat": 315}, "mem_gb": 9.97}
+{"step": 200, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.621974576252761, "tokens": 120000, "cumulative_loss_tokens": 24000000, "grad_norm": 0.62890625, "lr": 3e-05, "finish_rate": 0.977, "comp_len": 400.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.9, "frames": {"chat": 300}, "mem_gb": 9.76}
+[eval step 200] sample: 'To determine the rank of a matrix, we need to find the maximum number of linearly independent rows or columns. A matrix is said to be of rank \\( r \\) if it has \\( r \\) linearly independent rows or col'
+checkpoint snapshot queued -> outputs/healed/calib/glean_keep25_s1224/step0200

healed/calib/glean_keep25_s1224/wandb/run-20260717_210858-ypxqb0r8/logs/debug-core.log

@@ -0,0 +1,20 @@
+{"time":"2026-07-17T21:08:58.389031686-07:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmppr5y76l8/port-399817.txt","pid":399817,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-07-17T21:08:58.389833134-07:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":399817}
+{"time":"2026-07-17T21:08:58.389779872-07:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-399817-399924-1676098820/socket","Net":"unix"}}
+{"time":"2026-07-17T21:08:58.571304515-07:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-07-17T21:08:58.720383456-07:00","level":"INFO","msg":"handleInformInit: received","streamId":"ypxqb0r8","id":"1(@)"}
+{"time":"2026-07-17T21:08:58.932318492-07:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"ypxqb0r8","id":"1(@)"}
+{"time":"2026-07-17T21:09:04.264002367-07:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"s0qspwiwts5x"}
+{"time":"2026-07-17T22:31:39.159619829-07:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"s0qspwiwts5x"}
+{"time":"2026-07-17T22:31:39.911767273-07:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"s0qspwiwts5x"}
+{"time":"2026-07-17T22:31:39.915088596-07:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"ypxqb0r8","id":"1(@)"}
+{"time":"2026-07-17T22:31:39.920683976-07:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"ypxqb0r8","id":"1(@)"}
+{"time":"2026-07-17T22:31:40.552238826-07:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-07-17T22:31:40.552291018-07:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-07-17T22:31:40.552305578-07:00","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-07-17T22:31:40.552345899-07:00","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-07-17T22:31:40.55236888-07:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-07-17T22:31:40.552388721-07:00","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"}
+{"time":"2026-07-17T22:31:40.552401381-07:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-07-17T22:31:40.552474414-07:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-399817-399924-1676098820/socket","Net":"unix"}}
+{"time":"2026-07-17T22:31:40.552520895-07:00","level":"INFO","msg":"server is closed"}

healed/calib/glean_keep25_s1224/wandb/run-20260717_210858-ypxqb0r8/logs/debug.log

@@ -0,0 +1,24 @@
+2026-07-17 21:08:58,718 INFO    MainThread:399817 [wandb_setup.py:_flush():81] Current SDK version is 0.28.0
+2026-07-17 21:08:58,718 INFO    MainThread:399817 [wandb_setup.py:_flush():81] Configure stats pid to 399817
+2026-07-17 21:08:58,718 INFO    MainThread:399817 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-07-17 21:08:58,718 INFO    MainThread:399817 [wandb_init.py:setup_run_log_directory():725] Logging user logs to outputs/healed/calib/glean_keep25_s1224/wandb/run-20260717_210858-ypxqb0r8/logs/debug.log
+2026-07-17 21:08:58,718 INFO    MainThread:399817 [wandb_init.py:setup_run_log_directory():726] Logging internal logs to outputs/healed/calib/glean_keep25_s1224/wandb/run-20260717_210858-ypxqb0r8/logs/debug-internal.log
+2026-07-17 21:08:58,718 INFO    MainThread:399817 [wandb_init.py:init():768] calling init triggers
+2026-07-17 21:08:58,718 INFO    MainThread:399817 [wandb_init.py:init():773] wandb.init called with sweep_config: {}
+config: {'student': 'outputs/pruned/glean-0125inst-general-keep25', 'teacher': 'allenai/OLMoE-1B-7B-0125-Instruct', 'training_mode': 'off-policy', 'kl_direction': 'forward', 'dataset': 'allenai/RLVR-MATH', 'dataset_sources': None, 'max_difficulty': None, 'trajectories': 'outputs/teacher_trajectories/dolci_math_curated.jsonl', 'trajectory_dataset': 'allenai/Dolci-Instruct-RL', 'off_policy_frames': 'chat', 'off_policy_max_seq_len': 2048, 'topk_targets': 'outputs/teacher_trajectories/dolci_combined_top128', 'max_loss_tokens': None, 'loss_tokens_per_step': 120000, 'teacher_device': 'cuda:0', 'student_device': 'cuda:0', 'lr': 3e-05, 'optimizer': 'adamw8bit', 'weight_decay': 0.1, 'epochs': 9, 'prompts_per_step': 256, 'group_size': 1, 'rollout_batch': 64, 'micro_batch': 3, 'max_new_tokens': 256, 'max_prompt_len': 1024, 'warmup_steps': 10, 'max_grad_norm': 1.0, 'eval_every': 10, 'gsm8k_every': 0, 'gsm8k_n': 256, 'gsm8k_batch': 16, 'gsm8k_max_new_tokens': 512, 'gsm8k_frames': 'chat', 'save_every': 200, 'out_dir': 'outputs/healed/calib/glean_keep25_s1224', 'sweep': 200, 'wandb': True, 'wandb_project': 'glean-calib', 'wandb_run_name': 'calib_keep25_200', 'wandb_run_id': None, 'wandb_resume': None, 'wandb_mode': 'online', 'no_wandb_sync': False, 'debug': False, 'resume_from': 'outputs/healed/calib/glean_keep25_s1224/step0100', 'start_step': 0, 'no_grad_checkpointing': False, 'seed': 1224, 'no_teacher_overlap': False, 'sync_checkpoints': False, 'rollout_engine': 'hf', 'vllm_gpu': None, 'vllm_port': 8377, 'vllm_refresh_every': 5, 'vllm_serve_bin': 'vllm-plugin/.venv/bin/python', 'vllm_gpu_mem_util': 0.85, 'vllm_refresh_mode': 'reload', 'vllm_live_dir': None, 'resolved_kl_direction': 'forward', '_wandb': {}}
+2026-07-17 21:08:58,718 INFO    MainThread:399817 [wandb_init.py:init():816] starting backend
+2026-07-17 21:08:58,718 INFO    MainThread:399817 [wandb_init.py:init():831] sending inform_init request
+2026-07-17 21:08:58,933 INFO    MainThread:399817 [wandb_init.py:init():836] backend started and connected
+2026-07-17 21:08:58,935 INFO    MainThread:399817 [wandb_init.py:init():906] updated telemetry
+2026-07-17 21:08:58,941 INFO    MainThread:399817 [wandb_init.py:init():929] communicating run to backend with 90.0 second timeout
+2026-07-17 21:08:59,120 INFO    MainThread:399817 [wandb_init.py:init():974] starting run threads in backend
+2026-07-17 21:08:59,254 INFO    MainThread:399817 [wandb_run.py:_console_start():2523] atexit reg
+2026-07-17 21:08:59,254 INFO    MainThread:399817 [wandb_run.py:_redirect():2373] redirect: wrap_raw
+2026-07-17 21:08:59,254 INFO    MainThread:399817 [wandb_run.py:_redirect():2442] Wrapping output streams.
+2026-07-17 21:08:59,254 INFO    MainThread:399817 [wandb_run.py:_redirect():2465] Redirects installed.
+2026-07-17 21:08:59,256 INFO    MainThread:399817 [wandb_init.py:init():1012] run started, returning control to user process
+2026-07-17 22:31:39,157 INFO    MainThread:399817 [wandb_run.py:_finish():2285] finishing run hbfreed/glean-calib/ypxqb0r8
+2026-07-17 22:31:39,158 INFO    MainThread:399817 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0
+2026-07-17 22:31:39,159 INFO    MainThread:399817 [wandb_run.py:_restore():2472] restore
+2026-07-17 22:31:39,159 INFO    MainThread:399817 [wandb_run.py:_restore():2478] restore done
+2026-07-17 22:31:39,914 INFO    MainThread:399817 [wandb_run.py:_footer_sync_info():3895] logging synced files

healed/calib/glean_keep25_s1224/wandb/run-20260717_235739-5x8mg4as/files/output.log

@@ -0,0 +1,59 @@
+resumed student weights from outputs/healed/calib/glean_keep25_s1224/step0300 (fresh optimizer, step counter at 0)
+58360 cached top-128 chat trajectories / 22,295,631 unique tokens | 185 steps/epoch | 400 total steps | student params 2.09B | teacher overlap=False
+restored optimizer/scheduler state from step 300; rebuilt 228 paged buffers
+{"step": 301, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.351279286532104, "tokens": 120000, "cumulative_loss_tokens": 36120000, "grad_norm": 0.5390625, "lr": 3e-05, "finish_rate": 0.961, "comp_len": 394.7, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.0, "frames": {"chat": 304}, "mem_gb": 9.71}
+{"step": 302, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.5407799015927439, "tokens": 120000, "cumulative_loss_tokens": 36240000, "grad_norm": 0.6484375, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 321.7, "t_data_s": 0.1, "t_rollout_s": 0.0, "t_step_s": 51.3, "frames": {"chat": 373}, "mem_gb": 9.7}
+{"step": 303, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.3325376754036794, "tokens": 120000, "cumulative_loss_tokens": 36360000, "grad_norm": 0.53125, "lr": 3e-05, "finish_rate": 0.87, "comp_len": 400.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 46.5, "frames": {"chat": 300}, "mem_gb": 9.98}
+{"step": 304, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.5965203912614534, "tokens": 120000, "cumulative_loss_tokens": 36480000, "grad_norm": 0.6484375, "lr": 3e-05, "finish_rate": 0.982, "comp_len": 362.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 51.0, "frames": {"chat": 331}, "mem_gb": 9.82}
+{"step": 305, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.27112077200754237, "tokens": 120000, "cumulative_loss_tokens": 36600000, "grad_norm": 0.458984375, "lr": 3e-05, "finish_rate": 0.876, "comp_len": 463.3, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 41.4, "frames": {"chat": 259}, "mem_gb": 9.96}
+{"step": 306, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.535515980368418, "tokens": 120000, "cumulative_loss_tokens": 36720000, "grad_norm": 0.640625, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 335.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 53.3, "frames": {"chat": 358}, "mem_gb": 9.58}
+{"step": 307, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.6173073474912594, "tokens": 120000, "cumulative_loss_tokens": 36840000, "grad_norm": 0.66015625, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 339.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 53.5, "frames": {"chat": 354}, "mem_gb": 9.84}
+{"step": 308, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.30061435020910576, "tokens": 120000, "cumulative_loss_tokens": 36960000, "grad_norm": 0.498046875, "lr": 3e-05, "finish_rate": 0.939, "comp_len": 430.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 47.0, "frames": {"chat": 279}, "mem_gb": 9.9}
+{"step": 309, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.622384964984407, "tokens": 120000, "cumulative_loss_tokens": 37080000, "grad_norm": 0.671875, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 339.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 50.9, "frames": {"chat": 354}, "mem_gb": 9.65}
+{"step": 310, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.597136234145984, "tokens": 120000, "cumulative_loss_tokens": 37200000, "grad_norm": 0.6640625, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 359.3, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 51.4, "frames": {"chat": 334}, "mem_gb": 9.86}
+The attention mask is not set and cannot be inferred from input because pad token is same as eos token. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.
+[eval step 310] sample: "To find the rank of a matrix, we need to determine the maximum number of linearly independent rows or columns in the matrix. \n\nLet's represent the given matrix as follows:\n\n\\[\nA = \\begin{bmatrix}\n12 &"
+{"step": 311, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.5931549054012324, "tokens": 120000, "cumulative_loss_tokens": 37320000, "grad_norm": 0.63671875, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 367.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 49.2, "frames": {"chat": 327}, "mem_gb": 9.72}
+{"step": 312, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.5639030251913083, "tokens": 120000, "cumulative_loss_tokens": 37440000, "grad_norm": 0.6328125, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 328.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 51.4, "frames": {"chat": 365}, "mem_gb": 9.82}
+{"step": 313, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.31766464765102914, "tokens": 120000, "cumulative_loss_tokens": 37560000, "grad_norm": 0.5390625, "lr": 3e-05, "finish_rate": 0.869, "comp_len": 476.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.3, "frames": {"chat": 252}, "mem_gb": 10.03}
+{"step": 314, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.25770912591886397, "tokens": 120000, "cumulative_loss_tokens": 37680000, "grad_norm": 0.515625, "lr": 3e-05, "finish_rate": 0.867, "comp_len": 468.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 41.3, "frames": {"chat": 256}, "mem_gb": 9.99}
+{"step": 315, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.3189014506495868, "tokens": 120000, "cumulative_loss_tokens": 37800000, "grad_norm": 0.498046875, "lr": 3e-05, "finish_rate": 0.964, "comp_len": 397.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 43.1, "frames": {"chat": 302}, "mem_gb": 9.77}
+{"step": 316, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.5332931566566229, "tokens": 120000, "cumulative_loss_tokens": 37920000, "grad_norm": 0.65234375, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 342.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 54.0, "frames": {"chat": 350}, "mem_gb": 9.67}
+{"step": 317, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.3288269461247449, "tokens": 120000, "cumulative_loss_tokens": 38040000, "grad_norm": 0.515625, "lr": 3e-05, "finish_rate": 0.959, "comp_len": 411.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.9, "frames": {"chat": 292}, "mem_gb": 9.79}
+{"step": 318, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.5442317285993757, "tokens": 120000, "cumulative_loss_tokens": 38160000, "grad_norm": 0.62109375, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 337.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 52.7, "frames": {"chat": 356}, "mem_gb": 9.6}
+{"step": 319, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.383545723837552, "tokens": 120000, "cumulative_loss_tokens": 38280000, "grad_norm": 0.56640625, "lr": 3e-05, "finish_rate": 0.977, "comp_len": 345.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 49.0, "frames": {"chat": 347}, "mem_gb": 9.77}
+{"step": 320, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.3346157630101157, "tokens": 120000, "cumulative_loss_tokens": 38400000, "grad_norm": 0.52734375, "lr": 3e-05, "finish_rate": 0.976, "comp_len": 363.6, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 48.6, "frames": {"chat": 330}, "mem_gb": 9.96}
+[eval step 320] sample: "To find the rank of a matrix, we need to determine the maximum number of linearly independent rows or columns in the matrix. \n\nLet's represent the given matrix as follows:\n\n\\[\nA = \\begin{bmatrix}\n12 &"
+{"step": 321, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.5644299145538981, "tokens": 120000, "cumulative_loss_tokens": 38520000, "grad_norm": 0.6171875, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 330.6, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 54.5, "frames": {"chat": 363}, "mem_gb": 9.82}
+{"step": 322, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.5168023564506322, "tokens": 120000, "cumulative_loss_tokens": 38640000, "grad_norm": 0.625, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 339.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 56.9, "frames": {"chat": 354}, "mem_gb": 9.68}
+{"step": 323, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.2756425909321134, "tokens": 120000, "cumulative_loss_tokens": 38760000, "grad_norm": 0.490234375, "lr": 3e-05, "finish_rate": 0.93, "comp_len": 444.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 45.6, "frames": {"chat": 270}, "mem_gb": 9.95}
+{"step": 324, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.5347250625066459, "tokens": 120000, "cumulative_loss_tokens": 38880000, "grad_norm": 0.60546875, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 349.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 52.5, "frames": {"chat": 343}, "mem_gb": 9.72}
+{"step": 325, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.13697400885016348, "tokens": 120000, "cumulative_loss_tokens": 39000000, "grad_norm": 0.41796875, "lr": 3e-05, "finish_rate": 0.845, "comp_len": 517.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 37.1, "frames": {"chat": 232}, "mem_gb": 9.91}
+{"step": 326, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.5018270760613183, "tokens": 120000, "cumulative_loss_tokens": 39120000, "grad_norm": 0.62109375, "lr": 3e-05, "finish_rate": 0.98, "comp_len": 346.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 50.2, "frames": {"chat": 346}, "mem_gb": 9.86}
+{"step": 327, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.39994397104233503, "tokens": 120000, "cumulative_loss_tokens": 39240000, "grad_norm": 0.5625, "lr": 3e-05, "finish_rate": 0.951, "comp_len": 392.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.3, "frames": {"chat": 306}, "mem_gb": 9.95}
+{"step": 328, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.26500454536170387, "tokens": 120000, "cumulative_loss_tokens": 39360000, "grad_norm": 0.44140625, "lr": 3e-05, "finish_rate": 0.923, "comp_len": 439.6, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 40.3, "frames": {"chat": 273}, "mem_gb": 9.99}
+{"step": 329, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.2892410087935937, "tokens": 120000, "cumulative_loss_tokens": 39480000, "grad_norm": 0.486328125, "lr": 3e-05, "finish_rate": 0.959, "comp_len": 451.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 39.8, "frames": {"chat": 266}, "mem_gb": 9.82}
+{"step": 330, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.4476620569754392, "tokens": 120000, "cumulative_loss_tokens": 39600000, "grad_norm": 0.59765625, "lr": 3e-05, "finish_rate": 0.936, "comp_len": 401.3, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.1, "frames": {"chat": 299}, "mem_gb": 10.01}
+[eval step 330] sample: 'To compute the rank of a matrix, we need to determine the maximum number of linearly independent rows or columns in the matrix. \n\nGiven the matrix:\n\\[\n\\begin{bmatrix}\n12 & -16 & 4 & 16 \\\\\n-9 & 11 & -1'
+{"step": 331, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.30612562722607206, "tokens": 120000, "cumulative_loss_tokens": 39720000, "grad_norm": 0.5078125, "lr": 3e-05, "finish_rate": 0.88, "comp_len": 463.3, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 41.2, "frames": {"chat": 259}, "mem_gb": 10.1}
+{"step": 332, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.3648861061472135, "tokens": 120000, "cumulative_loss_tokens": 39840000, "grad_norm": 0.578125, "lr": 3e-05, "finish_rate": 0.966, "comp_len": 369.2, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 46.3, "frames": {"chat": 325}, "mem_gb": 9.83}
+{"step": 333, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.35644260462733607, "tokens": 120000, "cumulative_loss_tokens": 39960000, "grad_norm": 0.59765625, "lr": 3e-05, "finish_rate": 0.923, "comp_len": 442.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 42.0, "frames": {"chat": 271}, "mem_gb": 9.91}
+{"step": 334, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.5318697121323397, "tokens": 120000, "cumulative_loss_tokens": 40080000, "grad_norm": 0.6484375, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 334.3, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 49.8, "frames": {"chat": 359}, "mem_gb": 9.69}
+{"step": 335, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.5110903365676291, "tokens": 120000, "cumulative_loss_tokens": 40200000, "grad_norm": 0.59765625, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 347.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 51.1, "frames": {"chat": 345}, "mem_gb": 9.75}
+{"step": 336, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.5708266700956971, "tokens": 120000, "cumulative_loss_tokens": 40320000, "grad_norm": 0.6640625, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 301.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 52.4, "frames": {"chat": 398}, "mem_gb": 9.78}
+{"step": 337, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.3323912446099644, "tokens": 120000, "cumulative_loss_tokens": 40440000, "grad_norm": 0.5546875, "lr": 3e-05, "finish_rate": 0.964, "comp_len": 390.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.0, "frames": {"chat": 307}, "mem_gb": 9.98}
+{"step": 338, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.5195394132010949, "tokens": 120000, "cumulative_loss_tokens": 40560000, "grad_norm": 0.66015625, "lr": 3e-05, "finish_rate": 0.995, "comp_len": 308.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 57.0, "frames": {"chat": 389}, "mem_gb": 9.75}
+{"step": 339, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.10721138076403489, "tokens": 120000, "cumulative_loss_tokens": 40680000, "grad_norm": 0.390625, "lr": 3e-05, "finish_rate": 0.823, "comp_len": 519.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 37.6, "frames": {"chat": 231}, "mem_gb": 9.99}
+{"step": 340, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.5100581502144536, "tokens": 120000, "cumulative_loss_tokens": 40800000, "grad_norm": 0.6484375, "lr": 3e-05, "finish_rate": 0.988, "comp_len": 373.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 48.5, "frames": {"chat": 321}, "mem_gb": 9.66}
+[eval step 340] sample: 'To find the rank of a matrix, we need to determine the maximum number of linearly independent rows or columns. A matrix is said to be of rank \\( r \\) if it has \\( r \\) linearly independent rows or col'
+{"step": 341, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.5770065285284072, "tokens": 120000, "cumulative_loss_tokens": 40920000, "grad_norm": 0.703125, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 357.1, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 51.0, "frames": {"chat": 336}, "mem_gb": 9.69}
+{"step": 342, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.5449590804694531, "tokens": 120000, "cumulative_loss_tokens": 41040000, "grad_norm": 0.62890625, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 355.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 50.0, "frames": {"chat": 338}, "mem_gb": 9.84}
+{"step": 343, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.3079538588923402, "tokens": 120000, "cumulative_loss_tokens": 41160000, "grad_norm": 0.462890625, "lr": 3e-05, "finish_rate": 0.945, "comp_len": 436.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 44.4, "frames": {"chat": 275}, "mem_gb": 9.98}
+{"step": 344, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.21514355221434187, "tokens": 120000, "cumulative_loss_tokens": 41280000, "grad_norm": 0.45703125, "lr": 3e-05, "finish_rate": 0.801, "comp_len": 531.0, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 41.4, "frames": {"chat": 226}, "mem_gb": 10.01}
+{"step": 345, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.5597503078967954, "tokens": 120000, "cumulative_loss_tokens": 41400000, "grad_norm": 0.63671875, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 346.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 57.1, "frames": {"chat": 346}, "mem_gb": 9.8}
+{"step": 346, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.5308711614095917, "tokens": 120000, "cumulative_loss_tokens": 41520000, "grad_norm": 0.6015625, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 330.6, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 54.2, "frames": {"chat": 363}, "mem_gb": 9.61}
+{"step": 347, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.30469400957108783, "tokens": 120000, "cumulative_loss_tokens": 41640000, "grad_norm": 0.482421875, "lr": 3e-05, "finish_rate": 0.955, "comp_len": 383.4, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 47.4, "frames": {"chat": 313}, "mem_gb": 9.72}
+{"step": 348, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.5788642878273502, "tokens": 120000, "cumulative_loss_tokens": 41760000, "grad_norm": 0.61328125, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 323.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 52.6, "frames": {"chat": 371}, "mem_gb": 9.77}
+{"step": 349, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.43604542372375726, "tokens": 120000, "cumulative_loss_tokens": 41880000, "grad_norm": 0.56640625, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 373.8, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 49.0, "frames": {"chat": 321}, "mem_gb": 9.55}
+{"step": 350, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.5381324614154795, "tokens": 120000, "cumulative_loss_tokens": 42000000, "grad_norm": 0.6484375, "lr": 3e-05, "finish_rate": 0.997, "comp_len": 365.9, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 54.2, "frames": {"chat": 328}, "mem_gb": 9.64}
+[eval step 350] sample: "To find the rank of a matrix, we need to determine the maximum number of linearly independent rows or columns in the matrix. \n\nLet's represent the given matrix as follows:\n\n\\[\nA = \\begin{bmatrix}\n12 &"

healed/calib/glean_keep25_s1224/wandb/run-20260717_235739-5x8mg4as/files/requirements.txt

@@ -0,0 +1,129 @@
+certifi==2026.6.17
+fonttools==4.63.0
+urllib3==2.7.0
+requests==2.34.2
+nvidia-cublas-cu12==12.6.4.1
+packaging==26.2
+regex==2026.6.28
+portalocker==3.2.0
+safetensors==0.8.0
+datasets==5.0.0
+sqlitedict==2.1.0
+narwhals==2.23.0
+latex2sympy2_extended==1.11.0
+smmap==5.0.3
+colorama==0.4.6
+anyio==4.14.1
+cycler==0.12.1
+pytablewriter==1.2.1
+pydantic_core==2.46.4
+huggingface_hub==0.36.2
+charset-normalizer==3.4.8
+nvidia-cusolver-cu12==11.7.1.2
+joblib==1.5.3
+typing-inspection==0.4.2
+wandb==0.28.0
+bitsandbytes==0.49.2
+kiwisolver==1.5.0
+cuda-toolkit==12.6.3
+propcache==0.5.2
+Jinja2==3.1.6
+cuda-bindings==12.9.7
+pluggy==1.6.0
+pytest==9.1.1
+fsspec==2026.4.0
+tqdm==4.68.3
+psutil==7.2.2
+pydantic==2.13.4
+typepy==1.3.5
+xxhash==3.8.1
+DataProperty==1.1.1
+aiosignal==1.4.0
+annotated-types==0.7.0
+Pygments==2.20.0
+zstandard==0.25.0
+threadpoolctl==3.6.0
+nvidia-cudnn-cu12==9.10.2.21
+evaluate==0.4.6
+pillow==12.3.0
+GitPython==3.1.50
+httpx==0.28.1
+lm_eval==0.4.12
+pytz==2026.2
+mbstrdecoder==1.1.5
+immutabledict==4.3.1
+dill==0.4.1
+mpmath==1.3.0
+yarl==1.24.2
+word2number==1.1
+networkx==3.6.1
+einops==0.8.2
+tabulate==0.10.0
+pathvalidate==3.3.1
+tenacity==9.1.4
+sentry-sdk==2.64.0
+nvidia-nvshmem-cu12==3.4.5
+hf-xet==1.5.1
+six==1.17.0
+pyarrow==24.0.0
+nvidia-curand-cu12==10.3.7.77
+cuda-pathfinder==1.5.6
+sacrebleu==2.6.0
+triton==3.7.1
+frozenlist==1.8.0
+antlr4-python3-runtime==4.11.0
+tabledata==1.3.5
+h11==0.16.0
+nvidia-cusparselt-cu12==0.7.1
+nvidia-nvjitlink-cu12==12.6.85
+scipy==1.18.0
+rouge_score==0.1.2
+transformers==4.57.6
+multiprocess==0.70.19
+absl-py==2.5.0
+accelerate==1.14.0
+click==8.4.2
+lxml==6.1.1
+defusedxml==0.7.1
+setuptools==81.0.0
+nvidia-cuda-cupti-cu12==12.6.80
+httpcore==1.0.9
+langdetect==1.0.9
+nltk==3.10.0
+iniconfig==2.3.0
+nvidia-cuda-runtime-cu12==12.6.77
+nvidia-cusparse-cu12==12.5.4.2
+matplotlib==3.11.0
+protobuf==7.35.1
+nvidia-cufft-cu12==11.3.0.4
+pyparsing==3.3.2
+chardet==6.0.0.post1
+typing_extensions==4.16.0
+stanford-stk==0.7.1
+aiohappyeyeballs==2.7.1
+MarkupSafe==3.0.3
+nvidia-cufile-cu12==1.11.1.6
+tcolorpy==0.1.7
+scikit-learn==1.9.0
+tokenizers==0.22.2
+attrs==26.1.0
+gitdb==4.0.12
+aiohttp==3.14.1
+glean==0.0.1
+platformdirs==4.10.0
+megablocks==0.11.0.dev0
+nvidia-cuda-nvrtc-cu12==12.6.85
+pandas==3.0.3
+multidict==6.7.1
+torch==2.12.1+cu126
+contourpy==1.3.3
+nvidia-nvtx-cu12==12.6.77
+math-verify==0.9.0
+filelock==3.29.5
+idna==3.18
+nvidia-nccl-cu12==2.29.3
+python-dateutil==2.9.0.post0
+more-itertools==11.1.0
+numpy==2.0.2
+sympy==1.14.0
+PyYAML==6.0.3

healed/calib/glean_keep25_s1224/wandb/run-20260717_235739-5x8mg4as/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os": "Linux-7.0.11-76070011-generic-x86_64-with-glibc2.35",
+  "python": "CPython 3.12.12",
+  "startedAt": "2026-07-18T06:57:39.078210Z",
+  "args": [
+    "--training-mode",
+    "off-policy",
+    "--kl-direction",
+    "forward",
+    "--topk-targets",
+    "outputs/teacher_trajectories/dolci_combined_top128",
+    "--off-policy-frames",
+    "chat",
+    "--student-device",
+    "cuda:0",
+    "--student",
+    "outputs/pruned/glean-0125inst-general-keep25",
+    "--resume-from",
+    "outputs/healed/calib/glean_keep25_s1224/step0300",
+    "--lr",
+    "3e-5",
+    "--optimizer",
+    "adamw8bit",
+    "--epochs",
+    "9",
+    "--sweep",
+    "400",
+    "--micro-batch",
+    "3",
+    "--loss-tokens-per-step",
+    "120000",
+    "--gsm8k-every",
+    "0",
+    "--save-every",
+    "400",
+    "--seed",
+    "1224",
+    "--out-dir",
+    "outputs/healed/calib/glean_keep25_s1224",
+    "--wandb",
+    "--wandb-mode",
+    "online",
+    "--wandb-project",
+    "glean-calib",
+    "--wandb-run-name",
+    "calib_keep25_400"
+  ],
+  "program": "/home/henry/Documents/PythonProjects/variable-reap/scripts/11_distill_on_policy.py",
+  "codePath": "scripts/11_distill_on_policy.py",
+  "codePathLocal": "scripts/11_distill_on_policy.py",
+  "git": {
+    "remote": "https://github.com/hbfreed/variable-reap.git",
+    "commit": "247c7f0113d7a058b1db8044deddfd4456cc998d"
+  },
+  "email": "hbfreed@protonmail.com",
+  "root": "outputs/healed/calib/glean_keep25_s1224",
+  "host": "pop-os",
+  "executable": "/home/henry/Documents/PythonProjects/variable-reap/.venv/bin/python3",
+  "cpu_count": 24,
+  "cpu_count_logical": 48,
+  "gpu": "NVIDIA GeForce RTX 3090",
+  "gpu_count": 3,
+  "disk": {
+    "/": {
+      "total": "1958315118592",
+      "used": "1668439638016"
+    }
+  },
+  "memory": {
+    "total": "134900756480"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA GeForce RTX 3090",
+      "memoryTotal": "25769803776",
+      "cudaCores": 10496,
+      "architecture": "Ampere",
+      "uuid": "GPU-8ca70870-ddf2-d274-bcc5-182c2075bced"
+    },
+    {
+      "name": "NVIDIA GeForce RTX 3090",
+      "memoryTotal": "25769803776",
+      "cudaCores": 10496,
+      "architecture": "Ampere",
+      "uuid": "GPU-a6acf07f-31f5-618f-a5d0-c0017e7e2e27"
+    },
+    {
+      "name": "NVIDIA GeForce RTX 3090",
+      "memoryTotal": "25769803776",
+      "cudaCores": 10496,
+      "architecture": "Ampere",
+      "uuid": "GPU-864c54df-0130-7780-e271-8a5551d1733f"
+    }
+  ],
+  "cudaVersion": "13.0",
+  "writerId": "840j1ww7y2xmcobmkrrzrcafjpy0v2cg"
+}

healed/calib/glean_keep25_s1224/wandb/run-20260717_235739-5x8mg4as/logs/debug-core.log

@@ -0,0 +1,7 @@
+{"time":"2026-07-17T23:57:38.746866474-07:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp_vhp_drk/port-417245.txt","pid":417245,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-07-17T23:57:38.749038876-07:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":417245}
+{"time":"2026-07-17T23:57:38.748934082-07:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-417245-417348-471081702/socket","Net":"unix"}}
+{"time":"2026-07-17T23:57:38.927226477-07:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-07-17T23:57:39.082122299-07:00","level":"INFO","msg":"handleInformInit: received","streamId":"5x8mg4as","id":"1(@)"}
+{"time":"2026-07-17T23:57:39.29962067-07:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"5x8mg4as","id":"1(@)"}
+{"time":"2026-07-17T23:57:44.760027013-07:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"5pumthokbgux"}

healed/calib/glean_keep25_s1224/wandb/run-20260717_235739-5x8mg4as/logs/debug-internal.log

@@ -0,0 +1,335 @@
+{"time":"2026-07-17T23:57:39.082298694-07:00","level":"INFO","msg":"wandb-core"}
+{"time":"2026-07-17T23:57:39.082803661-07:00","level":"INFO","msg":"stream: starting","core version":"0.28.0"}
+{"time":"2026-07-17T23:57:39.299416513-07:00","level":"INFO","msg":"stream: created new stream","id":"5x8mg4as"}
+{"time":"2026-07-17T23:57:39.299505736-07:00","level":"INFO","msg":"handler: started"}
+{"time":"2026-07-17T23:57:39.29960975-07:00","level":"INFO","msg":"stream: started"}
+{"time":"2026-07-17T23:57:39.299642731-07:00","level":"INFO","msg":"writer: started","stream_id":"5x8mg4as"}
+{"time":"2026-07-17T23:57:39.299690392-07:00","level":"INFO","msg":"sender: started"}
+{"time":"2026-07-17T23:57:43.114110592-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
+{"time":"2026-07-17T23:57:43.36747684-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-17T23:57:58.115071634-07:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":0,"events_lines":2,"console_offset":1,"console_lines":2,"uploaded_len":2}
+{"time":"2026-07-17T23:57:58.3042581-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-17T23:58:13.114489315-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":2,"events_lines":2}
+{"time":"2026-07-17T23:58:13.354446239-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-17T23:58:28.114155095-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":4,"events_lines":2}
+{"time":"2026-07-17T23:58:28.361670208-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-17T23:58:43.114668252-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":0,"history_lines":1,"events_offset":6,"events_lines":2,"console_offset":3,"console_lines":1}
+{"time":"2026-07-17T23:58:43.380794731-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-17T23:58:58.114479326-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":8,"events_lines":2}
+{"time":"2026-07-17T23:58:58.355010999-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-17T23:59:13.11428169-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":10,"events_lines":2}
+{"time":"2026-07-17T23:59:13.36621911-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-17T23:59:28.114376993-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":12,"events_lines":2}
+{"time":"2026-07-17T23:59:28.407411352-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-17T23:59:43.114722165-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1,"history_lines":1,"events_offset":14,"events_lines":2,"console_offset":4,"console_lines":1}
+{"time":"2026-07-17T23:59:43.36167679-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-17T23:59:58.11478371-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":16,"events_lines":2}
+{"time":"2026-07-17T23:59:58.376816137-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:00:13.115050448-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":18,"events_lines":2}
+{"time":"2026-07-18T00:00:13.337920114-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:00:28.115271535-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2,"history_lines":1,"events_offset":20,"events_lines":2,"console_offset":5,"console_lines":1}
+{"time":"2026-07-18T00:00:28.311223644-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:00:43.115012036-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":22,"events_lines":2}
+{"time":"2026-07-18T00:00:43.328317707-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:00:58.114989365-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":24,"events_lines":2}
+{"time":"2026-07-18T00:00:58.380397443-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:01:13.115005465-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":3,"history_lines":1,"events_offset":26,"events_lines":2,"console_offset":6,"console_lines":1}
+{"time":"2026-07-18T00:01:13.372753281-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:01:28.114211478-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":28,"events_lines":2}
+{"time":"2026-07-18T00:01:28.352751581-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:01:43.114903211-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":30,"events_lines":2}
+{"time":"2026-07-18T00:01:43.346113242-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:01:58.114855479-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":4,"history_lines":1,"events_offset":32,"events_lines":2,"console_offset":7,"console_lines":1}
+{"time":"2026-07-18T00:01:58.394214737-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:02:13.114507257-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":34,"events_lines":2}
+{"time":"2026-07-18T00:02:13.35032796-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:02:28.114665431-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":36,"events_lines":2}
+{"time":"2026-07-18T00:02:28.330604799-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:02:43.114218666-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":5,"history_lines":1,"events_offset":38,"events_lines":2,"console_offset":8,"console_lines":1}
+{"time":"2026-07-18T00:02:43.349813792-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:02:58.114431912-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":40,"events_lines":2}
+{"time":"2026-07-18T00:02:58.34796902-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:03:13.11497337-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":42,"events_lines":2}
+{"time":"2026-07-18T00:03:13.371135223-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:03:28.114463482-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":44,"events_lines":2}
+{"time":"2026-07-18T00:03:28.510351401-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:03:43.115158585-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":6,"history_lines":1,"events_offset":46,"events_lines":2,"console_offset":9,"console_lines":1}
+{"time":"2026-07-18T00:03:43.356438258-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:03:58.114438721-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":48,"events_lines":2}
+{"time":"2026-07-18T00:03:58.403746787-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:04:13.114996949-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":50,"events_lines":2}
+{"time":"2026-07-18T00:04:13.319954444-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:04:28.114712109-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":7,"history_lines":1,"events_offset":52,"events_lines":2,"console_offset":10,"console_lines":1}
+{"time":"2026-07-18T00:04:28.298069932-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:04:43.115088431-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":54,"events_lines":2}
+{"time":"2026-07-18T00:04:43.343263752-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:04:58.114584924-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":56,"events_lines":2}
+{"time":"2026-07-18T00:04:58.380467467-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:05:13.114212081-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":58,"events_lines":2}
+{"time":"2026-07-18T00:05:13.321669939-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:05:28.114886002-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":8,"history_lines":1,"events_offset":60,"events_lines":2,"console_offset":11,"console_lines":1}
+{"time":"2026-07-18T00:05:28.39485017-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:05:43.114921113-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":62,"events_lines":2}
+{"time":"2026-07-18T00:05:43.291005047-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:05:58.114987824-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":64,"events_lines":2}
+{"time":"2026-07-18T00:05:58.382197452-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:06:13.114951543-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":9,"history_lines":1,"events_offset":66,"events_lines":2,"console_offset":12,"console_lines":3}
+{"time":"2026-07-18T00:06:13.338487941-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:06:28.114798568-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":68,"events_lines":2}
+{"time":"2026-07-18T00:06:28.381525599-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:06:43.11428473-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":70,"events_lines":2}
+{"time":"2026-07-18T00:06:43.38671635-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:06:58.114756205-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":10,"history_lines":1,"events_offset":72,"events_lines":2,"console_offset":15,"console_lines":1}
+{"time":"2026-07-18T00:06:58.348834501-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:07:13.114292059-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":74,"events_lines":2}
+{"time":"2026-07-18T00:07:13.425121915-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:07:28.114503866-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":76,"events_lines":2}
+{"time":"2026-07-18T00:07:28.324174287-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:07:43.114974851-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":78,"events_lines":2}
+{"time":"2026-07-18T00:07:43.347532967-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:07:58.114429613-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":11,"history_lines":1,"events_offset":80,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-18T00:07:58.420549253-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:08:13.114227056-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":82,"events_lines":2}
+{"time":"2026-07-18T00:08:13.33886902-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:08:28.115151465-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":84,"events_lines":2}
+{"time":"2026-07-18T00:08:28.386022154-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:08:43.114901944-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":12,"history_lines":1,"events_offset":86,"events_lines":2,"console_offset":17,"console_lines":1}
+{"time":"2026-07-18T00:08:43.452262134-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:08:58.114833119-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":88,"events_lines":2}
+{"time":"2026-07-18T00:08:58.370403463-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:09:13.114378541-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":13,"history_lines":1,"events_offset":90,"events_lines":2,"console_offset":18,"console_lines":1}
+{"time":"2026-07-18T00:09:13.349609065-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:09:28.114872174-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":92,"events_lines":2}
+{"time":"2026-07-18T00:09:28.353673755-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:09:43.114776289-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":94,"events_lines":2}
+{"time":"2026-07-18T00:09:43.354941715-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:09:58.114340831-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":14,"history_lines":1,"events_offset":96,"events_lines":2,"console_offset":19,"console_lines":1}
+{"time":"2026-07-18T00:09:58.374027461-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:10:13.114556996-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":98,"events_lines":2}
+{"time":"2026-07-18T00:10:13.375432485-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:10:28.115024578-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":100,"events_lines":2}
+{"time":"2026-07-18T00:10:28.35050344-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:10:43.114732706-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":102,"events_lines":2}
+{"time":"2026-07-18T00:10:43.299683562-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:10:58.115083454-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":15,"history_lines":1,"events_offset":104,"events_lines":2,"console_offset":20,"console_lines":1}
+{"time":"2026-07-18T00:10:58.394849816-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:11:13.114197372-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":106,"events_lines":2}
+{"time":"2026-07-18T00:11:13.356010573-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:11:28.114613943-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":108,"events_lines":2}
+{"time":"2026-07-18T00:11:28.320183789-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:11:43.11460004-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":16,"history_lines":1,"events_offset":110,"events_lines":2,"console_offset":21,"console_lines":1}
+{"time":"2026-07-18T00:11:43.347451675-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:11:58.114659189-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":112,"events_lines":2}
+{"time":"2026-07-18T00:11:58.377632247-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:12:13.115019018-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":114,"events_lines":2}
+{"time":"2026-07-18T00:12:13.308809625-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:12:28.114896081-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":116,"events_lines":2}
+{"time":"2026-07-18T00:12:28.366914448-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:12:43.114832806-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":17,"history_lines":1,"events_offset":118,"events_lines":2,"console_offset":22,"console_lines":1}
+{"time":"2026-07-18T00:12:43.40712624-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:12:58.114295675-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":120,"events_lines":2}
+{"time":"2026-07-18T00:12:58.431232912-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:13:13.114781269-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":122,"events_lines":2}
+{"time":"2026-07-18T00:13:13.311461101-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:13:28.115127707-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":18,"history_lines":1,"events_offset":124,"events_lines":2,"console_offset":23,"console_lines":1}
+{"time":"2026-07-18T00:13:28.390737882-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:13:43.115094234-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":126,"events_lines":2}
+{"time":"2026-07-18T00:13:43.380914006-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:13:58.115007558-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":128,"events_lines":2}
+{"time":"2026-07-18T00:13:58.450051722-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:14:13.114721907-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":19,"history_lines":1,"events_offset":130,"events_lines":2,"console_offset":24,"console_lines":2}
+{"time":"2026-07-18T00:14:13.343204958-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:14:28.114150635-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":132,"events_lines":2}
+{"time":"2026-07-18T00:14:28.323429893-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:14:43.11441008-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":134,"events_lines":2}
+{"time":"2026-07-18T00:14:43.41659135-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:14:58.114621394-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":136,"events_lines":2}
+{"time":"2026-07-18T00:14:58.345904627-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:15:13.114536859-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":20,"history_lines":1,"events_offset":138,"events_lines":2,"console_offset":26,"console_lines":1}
+{"time":"2026-07-18T00:15:13.415975374-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:15:28.114747723-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":140,"events_lines":2}
+{"time":"2026-07-18T00:15:28.423126657-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:15:43.115072881-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":142,"events_lines":2}
+{"time":"2026-07-18T00:15:43.358401111-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:15:58.11452616-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":144,"events_lines":2}
+{"time":"2026-07-18T00:15:58.38758083-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:16:13.114448954-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":21,"history_lines":1,"events_offset":146,"events_lines":2,"console_offset":27,"console_lines":1}
+{"time":"2026-07-18T00:16:13.366802202-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:16:28.114237865-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":148,"events_lines":2}
+{"time":"2026-07-18T00:16:41.047068925-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:16:43.114429768-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":150,"events_lines":2}
+{"time":"2026-07-18T00:16:50.768258887-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:16:58.114938902-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":22,"history_lines":1,"events_offset":152,"events_lines":2,"console_offset":28,"console_lines":1}
+{"time":"2026-07-18T00:16:58.37032772-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:17:13.114459993-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":154,"events_lines":2}
+{"time":"2026-07-18T00:17:13.353513603-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:17:28.115177124-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":156,"events_lines":2}
+{"time":"2026-07-18T00:17:28.323644075-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:17:43.114474458-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":23,"history_lines":1,"events_offset":158,"events_lines":2,"console_offset":29,"console_lines":1}
+{"time":"2026-07-18T00:17:43.40579956-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:17:58.114250988-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":160,"events_lines":2}
+{"time":"2026-07-18T00:17:58.368057623-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:18:13.115061271-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":162,"events_lines":2}
+{"time":"2026-07-18T00:18:13.378263557-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:18:28.114951854-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":24,"history_lines":1,"events_offset":164,"events_lines":2,"console_offset":30,"console_lines":1}
+{"time":"2026-07-18T00:18:28.363443375-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:18:43.114418344-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":166,"events_lines":2}
+{"time":"2026-07-18T00:18:43.32970248-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:18:58.114242345-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":168,"events_lines":2}
+{"time":"2026-07-18T00:18:58.338862589-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:19:13.115157722-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":25,"history_lines":1,"events_offset":170,"events_lines":2,"console_offset":31,"console_lines":1}
+{"time":"2026-07-18T00:19:13.357079566-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:19:28.114654793-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":172,"events_lines":2}
+{"time":"2026-07-18T00:19:28.368258762-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:19:43.114684871-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":174,"events_lines":2}
+{"time":"2026-07-18T00:19:43.433541451-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:19:58.114640846-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":26,"history_lines":1,"events_offset":176,"events_lines":2,"console_offset":32,"console_lines":1}
+{"time":"2026-07-18T00:19:58.39659977-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:20:13.114380865-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":178,"events_lines":2}
+{"time":"2026-07-18T00:20:13.33177363-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:20:28.114811786-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":180,"events_lines":2}
+{"time":"2026-07-18T00:20:28.336992879-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:20:43.114868285-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":27,"history_lines":1,"events_offset":182,"events_lines":2,"console_offset":33,"console_lines":1}
+{"time":"2026-07-18T00:20:43.360178121-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:20:58.114327704-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":184,"events_lines":2}
+{"time":"2026-07-18T00:20:58.364339745-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:21:13.114487766-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":28,"history_lines":1,"events_offset":186,"events_lines":2,"console_offset":34,"console_lines":1}
+{"time":"2026-07-18T00:21:13.453514791-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:21:28.114788873-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":188,"events_lines":2}
+{"time":"2026-07-18T00:21:28.369748197-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:21:43.114283373-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":190,"events_lines":2}
+{"time":"2026-07-18T00:21:43.357840871-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:21:58.114964322-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":29,"history_lines":1,"events_offset":192,"events_lines":2,"console_offset":35,"console_lines":2}
+{"time":"2026-07-18T00:21:58.42313785-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:22:13.114602148-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":194,"events_lines":2}
+{"time":"2026-07-18T00:22:13.382319232-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:22:28.115128742-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":196,"events_lines":2}
+{"time":"2026-07-18T00:22:28.344518763-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:22:43.11485666-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":30,"history_lines":1,"events_offset":198,"events_lines":2,"console_offset":37,"console_lines":1}
+{"time":"2026-07-18T00:22:43.352602136-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:22:58.115175868-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":200,"events_lines":2}
+{"time":"2026-07-18T00:22:58.38586381-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:23:13.114748861-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":202,"events_lines":2}
+{"time":"2026-07-18T00:23:13.322071404-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:23:28.114544511-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":31,"history_lines":1,"events_offset":204,"events_lines":2,"console_offset":38,"console_lines":1}
+{"time":"2026-07-18T00:23:28.336230528-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:23:43.114445215-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":206,"events_lines":2}
+{"time":"2026-07-18T00:23:43.343407252-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:23:58.115155935-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":208,"events_lines":2}
+{"time":"2026-07-18T00:23:58.387638746-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:24:13.114964916-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":32,"history_lines":1,"events_offset":210,"events_lines":2,"console_offset":39,"console_lines":1}
+{"time":"2026-07-18T00:24:13.372853896-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:24:28.1151713-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":212,"events_lines":2}
+{"time":"2026-07-18T00:24:28.33694889-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:24:43.114523435-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":214,"events_lines":2}
+{"time":"2026-07-18T00:24:43.426151617-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:24:58.114472151-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":33,"history_lines":1,"events_offset":216,"events_lines":2,"console_offset":40,"console_lines":1}
+{"time":"2026-07-18T00:24:58.381332727-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:25:13.114936683-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":218,"events_lines":2}
+{"time":"2026-07-18T00:25:13.344735038-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:25:28.114186105-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":220,"events_lines":2}
+{"time":"2026-07-18T00:25:28.354739724-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:25:43.114241644-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":222,"events_lines":2}
+{"time":"2026-07-18T00:25:43.347943497-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:25:58.114767273-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":34,"history_lines":1,"events_offset":224,"events_lines":2,"console_offset":41,"console_lines":1}
+{"time":"2026-07-18T00:25:58.412158638-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:26:13.114420203-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":226,"events_lines":2}
+{"time":"2026-07-18T00:26:13.401305454-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:26:28.114621231-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":228,"events_lines":2}
+{"time":"2026-07-18T00:26:28.332453784-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:26:43.11515672-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":35,"history_lines":1,"events_offset":230,"events_lines":2,"console_offset":42,"console_lines":1}
+{"time":"2026-07-18T00:26:43.366613387-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:26:58.11448949-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":232,"events_lines":2}
+{"time":"2026-07-18T00:26:58.332852261-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:27:13.114501082-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":234,"events_lines":2}
+{"time":"2026-07-18T00:27:13.332048486-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:27:28.114556786-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":36,"history_lines":1,"events_offset":236,"events_lines":2,"console_offset":43,"console_lines":1}
+{"time":"2026-07-18T00:27:28.378220634-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:27:43.114499085-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":238,"events_lines":2}
+{"time":"2026-07-18T00:27:43.389390272-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:27:58.114125675-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":240,"events_lines":2}
+{"time":"2026-07-18T00:27:58.392616049-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:28:13.115005585-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":242,"events_lines":2}
+{"time":"2026-07-18T00:28:13.364769596-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:28:28.114784719-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":37,"history_lines":1,"events_offset":244,"events_lines":2,"console_offset":44,"console_lines":1}
+{"time":"2026-07-18T00:28:28.357918003-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:28:43.114857503-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":246,"events_lines":2}
+{"time":"2026-07-18T00:28:43.322113679-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:28:58.114276606-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":38,"history_lines":1,"events_offset":248,"events_lines":2,"console_offset":45,"console_lines":1}
+{"time":"2026-07-18T00:28:58.444313743-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:29:13.114955549-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":250,"events_lines":2}
+{"time":"2026-07-18T00:29:13.35555949-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:29:28.114553048-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":252,"events_lines":2}
+{"time":"2026-07-18T00:29:28.413704061-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:29:43.114322482-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":254,"events_lines":2}
+{"time":"2026-07-18T00:29:43.369003235-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:29:58.114355874-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":39,"history_lines":1,"events_offset":256,"events_lines":2,"console_offset":46,"console_lines":2}
+{"time":"2026-07-18T00:29:58.279112905-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:30:13.114539572-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":258,"events_lines":2}
+{"time":"2026-07-18T00:30:13.403241312-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:30:28.114324117-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":260,"events_lines":2}
+{"time":"2026-07-18T00:30:28.36840901-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:30:43.114781193-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":40,"history_lines":1,"events_offset":262,"events_lines":2,"console_offset":48,"console_lines":1}
+{"time":"2026-07-18T00:30:43.398600393-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:30:58.114661161-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":264,"events_lines":2}
+{"time":"2026-07-18T00:30:58.345818561-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:31:13.114156006-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":266,"events_lines":2}
+{"time":"2026-07-18T00:31:13.378968761-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:31:28.114444997-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":268,"events_lines":2}
+{"time":"2026-07-18T00:31:28.436054957-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:31:43.114549521-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":41,"history_lines":1,"events_offset":270,"events_lines":2,"console_offset":49,"console_lines":1}
+{"time":"2026-07-18T00:31:43.352325299-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:31:58.114212262-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":272,"events_lines":2}
+{"time":"2026-07-18T00:31:58.362499915-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:32:13.114112251-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":274,"events_lines":2}
+{"time":"2026-07-18T00:32:13.356680496-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:32:28.114704512-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":42,"history_lines":1,"events_offset":276,"events_lines":2,"console_offset":50,"console_lines":1}
+{"time":"2026-07-18T00:32:28.414907739-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:32:43.115086876-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":278,"events_lines":2}
+{"time":"2026-07-18T00:32:43.32505073-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:32:58.11457102-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":43,"history_lines":1,"events_offset":280,"events_lines":2,"console_offset":51,"console_lines":1}
+{"time":"2026-07-18T00:32:58.422558764-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:33:13.114309703-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":282,"events_lines":2}
+{"time":"2026-07-18T00:33:13.319519991-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:33:28.115009927-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":284,"events_lines":2}
+{"time":"2026-07-18T00:33:28.328599711-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:33:43.114340757-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":286,"events_lines":2}
+{"time":"2026-07-18T00:33:43.376808316-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:33:58.114748932-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":44,"history_lines":1,"events_offset":288,"events_lines":2,"console_offset":52,"console_lines":1}
+{"time":"2026-07-18T00:33:58.37598167-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:34:13.115103685-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":290,"events_lines":2}
+{"time":"2026-07-18T00:34:13.373147588-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:34:28.114633361-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":292,"events_lines":2}
+{"time":"2026-07-18T00:34:28.410383332-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:34:43.114404145-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":294,"events_lines":2}
+{"time":"2026-07-18T00:34:43.360548757-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:34:58.11481105-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":45,"history_lines":1,"events_offset":296,"events_lines":2,"console_offset":53,"console_lines":1}
+{"time":"2026-07-18T00:34:58.368686176-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:35:13.114615074-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":298,"events_lines":2}
+{"time":"2026-07-18T00:35:13.424068036-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:35:28.11503666-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":300,"events_lines":2}
+{"time":"2026-07-18T00:35:28.355005379-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:35:43.114608837-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":46,"history_lines":1,"events_offset":302,"events_lines":2,"console_offset":54,"console_lines":1}
+{"time":"2026-07-18T00:35:43.412252981-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:35:58.114767114-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":304,"events_lines":2}
+{"time":"2026-07-18T00:35:58.348510919-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:36:13.114162146-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":306,"events_lines":2}
+{"time":"2026-07-18T00:36:13.359668117-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:36:28.11485477-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":47,"history_lines":1,"events_offset":308,"events_lines":2,"console_offset":55,"console_lines":1}
+{"time":"2026-07-18T00:36:28.41786534-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:36:43.114117448-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":310,"events_lines":2}
+{"time":"2026-07-18T00:36:43.40898333-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:36:58.114940316-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":312,"events_lines":2}
+{"time":"2026-07-18T00:36:58.414177222-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:37:13.11470599-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":314,"events_lines":2}
+{"time":"2026-07-18T00:37:13.404410923-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:37:28.115211578-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":48,"history_lines":1,"events_offset":316,"events_lines":2,"console_offset":56,"console_lines":1}
+{"time":"2026-07-18T00:37:28.463553977-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:37:43.114237328-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":318,"events_lines":2}
+{"time":"2026-07-18T00:37:43.38473356-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:37:58.115021065-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":320,"events_lines":2}
+{"time":"2026-07-18T00:37:58.418948895-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:38:13.114946814-07:00","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":49,"history_lines":1,"events_offset":322,"events_lines":2,"console_offset":57,"console_lines":2}
+{"time":"2026-07-18T00:38:13.421072086-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-18T00:38:28.114884044-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":324,"events_lines":2}
+{"time":"2026-07-18T00:38:28.320243707-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}

healed/calib/glean_keep25_s1224/wandb/run-20260717_235739-5x8mg4as/logs/debug.log

@@ -0,0 +1,19 @@
+2026-07-17 23:57:39,079 INFO    MainThread:417245 [wandb_setup.py:_flush():81] Current SDK version is 0.28.0
+2026-07-17 23:57:39,080 INFO    MainThread:417245 [wandb_setup.py:_flush():81] Configure stats pid to 417245
+2026-07-17 23:57:39,080 INFO    MainThread:417245 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-07-17 23:57:39,080 INFO    MainThread:417245 [wandb_init.py:setup_run_log_directory():725] Logging user logs to outputs/healed/calib/glean_keep25_s1224/wandb/run-20260717_235739-5x8mg4as/logs/debug.log
+2026-07-17 23:57:39,080 INFO    MainThread:417245 [wandb_init.py:setup_run_log_directory():726] Logging internal logs to outputs/healed/calib/glean_keep25_s1224/wandb/run-20260717_235739-5x8mg4as/logs/debug-internal.log
+2026-07-17 23:57:39,080 INFO    MainThread:417245 [wandb_init.py:init():768] calling init triggers
+2026-07-17 23:57:39,080 INFO    MainThread:417245 [wandb_init.py:init():773] wandb.init called with sweep_config: {}
+config: {'student': 'outputs/pruned/glean-0125inst-general-keep25', 'teacher': 'allenai/OLMoE-1B-7B-0125-Instruct', 'training_mode': 'off-policy', 'kl_direction': 'forward', 'dataset': 'allenai/RLVR-MATH', 'dataset_sources': None, 'max_difficulty': None, 'trajectories': 'outputs/teacher_trajectories/dolci_math_curated.jsonl', 'trajectory_dataset': 'allenai/Dolci-Instruct-RL', 'off_policy_frames': 'chat', 'off_policy_max_seq_len': 2048, 'topk_targets': 'outputs/teacher_trajectories/dolci_combined_top128', 'max_loss_tokens': None, 'loss_tokens_per_step': 120000, 'teacher_device': 'cuda:0', 'student_device': 'cuda:0', 'lr': 3e-05, 'optimizer': 'adamw8bit', 'weight_decay': 0.1, 'epochs': 9, 'prompts_per_step': 256, 'group_size': 1, 'rollout_batch': 64, 'micro_batch': 3, 'max_new_tokens': 256, 'max_prompt_len': 1024, 'warmup_steps': 10, 'max_grad_norm': 1.0, 'eval_every': 10, 'gsm8k_every': 0, 'gsm8k_n': 256, 'gsm8k_batch': 16, 'gsm8k_max_new_tokens': 512, 'gsm8k_frames': 'chat', 'save_every': 400, 'out_dir': 'outputs/healed/calib/glean_keep25_s1224', 'sweep': 400, 'wandb': True, 'wandb_project': 'glean-calib', 'wandb_run_name': 'calib_keep25_400', 'wandb_run_id': None, 'wandb_resume': None, 'wandb_mode': 'online', 'no_wandb_sync': False, 'debug': False, 'resume_from': 'outputs/healed/calib/glean_keep25_s1224/step0300', 'start_step': 0, 'no_grad_checkpointing': False, 'seed': 1224, 'no_teacher_overlap': False, 'sync_checkpoints': False, 'rollout_engine': 'hf', 'vllm_gpu': None, 'vllm_port': 8377, 'vllm_refresh_every': 5, 'vllm_serve_bin': 'vllm-plugin/.venv/bin/python', 'vllm_gpu_mem_util': 0.85, 'vllm_refresh_mode': 'reload', 'vllm_live_dir': None, 'resolved_kl_direction': 'forward', '_wandb': {}}
+2026-07-17 23:57:39,080 INFO    MainThread:417245 [wandb_init.py:init():816] starting backend
+2026-07-17 23:57:39,080 INFO    MainThread:417245 [wandb_init.py:init():831] sending inform_init request
+2026-07-17 23:57:39,300 INFO    MainThread:417245 [wandb_init.py:init():836] backend started and connected
+2026-07-17 23:57:39,302 INFO    MainThread:417245 [wandb_init.py:init():906] updated telemetry
+2026-07-17 23:57:39,308 INFO    MainThread:417245 [wandb_init.py:init():929] communicating run to backend with 90.0 second timeout
+2026-07-17 23:57:39,616 INFO    MainThread:417245 [wandb_init.py:init():974] starting run threads in backend
+2026-07-17 23:57:39,749 INFO    MainThread:417245 [wandb_run.py:_console_start():2523] atexit reg
+2026-07-17 23:57:39,749 INFO    MainThread:417245 [wandb_run.py:_redirect():2373] redirect: wrap_raw
+2026-07-17 23:57:39,749 INFO    MainThread:417245 [wandb_run.py:_redirect():2442] Wrapping output streams.
+2026-07-17 23:57:39,749 INFO    MainThread:417245 [wandb_run.py:_redirect():2465] Redirects installed.
+2026-07-17 23:57:39,751 INFO    MainThread:417245 [wandb_init.py:init():1012] run started, returning control to user process

healed/calib/glean_keep50_s1224.eval.log

@@ -0,0 +1,81 @@
+2026-07-17T21:12:38-07:00 serving outputs/healed/calib/glean_keep50_s1224/step0100 on GPU 1 port 8431
+2026-07-17T21:12:38-07:00 waiting for server /health ...
+2026-07-17T21:13:08-07:00 server up; chat pass [gsm8k_cot_zeroshot]
+2026-07-17:21:13:09 WARNING  [config.evaluate_config:287] --limit SHOULD ONLY BE USED FOR TESTING. REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.
+2026-07-17:21:13:16 INFO     [_cli.run:388] Selected Tasks: ['gsm8k_cot_zeroshot']
+2026-07-17:21:13:17 INFO     [evaluator:214] Setting random seed to 0 | Setting numpy seed to 1234 | Setting torch manual seed to 1234 | Setting fewshot manual seed to 1234
+2026-07-17:21:13:17 WARNING  [evaluator:226] generation_kwargs: {'max_gen_toks': 1280} specified through cli, these settings will update set parameters in yaml tasks. Ensure 'do_sample=True' for non-greedy decoding!
+2026-07-17:21:13:17 INFO     [evaluator:239] Initializing local-chat-completions model, with arguments: {'model': 'student', 'base_url': 'http://127.0.0.1:8431/v1/chat/completions', 'num_concurrent': 48, 'tokenized_requests': False, 'max_retries': 3}
+2026-07-17:21:13:17 INFO     [models.api_models:179] Using max length 2048 - 1
+2026-07-17:21:13:17 INFO     [models.api_models:200] Using tokenizer None
+2026-07-17:21:13:19 INFO     [evaluator_utils:446] Selected tasks:
+2026-07-17:21:13:19 INFO     [evaluator_utils:480] Task: gsm8k_cot_zeroshot (gsm8k/gsm8k-cot-zeroshot.yaml)
+2026-07-17:21:13:19 INFO     [evaluator:314] gsm8k_cot_zeroshot: Using gen_kwargs: {'until': ['Q:', '</s>', '<|im_end|>'], 'do_sample': False, 'max_gen_toks': 1280}
+2026-07-17:21:13:19 INFO     [api.task:312] Building contexts for gsm8k_cot_zeroshot on rank 0...
+
  0%|          | 0/500 [00:00<?, ?it/s]
 37%|███▋      | 187/500 [00:00<00:00, 1863.77it/s]
 77%|███████▋  | 383/500 [00:00<00:00, 1918.66it/s]
+2026-07-17:21:13:19 INFO     [evaluator:585] Running generate_until requests
+2026-07-17:21:13:19 INFO     [models.api_models:747] Tokenized requests are disabled. Context + generation length is not checked.
+
+2026-07-17:21:14:20 INFO     [loggers.evaluation_tracker:247] Saving results aggregated
+2026-07-17:21:14:20 INFO     [loggers.evaluation_tracker:119] Saving per-task samples to outputs/evals/general_suite/calib/keep50_step100/student/*.jsonl
+local-chat-completions ({'model': 'student', 'base_url': 'http://127.0.0.1:8431/v1/chat/completions', 'num_concurrent': 48, 'tokenized_requests': False, 'max_retries': 3}), gen_kwargs: ({'max_gen_toks': 1280}), limit: 500.0, num_fewshot: None, batch_size: 1
+|      Tasks       |Version|     Filter     |n-shot|  Metric   |   |Value|   |Stderr|
+|------------------|------:|----------------|-----:|-----------|---|----:|---|-----:|
+|gsm8k_cot_zeroshot|      3|flexible-extract|     0|exact_match|↑  |0.558|±  |0.0222|
+|                  |       |strict-match    |     0|exact_match|↑  |0.000|±  |0.0000|
+
+2026-07-17T21:14:21-07:00 lm_eval exit=0 -> outputs/evals/general_suite/calib/keep50_step100
+2026-07-17T22:41:41-07:00 serving outputs/healed/calib/glean_keep50_s1224/step0200 on GPU 1 port 8431
+2026-07-17T22:41:41-07:00 waiting for server /health ...
+2026-07-17T22:42:11-07:00 server up; chat pass [gsm8k_cot_zeroshot]
+2026-07-17:22:42:11 WARNING  [config.evaluate_config:287] --limit SHOULD ONLY BE USED FOR TESTING. REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.
+2026-07-17:22:42:18 INFO     [_cli.run:388] Selected Tasks: ['gsm8k_cot_zeroshot']
+2026-07-17:22:42:19 INFO     [evaluator:214] Setting random seed to 0 | Setting numpy seed to 1234 | Setting torch manual seed to 1234 | Setting fewshot manual seed to 1234
+2026-07-17:22:42:19 WARNING  [evaluator:226] generation_kwargs: {'max_gen_toks': 1280} specified through cli, these settings will update set parameters in yaml tasks. Ensure 'do_sample=True' for non-greedy decoding!
+2026-07-17:22:42:19 INFO     [evaluator:239] Initializing local-chat-completions model, with arguments: {'model': 'student', 'base_url': 'http://127.0.0.1:8431/v1/chat/completions', 'num_concurrent': 48, 'tokenized_requests': False, 'max_retries': 3}
+2026-07-17:22:42:19 INFO     [models.api_models:179] Using max length 2048 - 1
+2026-07-17:22:42:19 INFO     [models.api_models:200] Using tokenizer None
+2026-07-17:22:42:21 INFO     [evaluator_utils:446] Selected tasks:
+2026-07-17:22:42:21 INFO     [evaluator_utils:480] Task: gsm8k_cot_zeroshot (gsm8k/gsm8k-cot-zeroshot.yaml)
+2026-07-17:22:42:21 INFO     [evaluator:314] gsm8k_cot_zeroshot: Using gen_kwargs: {'until': ['Q:', '</s>', '<|im_end|>'], 'do_sample': False, 'max_gen_toks': 1280}
+2026-07-17:22:42:21 INFO     [api.task:312] Building contexts for gsm8k_cot_zeroshot on rank 0...
+
  0%|          | 0/500 [00:00<?, ?it/s]
 37%|███▋      | 186/500 [00:00<00:00, 1855.94it/s]
 76%|███████▌  | 380/500 [00:00<00:00, 1903.22it/s]
+2026-07-17:22:42:21 INFO     [evaluator:585] Running generate_until requests
+2026-07-17:22:42:21 INFO     [models.api_models:747] Tokenized requests are disabled. Context + generation length is not checked.
+
+2026-07-17:22:43:17 INFO     [loggers.evaluation_tracker:247] Saving results aggregated
+2026-07-17:22:43:17 INFO     [loggers.evaluation_tracker:119] Saving per-task samples to outputs/evals/general_suite/calib/keep50_step200/student/*.jsonl
+local-chat-completions ({'model': 'student', 'base_url': 'http://127.0.0.1:8431/v1/chat/completions', 'num_concurrent': 48, 'tokenized_requests': False, 'max_retries': 3}), gen_kwargs: ({'max_gen_toks': 1280}), limit: 500.0, num_fewshot: None, batch_size: 1
+|      Tasks       |Version|     Filter     |n-shot|  Metric   |   |Value|   |Stderr|
+|------------------|------:|----------------|-----:|-----------|---|----:|---|-----:|
+|gsm8k_cot_zeroshot|      3|flexible-extract|     0|exact_match|↑  |0.564|±  |0.0222|
+|                  |       |strict-match    |     0|exact_match|↑  |0.000|±  |0.0000|
+
+2026-07-17T22:43:18-07:00 lm_eval exit=0 -> outputs/evals/general_suite/calib/keep50_step200
+2026-07-18T00:09:57-07:00 serving outputs/healed/calib/glean_keep50_s1224/step0300 on GPU 1 port 8431
+2026-07-18T00:09:57-07:00 waiting for server /health ...
+2026-07-18T00:10:27-07:00 server up; chat pass [gsm8k_cot_zeroshot]
+2026-07-18:00:10:28 WARNING  [config.evaluate_config:287] --limit SHOULD ONLY BE USED FOR TESTING. REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.
+2026-07-18:00:10:34 INFO     [_cli.run:388] Selected Tasks: ['gsm8k_cot_zeroshot']
+2026-07-18:00:10:36 INFO     [evaluator:214] Setting random seed to 0 | Setting numpy seed to 1234 | Setting torch manual seed to 1234 | Setting fewshot manual seed to 1234
+2026-07-18:00:10:36 WARNING  [evaluator:226] generation_kwargs: {'max_gen_toks': 1280} specified through cli, these settings will update set parameters in yaml tasks. Ensure 'do_sample=True' for non-greedy decoding!
+2026-07-18:00:10:36 INFO     [evaluator:239] Initializing local-chat-completions model, with arguments: {'model': 'student', 'base_url': 'http://127.0.0.1:8431/v1/chat/completions', 'num_concurrent': 48, 'tokenized_requests': False, 'max_retries': 3}
+2026-07-18:00:10:36 INFO     [models.api_models:179] Using max length 2048 - 1
+2026-07-18:00:10:36 INFO     [models.api_models:200] Using tokenizer None
+2026-07-18:00:10:37 INFO     [evaluator_utils:446] Selected tasks:
+2026-07-18:00:10:37 INFO     [evaluator_utils:480] Task: gsm8k_cot_zeroshot (gsm8k/gsm8k-cot-zeroshot.yaml)
+2026-07-18:00:10:37 INFO     [evaluator:314] gsm8k_cot_zeroshot: Using gen_kwargs: {'until': ['Q:', '</s>', '<|im_end|>'], 'do_sample': False, 'max_gen_toks': 1280}
+2026-07-18:00:10:37 INFO     [api.task:312] Building contexts for gsm8k_cot_zeroshot on rank 0...
+
  0%|          | 0/500 [00:00<?, ?it/s]
 38%|███▊      | 188/500 [00:00<00:00, 1873.78it/s]
 77%|███████▋  | 383/500 [00:00<00:00, 1916.13it/s]
+2026-07-18:00:10:38 INFO     [evaluator:585] Running generate_until requests
+2026-07-18:00:10:38 INFO     [models.api_models:747] Tokenized requests are disabled. Context + generation length is not checked.
+
+2026-07-18:00:11:36 INFO     [loggers.evaluation_tracker:247] Saving results aggregated
+2026-07-18:00:11:36 INFO     [loggers.evaluation_tracker:119] Saving per-task samples to outputs/evals/general_suite/calib/keep50_step300/student/*.jsonl
+local-chat-completions ({'model': 'student', 'base_url': 'http://127.0.0.1:8431/v1/chat/completions', 'num_concurrent': 48, 'tokenized_requests': False, 'max_retries': 3}), gen_kwargs: ({'max_gen_toks': 1280}), limit: 500.0, num_fewshot: None, batch_size: 1
+|      Tasks       |Version|     Filter     |n-shot|  Metric   |   |Value|   |Stderr|
+|------------------|------:|----------------|-----:|-----------|---|----:|---|-----:|
+|gsm8k_cot_zeroshot|      3|flexible-extract|     0|exact_match|↑  |0.566|±  |0.0222|
+|                  |       |strict-match    |     0|exact_match|↑  |0.000|±  |0.0000|
+
+2026-07-18T00:11:37-07:00 lm_eval exit=0 -> outputs/evals/general_suite/calib/keep50_step300

healed/calib/glean_keep50_s1224/args.json

@@ -0,0 +1,62 @@
+{
+  "student": "outputs/pruned/glean-0125inst-general-keep50",
+  "teacher": "allenai/OLMoE-1B-7B-0125-Instruct",
+  "training_mode": "off-policy",
+  "kl_direction": "forward",
+  "dataset": "allenai/RLVR-MATH",
+  "dataset_sources": null,
+  "max_difficulty": null,
+  "trajectories": "outputs/teacher_trajectories/dolci_math_curated.jsonl",
+  "trajectory_dataset": "allenai/Dolci-Instruct-RL",
+  "off_policy_frames": "chat",
+  "off_policy_max_seq_len": 2048,
+  "topk_targets": "outputs/teacher_trajectories/dolci_combined_top128",
+  "max_loss_tokens": null,
+  "loss_tokens_per_step": 120000,
+  "teacher_device": "cuda:0",
+  "student_device": "cuda:0",
+  "lr": 3e-05,
+  "optimizer": "adamw8bit",
+  "weight_decay": 0.1,
+  "epochs": 9,
+  "prompts_per_step": 256,
+  "group_size": 1,
+  "rollout_batch": 64,
+  "micro_batch": 3,
+  "max_new_tokens": 256,
+  "max_prompt_len": 1024,
+  "warmup_steps": 10,
+  "max_grad_norm": 1.0,
+  "eval_every": 10,
+  "gsm8k_every": 0,
+  "gsm8k_n": 256,
+  "gsm8k_batch": 16,
+  "gsm8k_max_new_tokens": 512,
+  "gsm8k_frames": "chat",
+  "save_every": 400,
+  "out_dir": "outputs/healed/calib/glean_keep50_s1224",
+  "sweep": 400,
+  "wandb": true,
+  "wandb_project": "glean-calib",
+  "wandb_run_name": "calib_keep50_400",
+  "wandb_run_id": null,
+  "wandb_resume": null,
+  "wandb_mode": "online",
+  "no_wandb_sync": false,
+  "debug": false,
+  "resume_from": "outputs/healed/calib/glean_keep50_s1224/step0300",
+  "start_step": 0,
+  "no_grad_checkpointing": false,
+  "seed": 1224,
+  "no_teacher_overlap": false,
+  "sync_checkpoints": false,
+  "rollout_engine": "hf",
+  "vllm_gpu": null,
+  "vllm_port": 8377,
+  "vllm_refresh_every": 5,
+  "vllm_serve_bin": "vllm-plugin/.venv/bin/python",
+  "vllm_gpu_mem_util": 0.85,
+  "vllm_refresh_mode": "reload",
+  "vllm_live_dir": null,
+  "resolved_kl_direction": "forward"
+}

healed/calib/glean_keep75_s1224.eval.log

@@ -0,0 +1,54 @@
+2026-07-17T21:35:29-07:00 serving outputs/healed/calib/glean_keep75_s1224/step0100 on GPU 2 port 8432
+2026-07-17T21:35:29-07:00 waiting for server /health ...
+2026-07-17T21:35:59-07:00 server up; chat pass [gsm8k_cot_zeroshot]
+2026-07-17:21:36:00 WARNING  [config.evaluate_config:287] --limit SHOULD ONLY BE USED FOR TESTING. REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.
+2026-07-17:21:36:08 INFO     [_cli.run:388] Selected Tasks: ['gsm8k_cot_zeroshot']
+2026-07-17:21:36:09 INFO     [evaluator:214] Setting random seed to 0 | Setting numpy seed to 1234 | Setting torch manual seed to 1234 | Setting fewshot manual seed to 1234
+2026-07-17:21:36:09 WARNING  [evaluator:226] generation_kwargs: {'max_gen_toks': 1280} specified through cli, these settings will update set parameters in yaml tasks. Ensure 'do_sample=True' for non-greedy decoding!
+2026-07-17:21:36:09 INFO     [evaluator:239] Initializing local-chat-completions model, with arguments: {'model': 'student', 'base_url': 'http://127.0.0.1:8432/v1/chat/completions', 'num_concurrent': 48, 'tokenized_requests': False, 'max_retries': 3}
+2026-07-17:21:36:09 INFO     [models.api_models:179] Using max length 2048 - 1
+2026-07-17:21:36:09 INFO     [models.api_models:200] Using tokenizer None
+2026-07-17:21:36:10 INFO     [evaluator_utils:446] Selected tasks:
+2026-07-17:21:36:10 INFO     [evaluator_utils:480] Task: gsm8k_cot_zeroshot (gsm8k/gsm8k-cot-zeroshot.yaml)
+2026-07-17:21:36:10 INFO     [evaluator:314] gsm8k_cot_zeroshot: Using gen_kwargs: {'until': ['Q:', '</s>', '<|im_end|>'], 'do_sample': False, 'max_gen_toks': 1280}
+2026-07-17:21:36:10 INFO     [api.task:312] Building contexts for gsm8k_cot_zeroshot on rank 0...
+
  0%|          | 0/500 [00:00<?, ?it/s]
 38%|███▊      | 188/500 [00:00<00:00, 1879.71it/s]
 76%|███████▋  | 382/500 [00:00<00:00, 1914.16it/s]
+2026-07-17:21:36:11 INFO     [evaluator:585] Running generate_until requests
+2026-07-17:21:36:11 INFO     [models.api_models:747] Tokenized requests are disabled. Context + generation length is not checked.
+
+2026-07-17:21:37:06 INFO     [loggers.evaluation_tracker:247] Saving results aggregated
+2026-07-17:21:37:06 INFO     [loggers.evaluation_tracker:119] Saving per-task samples to outputs/evals/general_suite/calib/keep75_step100/student/*.jsonl
+local-chat-completions ({'model': 'student', 'base_url': 'http://127.0.0.1:8432/v1/chat/completions', 'num_concurrent': 48, 'tokenized_requests': False, 'max_retries': 3}), gen_kwargs: ({'max_gen_toks': 1280}), limit: 500.0, num_fewshot: None, batch_size: 1
+|      Tasks       |Version|     Filter     |n-shot|  Metric   |   |Value|   |Stderr|
+|------------------|------:|----------------|-----:|-----------|---|----:|---|-----:|
+|gsm8k_cot_zeroshot|      3|flexible-extract|     0|exact_match|↑  |0.652|±  |0.0213|
+|                  |       |strict-match    |     0|exact_match|↑  |0.000|±  |0.0000|
+
+2026-07-17T21:37:07-07:00 lm_eval exit=0 -> outputs/evals/general_suite/calib/keep75_step100
+2026-07-17T23:22:47-07:00 serving outputs/healed/calib/glean_keep75_s1224/step0200 on GPU 2 port 8432
+2026-07-17T23:22:47-07:00 waiting for server /health ...
+2026-07-17T23:23:17-07:00 server up; chat pass [gsm8k_cot_zeroshot]
+2026-07-17:23:23:17 WARNING  [config.evaluate_config:287] --limit SHOULD ONLY BE USED FOR TESTING. REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.
+2026-07-17:23:23:24 INFO     [_cli.run:388] Selected Tasks: ['gsm8k_cot_zeroshot']
+2026-07-17:23:23:25 INFO     [evaluator:214] Setting random seed to 0 | Setting numpy seed to 1234 | Setting torch manual seed to 1234 | Setting fewshot manual seed to 1234
+2026-07-17:23:23:25 WARNING  [evaluator:226] generation_kwargs: {'max_gen_toks': 1280} specified through cli, these settings will update set parameters in yaml tasks. Ensure 'do_sample=True' for non-greedy decoding!
+2026-07-17:23:23:25 INFO     [evaluator:239] Initializing local-chat-completions model, with arguments: {'model': 'student', 'base_url': 'http://127.0.0.1:8432/v1/chat/completions', 'num_concurrent': 48, 'tokenized_requests': False, 'max_retries': 3}
+2026-07-17:23:23:26 INFO     [models.api_models:179] Using max length 2048 - 1
+2026-07-17:23:23:26 INFO     [models.api_models:200] Using tokenizer None
+2026-07-17:23:23:27 INFO     [evaluator_utils:446] Selected tasks:
+2026-07-17:23:23:27 INFO     [evaluator_utils:480] Task: gsm8k_cot_zeroshot (gsm8k/gsm8k-cot-zeroshot.yaml)
+2026-07-17:23:23:27 INFO     [evaluator:314] gsm8k_cot_zeroshot: Using gen_kwargs: {'until': ['Q:', '</s>', '<|im_end|>'], 'do_sample': False, 'max_gen_toks': 1280}
+2026-07-17:23:23:27 INFO     [api.task:312] Building contexts for gsm8k_cot_zeroshot on rank 0...
+
  0%|          | 0/500 [00:00<?, ?it/s]
 37%|███▋      | 187/500 [00:00<00:00, 1865.76it/s]
 77%|███████▋  | 383/500 [00:00<00:00, 1920.11it/s]
+2026-07-17:23:23:27 INFO     [evaluator:585] Running generate_until requests
+2026-07-17:23:23:27 INFO     [models.api_models:747] Tokenized requests are disabled. Context + generation length is not checked.
+
+2026-07-17:23:24:32 INFO     [loggers.evaluation_tracker:247] Saving results aggregated
+2026-07-17:23:24:32 INFO     [loggers.evaluation_tracker:119] Saving per-task samples to outputs/evals/general_suite/calib/keep75_step200/student/*.jsonl
+local-chat-completions ({'model': 'student', 'base_url': 'http://127.0.0.1:8432/v1/chat/completions', 'num_concurrent': 48, 'tokenized_requests': False, 'max_retries': 3}), gen_kwargs: ({'max_gen_toks': 1280}), limit: 500.0, num_fewshot: None, batch_size: 1
+|      Tasks       |Version|     Filter     |n-shot|  Metric   |   |Value|   |Stderr|
+|------------------|------:|----------------|-----:|-----------|---|----:|---|-----:|
+|gsm8k_cot_zeroshot|      3|flexible-extract|     0|exact_match|↑  |0.634|±  |0.0216|
+|                  |       |strict-match    |     0|exact_match|↑  |0.000|±  |0.0000|
+
+2026-07-17T23:24:33-07:00 lm_eval exit=0 -> outputs/evals/general_suite/calib/keep75_step200

healed/healing_breadth/glean_math_keep25_seed1224.console.log

@@ -0,0 +1,120 @@
+/home/henry/Documents/PythonProjects/variable-reap/.venv/lib/python3.12/site-packages/megablocks/grouped_gemm_util.py:10: UserWarning: Grouped GEMM not available.
+  warnings.warn('Grouped GEMM not available.')
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from /home/henry/.netrc.
+wandb: Currently logged in as: hbfreed to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: setting up run zx83gucy
+wandb: Tracking run with wandb version 0.28.0
+wandb: Run data is saved locally in outputs/healed/healing_breadth/glean_math_keep25_seed1224/wandb/run-20260714_205307-zx83gucy
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run heal-glean_math_keep25-seed1224-run3
+wandb: ⭐️ View project at https://wandb.ai/hbfreed/glean-heal
+wandb: 🚀 View run at https://wandb.ai/hbfreed/glean-heal/runs/zx83gucy
+
+starting vllm rollout server on GPU GPU-864c54df-0130-7780-e271-8a5551d1733f (port 8377) ...
+vllm server healthy in 24s
+dataset_source filter ['omega', 'polaris', 'orz_math', 'mathsub', 'dapo-math'] -> 63998 prompts
+
+difficulty <= 4 -> 63998 prompts
+WARNING: difficulty filter removed nothing — the slice likely carries difficulty=None (Dolci math sources do), so the teacher-competence guard is NOT in effect
+
+
+63977 prompts | 940 steps/epoch | 50 total steps | student params 2.09B | teacher overlap=True
+rollout short by 2,838 tokens; generating top-up 1 from 3 prompts
+{"step": 1, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 1.8929568549315134, "tokens": 120000, "cumulative_loss_tokens": 120000, "grad_norm": 47.5, "lr": 6e-06, "finish_rate": 0.216, "comp_len": 431.7, "t_data_s": 0.0, "t_rollout_s": 36.8, "t_step_s": 87.2, "t_refresh_s": 0.3, "mem_gb": 10.33}
+The attention mask is not set and cannot be inferred from input because pad token is same as eos token. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.
+[eval step 1] sample: '\nThe only number that could have been placed along thethe only number that is 75 divided by 5. The problem is to determine the maximum number of maames that could have been placed.\n\nThe only number th'
+rollout short by 21,055 tokens; generating top-up 1 from 19 prompts
+{"step": 2, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 1.8332483816136917, "tokens": 120000, "cumulative_loss_tokens": 240000, "grad_norm": 38.5, "lr": 9e-06, "finish_rate": 0.337, "comp_len": 371.5, "t_data_s": 0.0, "t_rollout_s": 38.3, "t_step_s": 87.5, "t_refresh_s": 0.3, "mem_gb": 10.45}
+rollout short by 13,628 tokens; generating top-up 1 from 11 prompts
+{"step": 3, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 1.7165187035838763, "tokens": 120000, "cumulative_loss_tokens": 360000, "grad_norm": 33.75, "lr": 1.2e-05, "finish_rate": 0.387, "comp_len": 393.4, "t_data_s": 0.0, "t_rollout_s": 37.4, "t_step_s": 84.6, "t_refresh_s": 0.3, "mem_gb": 10.35}
+rollout short by 12,030 tokens; generating top-up 1 from 10 prompts
+{"step": 4, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 1.5216001023660104, "tokens": 120000, "cumulative_loss_tokens": 480000, "grad_norm": 25.875, "lr": 1.5e-05, "finish_rate": 0.303, "comp_len": 394.7, "t_data_s": 0.0, "t_rollout_s": 36.1, "t_step_s": 80.7, "t_refresh_s": 0.3, "mem_gb": 10.37}
+rollout short by 16,574 tokens; generating top-up 1 from 14 prompts
+{"step": 5, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 1.4106985497762758, "tokens": 120000, "cumulative_loss_tokens": 600000, "grad_norm": 29.875, "lr": 1.8e-05, "finish_rate": 0.429, "comp_len": 378.5, "t_data_s": 0.0, "t_rollout_s": 36.3, "t_step_s": 87.7, "t_refresh_s": 0.3, "mem_gb": 10.36}
+rollout short by 14,735 tokens; generating top-up 1 from 12 prompts
+{"step": 6, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 1.2116640572726727, "tokens": 120000, "cumulative_loss_tokens": 720000, "grad_norm": 28.625, "lr": 2.1e-05, "finish_rate": 0.55, "comp_len": 385.9, "t_data_s": 0.0, "t_rollout_s": 39.7, "t_step_s": 90.7, "t_refresh_s": 0.3, "mem_gb": 10.36}
+rollout short by 9,665 tokens; generating top-up 1 from 8 prompts
+{"step": 7, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 1.0176955454071364, "tokens": 120000, "cumulative_loss_tokens": 840000, "grad_norm": 12.9375, "lr": 2.4e-05, "finish_rate": 0.5, "comp_len": 402.7, "t_data_s": 0.0, "t_rollout_s": 39.5, "t_step_s": 85.6, "t_refresh_s": 0.3, "mem_gb": 10.37}
+{"step": 8, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.8163571097140511, "tokens": 120000, "cumulative_loss_tokens": 960000, "grad_norm": 7.1875, "lr": 2.7000000000000002e-05, "finish_rate": 0.317, "comp_len": 452.8, "t_data_s": 0.0, "t_rollout_s": 28.8, "t_step_s": 69.0, "t_refresh_s": 0.3, "mem_gb": 10.37}
+{"step": 9, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.7240555229172111, "tokens": 120000, "cumulative_loss_tokens": 1080000, "grad_norm": 5.875, "lr": 3e-05, "finish_rate": 0.193, "comp_len": 491.8, "t_data_s": 0.0, "t_rollout_s": 33.3, "t_step_s": 77.6, "t_refresh_s": 0.3, "mem_gb": 10.38}
+{"step": 10, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.691375209479034, "tokens": 120000, "cumulative_loss_tokens": 1200000, "grad_norm": 3.734375, "lr": 3e-05, "finish_rate": 0.224, "comp_len": 480.0, "t_data_s": 0.0, "t_rollout_s": 31.9, "t_step_s": 69.5, "t_refresh_s": 0.3, "mem_gb": 10.54}
+[eval step 10] sample: "To solve this problem, we need to break it down into manageable steps and use logical reasoning. Let's start by analyzing the given conditions and the constraints.\n\n1. **Understanding the Problem:**\n "
+{"step": 11, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.6840026769215862, "tokens": 120000, "cumulative_loss_tokens": 1320000, "grad_norm": 3.453125, "lr": 3e-05, "finish_rate": 0.181, "comp_len": 493.8, "t_data_s": 0.0, "t_rollout_s": 33.9, "t_step_s": 70.7, "t_refresh_s": 0.3, "mem_gb": 10.38}
+{"step": 12, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.6246620353197058, "tokens": 120000, "cumulative_loss_tokens": 1440000, "grad_norm": 3.0625, "lr": 3e-05, "finish_rate": 0.165, "comp_len": 493.8, "t_data_s": 0.0, "t_rollout_s": 34.2, "t_step_s": 71.5, "t_refresh_s": 0.3, "mem_gb": 10.52}
+{"step": 13, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.5813416921809316, "tokens": 120000, "cumulative_loss_tokens": 1560000, "grad_norm": 2.328125, "lr": 3e-05, "finish_rate": 0.145, "comp_len": 495.9, "t_data_s": 0.0, "t_rollout_s": 33.7, "t_step_s": 70.4, "t_refresh_s": 0.3, "mem_gb": 10.42}
+{"step": 14, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.6156017889492214, "tokens": 120000, "cumulative_loss_tokens": 1680000, "grad_norm": 3.828125, "lr": 3e-05, "finish_rate": 0.139, "comp_len": 491.8, "t_data_s": 0.0, "t_rollout_s": 33.0, "t_step_s": 69.6, "t_refresh_s": 0.3, "mem_gb": 10.46}
+{"step": 15, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.5399293902300298, "tokens": 120000, "cumulative_loss_tokens": 1800000, "grad_norm": 2.265625, "lr": 3e-05, "finish_rate": 0.228, "comp_len": 487.8, "t_data_s": 0.0, "t_rollout_s": 33.8, "t_step_s": 71.3, "t_refresh_s": 0.3, "mem_gb": 10.46}
+{"step": 16, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.5018366255449752, "tokens": 120000, "cumulative_loss_tokens": 1920000, "grad_norm": 2.234375, "lr": 3e-05, "finish_rate": 0.2, "comp_len": 489.8, "t_data_s": 0.0, "t_rollout_s": 33.8, "t_step_s": 71.5, "t_refresh_s": 0.3, "mem_gb": 10.44}
+{"step": 17, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.49358773938765127, "tokens": 120000, "cumulative_loss_tokens": 2040000, "grad_norm": 2.25, "lr": 3e-05, "finish_rate": 0.255, "comp_len": 485.8, "t_data_s": 0.0, "t_rollout_s": 32.9, "t_step_s": 70.3, "t_refresh_s": 0.3, "mem_gb": 10.36}
+{"step": 18, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.45969002382730445, "tokens": 120000, "cumulative_loss_tokens": 2160000, "grad_norm": 2.203125, "lr": 3e-05, "finish_rate": 0.157, "comp_len": 495.9, "t_data_s": 0.0, "t_rollout_s": 34.5, "t_step_s": 71.0, "t_refresh_s": 0.3, "mem_gb": 10.42}
+{"step": 19, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.4492309203994771, "tokens": 120000, "cumulative_loss_tokens": 2280000, "grad_norm": 2.03125, "lr": 3e-05, "finish_rate": 0.192, "comp_len": 489.8, "t_data_s": 0.0, "t_rollout_s": 34.1, "t_step_s": 71.6, "t_refresh_s": 0.3, "mem_gb": 10.47}
+{"step": 20, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.4397894059561193, "tokens": 120000, "cumulative_loss_tokens": 2400000, "grad_norm": 2.0, "lr": 3e-05, "finish_rate": 0.164, "comp_len": 491.8, "t_data_s": 0.0, "t_rollout_s": 33.7, "t_step_s": 70.7, "t_refresh_s": 0.3, "mem_gb": 10.5}
+[eval step 20] sample: "To solve this problem, we need to use combinatorial reasoning and constraints. Let's break it down step-by-step:\n\n1. **Define Variables:**\n   - Let \\( m \\) be the number of mples.\n   - Let \\( l \\) be "
+{"step": 21, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.42920704178611435, "tokens": 120000, "cumulative_loss_tokens": 2520000, "grad_norm": 1.890625, "lr": 3e-05, "finish_rate": 0.214, "comp_len": 483.9, "t_data_s": 0.0, "t_rollout_s": 31.8, "t_step_s": 68.7, "t_refresh_s": 0.3, "mem_gb": 10.38}
+{"step": 22, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.4285325840468208, "tokens": 120000, "cumulative_loss_tokens": 2640000, "grad_norm": 1.8359375, "lr": 3e-05, "finish_rate": 0.209, "comp_len": 491.8, "t_data_s": 0.0, "t_rollout_s": 34.4, "t_step_s": 71.7, "t_refresh_s": 0.3, "mem_gb": 10.42}
+{"step": 23, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.39100426478857797, "tokens": 120000, "cumulative_loss_tokens": 2760000, "grad_norm": 2.078125, "lr": 3e-05, "finish_rate": 0.117, "comp_len": 500.0, "t_data_s": 0.0, "t_rollout_s": 35.1, "t_step_s": 71.4, "t_refresh_s": 0.3, "mem_gb": 10.43}
+{"step": 24, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.44906475950367747, "tokens": 120000, "cumulative_loss_tokens": 2880000, "grad_norm": 2.09375, "lr": 3e-05, "finish_rate": 0.172, "comp_len": 491.8, "t_data_s": 0.0, "t_rollout_s": 33.4, "t_step_s": 71.4, "t_refresh_s": 0.3, "mem_gb": 10.51}
+{"step": 25, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.4109231275471548, "tokens": 120000, "cumulative_loss_tokens": 3000000, "grad_norm": 1.8125, "lr": 3e-05, "finish_rate": 0.25, "comp_len": 483.9, "t_data_s": 0.0, "t_rollout_s": 32.5, "t_step_s": 70.7, "t_refresh_s": 0.3, "mem_gb": 10.53}
+checkpoint snapshot queued -> outputs/healed/healing_breadth/glean_math_keep25_seed1224/step0025
+{"step": 26, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.3922373386658728, "tokens": 120000, "cumulative_loss_tokens": 3120000, "grad_norm": 1.8125, "lr": 3e-05, "finish_rate": 0.251, "comp_len": 478.1, "t_data_s": 0.0, "t_rollout_s": 32.5, "t_step_s": 71.2, "t_refresh_s": 0.3, "mem_gb": 10.48}
+{"step": 27, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.38296805831243597, "tokens": 120000, "cumulative_loss_tokens": 3240000, "grad_norm": 1.859375, "lr": 3e-05, "finish_rate": 0.199, "comp_len": 487.8, "t_data_s": 0.0, "t_rollout_s": 33.3, "t_step_s": 71.8, "t_refresh_s": 0.4, "mem_gb": 10.48}
+{"step": 28, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.38206375452155866, "tokens": 120000, "cumulative_loss_tokens": 3360000, "grad_norm": 1.6875, "lr": 3e-05, "finish_rate": 0.152, "comp_len": 493.8, "t_data_s": 0.0, "t_rollout_s": 33.5, "t_step_s": 70.9, "t_refresh_s": 0.3, "mem_gb": 10.5}
+{"step": 29, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.3743179695730408, "tokens": 120000, "cumulative_loss_tokens": 3480000, "grad_norm": 1.7421875, "lr": 3e-05, "finish_rate": 0.177, "comp_len": 493.8, "t_data_s": 0.0, "t_rollout_s": 34.0, "t_step_s": 72.0, "t_refresh_s": 0.4, "mem_gb": 10.49}
+{"step": 30, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.38618751456712685, "tokens": 120000, "cumulative_loss_tokens": 3600000, "grad_norm": 1.796875, "lr": 3e-05, "finish_rate": 0.242, "comp_len": 483.9, "t_data_s": 0.0, "t_rollout_s": 32.7, "t_step_s": 69.3, "t_refresh_s": 0.3, "mem_gb": 10.45}
+[eval step 30] sample: 'To solve this problem, we need to determine the maximum number of maples that can be planted along the alley given the constraints:\n\n1. The total number of trees is 75.\n2. There are no two maples betw'
+{"step": 31, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.3496829746659845, "tokens": 120000, "cumulative_loss_tokens": 3720000, "grad_norm": 1.6328125, "lr": 3e-05, "finish_rate": 0.232, "comp_len": 480.0, "t_data_s": 0.0, "t_rollout_s": 30.5, "t_step_s": 68.1, "t_refresh_s": 0.3, "mem_gb": 10.45}
+{"step": 32, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.3457430052795758, "tokens": 120000, "cumulative_loss_tokens": 3840000, "grad_norm": 1.7890625, "lr": 3e-05, "finish_rate": 0.194, "comp_len": 485.8, "t_data_s": 0.0, "t_rollout_s": 32.5, "t_step_s": 69.8, "t_refresh_s": 0.3, "mem_gb": 10.48}
+{"step": 33, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.36799976486029723, "tokens": 120000, "cumulative_loss_tokens": 3960000, "grad_norm": 1.71875, "lr": 3e-05, "finish_rate": 0.232, "comp_len": 480.0, "t_data_s": 0.0, "t_rollout_s": 31.6, "t_step_s": 69.4, "t_refresh_s": 0.3, "mem_gb": 10.44}
+{"step": 34, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.3638427472679565, "tokens": 120000, "cumulative_loss_tokens": 4080000, "grad_norm": 1.6484375, "lr": 3e-05, "finish_rate": 0.257, "comp_len": 481.9, "t_data_s": 0.0, "t_rollout_s": 31.5, "t_step_s": 69.5, "t_refresh_s": 0.3, "mem_gb": 10.51}
+{"step": 35, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.36085621776568394, "tokens": 120000, "cumulative_loss_tokens": 4200000, "grad_norm": 1.7265625, "lr": 3e-05, "finish_rate": 0.241, "comp_len": 481.9, "t_data_s": 0.0, "t_rollout_s": 32.7, "t_step_s": 71.0, "t_refresh_s": 0.3, "mem_gb": 10.47}
+{"step": 36, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.327357999405389, "tokens": 120000, "cumulative_loss_tokens": 4320000, "grad_norm": 1.5, "lr": 3e-05, "finish_rate": 0.237, "comp_len": 481.9, "t_data_s": 0.0, "t_rollout_s": 32.7, "t_step_s": 71.1, "t_refresh_s": 0.3, "mem_gb": 10.5}
+{"step": 37, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.34588671219460665, "tokens": 120000, "cumulative_loss_tokens": 4440000, "grad_norm": 1.6875, "lr": 3e-05, "finish_rate": 0.171, "comp_len": 489.8, "t_data_s": 0.0, "t_rollout_s": 32.9, "t_step_s": 70.7, "t_refresh_s": 0.3, "mem_gb": 10.42}
+{"step": 38, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.34063462760510543, "tokens": 120000, "cumulative_loss_tokens": 4560000, "grad_norm": 1.7109375, "lr": 3e-05, "finish_rate": 0.21, "comp_len": 483.9, "t_data_s": 0.0, "t_rollout_s": 32.2, "t_step_s": 69.0, "t_refresh_s": 0.3, "mem_gb": 10.49}
+{"step": 39, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.336651858022064, "tokens": 120000, "cumulative_loss_tokens": 4680000, "grad_norm": 1.4921875, "lr": 3e-05, "finish_rate": 0.238, "comp_len": 483.9, "t_data_s": 0.0, "t_rollout_s": 31.0, "t_step_s": 68.4, "t_refresh_s": 0.3, "mem_gb": 10.47}
+{"step": 40, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.32966975886846583, "tokens": 120000, "cumulative_loss_tokens": 4800000, "grad_norm": 1.6875, "lr": 3e-05, "finish_rate": 0.246, "comp_len": 483.9, "t_data_s": 0.0, "t_rollout_s": 32.7, "t_step_s": 70.5, "t_refresh_s": 0.3, "mem_gb": 10.37}
+[eval step 40] sample: "To solve this problem, we need to use combinatorial reasoning and constraints. Let's break down the problem step-by-step:\n\n1. **Define Variables:**\n   - Let \\( m \\) be the number of maples.\n   - Let \\"
+{"step": 41, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.31904542534798386, "tokens": 120000, "cumulative_loss_tokens": 4920000, "grad_norm": 1.703125, "lr": 3e-05, "finish_rate": 0.24, "comp_len": 480.0, "t_data_s": 0.0, "t_rollout_s": 31.7, "t_step_s": 70.4, "t_refresh_s": 0.3, "mem_gb": 10.53}
+{"step": 42, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.3319877192765474, "tokens": 120000, "cumulative_loss_tokens": 5040000, "grad_norm": 1.6796875, "lr": 3e-05, "finish_rate": 0.213, "comp_len": 481.9, "t_data_s": 0.0, "t_rollout_s": 32.3, "t_step_s": 69.9, "t_refresh_s": 0.3, "mem_gb": 10.46}
+{"step": 43, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.32353154991058014, "tokens": 120000, "cumulative_loss_tokens": 5160000, "grad_norm": 1.578125, "lr": 3e-05, "finish_rate": 0.293, "comp_len": 481.9, "t_data_s": 0.0, "t_rollout_s": 32.2, "t_step_s": 69.6, "t_refresh_s": 0.3, "mem_gb": 10.38}
+{"step": 44, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.3083366705554227, "tokens": 120000, "cumulative_loss_tokens": 5280000, "grad_norm": 1.59375, "lr": 3e-05, "finish_rate": 0.217, "comp_len": 481.9, "t_data_s": 0.0, "t_rollout_s": 31.9, "t_step_s": 69.0, "t_refresh_s": 0.3, "mem_gb": 10.44}
+{"step": 45, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.3119819214983533, "tokens": 120000, "cumulative_loss_tokens": 5400000, "grad_norm": 1.59375, "lr": 3e-05, "finish_rate": 0.245, "comp_len": 481.9, "t_data_s": 0.0, "t_rollout_s": 31.6, "t_step_s": 68.3, "t_refresh_s": 0.3, "mem_gb": 10.36}
+{"step": 46, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.30937388323086, "tokens": 120000, "cumulative_loss_tokens": 5520000, "grad_norm": 1.5, "lr": 3e-05, "finish_rate": 0.252, "comp_len": 480.0, "t_data_s": 0.0, "t_rollout_s": 31.8, "t_step_s": 69.8, "t_refresh_s": 0.3, "mem_gb": 10.43}
+{"step": 47, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.2979762297540903, "tokens": 120000, "cumulative_loss_tokens": 5640000, "grad_norm": 1.6171875, "lr": 3e-05, "finish_rate": 0.186, "comp_len": 485.8, "t_data_s": 0.0, "t_rollout_s": 31.0, "t_step_s": 68.3, "t_refresh_s": 0.3, "mem_gb": 10.46}
+{"step": 48, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.31448883896246554, "tokens": 120000, "cumulative_loss_tokens": 5760000, "grad_norm": 1.6015625, "lr": 3e-05, "finish_rate": 0.201, "comp_len": 481.9, "t_data_s": 0.0, "t_rollout_s": 32.0, "t_step_s": 70.8, "t_refresh_s": 0.3, "mem_gb": 10.43}
+{"step": 49, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.2903304275934895, "tokens": 120000, "cumulative_loss_tokens": 5880000, "grad_norm": 1.3671875, "lr": 3e-05, "finish_rate": 0.231, "comp_len": 485.8, "t_data_s": 0.0, "t_rollout_s": 31.9, "t_step_s": 69.4, "t_refresh_s": 0.3, "mem_gb": 10.52}
+{"step": 50, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.30557283139005303, "tokens": 120000, "cumulative_loss_tokens": 6000000, "grad_norm": 1.5859375, "lr": 3e-05, "finish_rate": 0.25, "comp_len": 483.9, "t_data_s": 0.0, "t_rollout_s": 33.3, "t_step_s": 64.3, "t_refresh_s": 0.0, "mem_gb": 10.54}
+[eval step 50] sample: "To solve this problem, we need to use combinatorial reasoning and constraints. Let's break down the problem step-by-step:\n\n1. **Define Variables:**\n   - Let \\( m \\) be the number of maples.\n   - Let \\"
+checkpoint snapshot queued -> outputs/healed/healing_breadth/glean_math_keep25_seed1224/step0050
+wandb: updating run metadata
+wandb: uploading output.log; uploading wandb-summary.json; uploading config.yaml
+wandb: 
+wandb: Run history:
+wandb:               comp_len ▄▁▂▂▁▃▅█▇███▇▇▇▇█▇██▇▇▇██▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇
+wandb: cumulative_loss_tokens ▁▁▁▁▂▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇██
+wandb:                  epoch ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁
+wandb:            finish_rate ▃▅▆▄▇█▅▂▃▂▂▁▃▃▄▂▂▃▃▁▃▃▂▂▂▃▂▃▄▃▂▃▃▃▃▃▃▃▂▃
+wandb:              grad_norm █▇▆▅▅▃▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁��▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁
+wandb:                     lr ▁▂▃▄▅▆▇█████████████████████████████████
+wandb:                 mem_gb ▁▅▂▂▂▂▂▃█▃▄▅▅▂▄▇▃▄▄▇▆▇▆▅▅▅▇▆▇▄▆▂█▅▃▂▄▅▄█
+wandb:             reverse_kl ██▇▆▆▄▃▃▃▂▂▂▂▂▂▂▂▂▁▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁
+wandb:                   step ▁▁▁▂▂▂▂▂▂▃▃▃▃▃▃▄▄▄▄▄▅▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇███
+wandb:               t_data_s ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁
+wandb:                     +4 ...
+wandb: 
+wandb: Run summary:
+wandb:               comp_len 483.9
+wandb: cumulative_loss_tokens 6000000
+wandb:                  epoch 0
+wandb:            finish_rate 0.25
+wandb:              grad_norm 1.58594
+wandb:                     lr 3e-05
+wandb:                 mem_gb 10.54
+wandb:             reverse_kl 0.30557
+wandb:                   step 50
+wandb:               t_data_s 0
+wandb:                     +5 ...
+wandb: 
+wandb: 🚀 View run heal-glean_math_keep25-seed1224-run3 at: https://wandb.ai/hbfreed/glean-heal/runs/zx83gucy
+wandb: ⭐️ View project at: https://wandb.ai/hbfreed/glean-heal
+wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)
+wandb: Find logs at: outputs/healed/healing_breadth/glean_math_keep25_seed1224/wandb/run-20260714_205307-zx83gucy/logs

healed/healing_breadth/glean_math_keep25_seed1224_long.console.log

@@ -0,0 +1,240 @@
+/home/henry/Documents/PythonProjects/variable-reap/.venv/lib/python3.12/site-packages/megablocks/grouped_gemm_util.py:10: UserWarning: Grouped GEMM not available.
+  warnings.warn('Grouped GEMM not available.')
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from /home/henry/.netrc.
+wandb: Currently logged in as: hbfreed to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: setting up run vpov0urn
+wandb: Tracking run with wandb version 0.28.0
+wandb: Run data is saved locally in outputs/healed/healing_breadth/glean_math_keep25_seed1224_long/wandb/run-20260715_063332-vpov0urn
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run heal-glean-math-keep25-seed1224-long-step500
+wandb: ⭐️ View project at https://wandb.ai/hbfreed/glean-heal
+wandb: 🚀 View run at https://wandb.ai/hbfreed/glean-heal/runs/vpov0urn
+
+resumed student weights from outputs/healed/healing_breadth/glean_math_keep25_seed1224/step0050 (fresh optimizer, step counter at 50)
+starting vllm rollout server on GPU GPU-864c54df-0130-7780-e271-8a5551d1733f (port 8377) ...
+vllm server healthy in 24s
+dataset_source filter ['omega', 'polaris', 'orz_math', 'mathsub', 'dapo-math'] -> 63998 prompts
+
+difficulty <= 4 -> 63998 prompts
+WARNING: difficulty filter removed nothing — the slice likely carries difficulty=None (Dolci math sources do), so the teacher-competence guard is NOT in effect
+
+
+63977 prompts | 940 steps/epoch | 500 total steps | student params 2.09B | teacher overlap=True
+restored optimizer/scheduler state from step 50; rebuilt 228 paged buffers
+{"step": 51, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.36993047905315957, "tokens": 120000, "cumulative_loss_tokens": 6120000, "grad_norm": 2.203125, "lr": 3e-05, "finish_rate": 0.025, "comp_len": 508.5, "t_data_s": 0.0, "t_rollout_s": 37.6, "t_step_s": 80.2, "t_refresh_s": 0.3, "mem_gb": 10.4}
+wandb: updating run metadata
+wandb: uploading summary
+wandb: 
+wandb: Run history:
+wandb:               comp_len ▁
+wandb: cumulative_loss_tokens ▁
+wandb:                  epoch ▁
+wandb:            finish_rate ▁
+wandb:              grad_norm ▁
+wandb:                     lr ▁
+wandb:                 mem_gb ▁
+wandb:             reverse_kl ▁
+wandb:                   step ▁
+wandb:               t_data_s ▁
+wandb:                     +4 ...
+wandb: 
+wandb: Run summary:
+wandb:               comp_len 508.5
+wandb: cumulative_loss_tokens 6120000
+wandb:                  epoch 0
+wandb:            finish_rate 0.025
+wandb:              grad_norm 2.20312
+wandb:                     lr 3e-05
+wandb:                 mem_gb 10.4
+wandb:             reverse_kl 0.36993
+wandb:                   step 51
+wandb:               t_data_s 0
+wandb:                     +5 ...
+wandb: 
+wandb: 🚀 View run heal-glean-math-keep25-seed1224-long-step500 at: https://wandb.ai/hbfreed/glean-heal/runs/vpov0urn
+wandb: ⭐️ View project at: https://wandb.ai/hbfreed/glean-heal
+wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)
+wandb: Find logs at: outputs/healed/healing_breadth/glean_math_keep25_seed1224_long/wandb/run-20260715_063332-vpov0urn/logs
+Traceback (most recent call last):
+  File "/home/henry/Documents/PythonProjects/variable-reap/scripts/11_distill_on_policy.py", line 1017, in <module>
+    main()
+  File "/home/henry/Documents/PythonProjects/variable-reap/scripts/11_distill_on_policy.py", line 860, in main
+    save_student(student, tokenizer, live)
+  File "/home/henry/Documents/PythonProjects/variable-reap/scripts/11_distill_on_policy.py", line 974, in save_student
+    write_student_snapshot(student, tokenizer, path, snapshot_student(student))
+  File "/home/henry/Documents/PythonProjects/variable-reap/scripts/11_distill_on_policy.py", line 961, in write_student_snapshot
+    student.save_pretrained(path, state_dict=state_dict)
+  File "/home/henry/Documents/PythonProjects/variable-reap/.venv/lib/python3.12/site-packages/transformers/modeling_utils.py", line 4173, in save_pretrained
+    safe_save_file(shard, os.path.join(save_directory, shard_file), metadata=metadata)
+  File "/home/henry/Documents/PythonProjects/variable-reap/.venv/lib/python3.12/site-packages/safetensors/torch.py", line 323, in save_file
+    serialize_file(
+safetensors._safetensors_rust.SafetensorError: Error while serializing: I/O error: No space left on device (os error 28)
+/home/henry/Documents/PythonProjects/variable-reap/.venv/lib/python3.12/site-packages/megablocks/grouped_gemm_util.py:10: UserWarning: Grouped GEMM not available.
+  warnings.warn('Grouped GEMM not available.')
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from /home/henry/.netrc.
+wandb: Currently logged in as: hbfreed to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: setting up run i5fg3xzq
+wandb: Tracking run with wandb version 0.28.0
+wandb: Run data is saved locally in outputs/healed/healing_breadth/glean_math_keep25_seed1224_long/wandb/run-20260715_071701-i5fg3xzq
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run heal-glean-math-keep25-seed1224-long-step500
+wandb: ⭐️ View project at https://wandb.ai/hbfreed/glean-heal
+wandb: 🚀 View run at https://wandb.ai/hbfreed/glean-heal/runs/i5fg3xzq
+
+resumed student weights from outputs/healed/healing_breadth/glean_math_keep25_seed1224/step0050 (fresh optimizer, step counter at 50)
+starting vllm rollout server on GPU GPU-864c54df-0130-7780-e271-8a5551d1733f (port 8377) ...
+vllm server healthy in 24s
+dataset_source filter ['omega', 'polaris', 'orz_math', 'mathsub', 'dapo-math'] -> 63998 prompts
+difficulty <= 4 -> 63998 prompts
+WARNING: difficulty filter removed nothing — the slice likely carries difficulty=None (Dolci math sources do), so the teacher-competence guard is NOT in effect
+63977 prompts | 940 steps/epoch | 500 total steps | student params 2.09B | teacher overlap=True
+restored optimizer/scheduler state from step 50; rebuilt 228 paged buffers
+{"step": 51, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.36993047905315957, "tokens": 120000, "cumulative_loss_tokens": 6120000, "grad_norm": 2.203125, "lr": 3e-05, "finish_rate": 0.025, "comp_len": 508.5, "t_data_s": 0.0, "t_rollout_s": 37.4, "t_step_s": 79.7, "t_refresh_s": 0.4, "mem_gb": 10.4}
+{"step": 52, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.3874280411116779, "tokens": 120000, "cumulative_loss_tokens": 6240000, "grad_norm": 1.765625, "lr": 3e-05, "finish_rate": 0.021, "comp_len": 508.5, "t_data_s": 0.0, "t_rollout_s": 37.0, "t_step_s": 72.9, "t_refresh_s": 0.3, "mem_gb": 10.38}
+{"step": 53, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.3746817006888489, "tokens": 120000, "cumulative_loss_tokens": 6360000, "grad_norm": 1.703125, "lr": 3e-05, "finish_rate": 0.046, "comp_len": 506.3, "t_data_s": 0.0, "t_rollout_s": 37.1, "t_step_s": 73.6, "t_refresh_s": 0.3, "mem_gb": 10.45}
+{"step": 54, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.35510130622684954, "tokens": 120000, "cumulative_loss_tokens": 6480000, "grad_norm": 2.140625, "lr": 3e-05, "finish_rate": 0.021, "comp_len": 508.5, "t_data_s": 0.0, "t_rollout_s": 37.7, "t_step_s": 74.6, "t_refresh_s": 0.3, "mem_gb": 10.49}
+{"step": 55, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.34786954234316947, "tokens": 120000, "cumulative_loss_tokens": 6600000, "grad_norm": 1.8359375, "lr": 3e-05, "finish_rate": 0.025, "comp_len": 508.5, "t_data_s": 0.0, "t_rollout_s": 36.9, "t_step_s": 72.5, "t_refresh_s": 0.3, "mem_gb": 10.37}
+{"step": 56, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.3465809899068127, "tokens": 120000, "cumulative_loss_tokens": 6720000, "grad_norm": 1.6796875, "lr": 3e-05, "finish_rate": 0.038, "comp_len": 508.5, "t_data_s": 0.0, "t_rollout_s": 36.9, "t_step_s": 72.6, "t_refresh_s": 0.3, "mem_gb": 10.43}
+{"step": 57, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.37013316290453074, "tokens": 120000, "cumulative_loss_tokens": 6840000, "grad_norm": 2.34375, "lr": 3e-05, "finish_rate": 0.059, "comp_len": 506.3, "t_data_s": 0.0, "t_rollout_s": 36.9, "t_step_s": 72.9, "t_refresh_s": 0.3, "mem_gb": 10.51}
+{"step": 58, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.3522711561133464, "tokens": 120000, "cumulative_loss_tokens": 6960000, "grad_norm": 1.6328125, "lr": 3e-05, "finish_rate": 0.104, "comp_len": 500.0, "t_data_s": 0.0, "t_rollout_s": 35.5, "t_step_s": 72.8, "t_refresh_s": 0.3, "mem_gb": 10.47}
+{"step": 59, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.3397159261740744, "tokens": 120000, "cumulative_loss_tokens": 7080000, "grad_norm": 1.8203125, "lr": 3e-05, "finish_rate": 0.055, "comp_len": 506.3, "t_data_s": 0.0, "t_rollout_s": 36.8, "t_step_s": 72.5, "t_refresh_s": 0.3, "mem_gb": 10.37}
+{"step": 60, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.3382764852608244, "tokens": 120000, "cumulative_loss_tokens": 7200000, "grad_norm": 1.7265625, "lr": 3e-05, "finish_rate": 0.08, "comp_len": 504.2, "t_data_s": 0.0, "t_rollout_s": 36.7, "t_step_s": 72.7, "t_refresh_s": 0.3, "mem_gb": 10.42}
+The attention mask is not set and cannot be inferred from input because pad token is same as eos token. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.
+[eval step 60] sample: "To solve this problem, we need to consider the constraints and the total number of trees. Let's denote:\n- \\( m \\) as the number of maples.\n- \\( l \\) as the number of larks.\n\nGiven:\n1. The total number"
+{"step": 60, "gsm8k_n": 64, "gsm8k_quick_chat": 0.390625, "t_eval_s": 46.9}
+{"step": 61, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.31868271032124756, "tokens": 120000, "cumulative_loss_tokens": 7320000, "grad_norm": 1.5859375, "lr": 3e-05, "finish_rate": 0.059, "comp_len": 506.3, "t_data_s": 0.0, "t_rollout_s": 37.5, "t_step_s": 74.1, "t_refresh_s": 0.3, "mem_gb": 12.44}
+{"step": 62, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.3191785570286214, "tokens": 120000, "cumulative_loss_tokens": 7440000, "grad_norm": 1.5, "lr": 3e-05, "finish_rate": 0.063, "comp_len": 506.3, "t_data_s": 0.0, "t_rollout_s": 36.9, "t_step_s": 73.1, "t_refresh_s": 0.3, "mem_gb": 10.43}
+{"step": 63, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.3180096957164506, "tokens": 120000, "cumulative_loss_tokens": 7560000, "grad_norm": 1.5625, "lr": 3e-05, "finish_rate": 0.092, "comp_len": 504.2, "t_data_s": 0.0, "t_rollout_s": 35.3, "t_step_s": 70.9, "t_refresh_s": 0.3, "mem_gb": 10.37}
+{"step": 64, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.288780878906697, "tokens": 120000, "cumulative_loss_tokens": 7680000, "grad_norm": 1.7578125, "lr": 3e-05, "finish_rate": 0.046, "comp_len": 504.2, "t_data_s": 0.0, "t_rollout_s": 37.6, "t_step_s": 74.6, "t_refresh_s": 0.3, "mem_gb": 10.51}
+{"step": 65, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.29233540159290033, "tokens": 120000, "cumulative_loss_tokens": 7800000, "grad_norm": 1.4921875, "lr": 3e-05, "finish_rate": 0.084, "comp_len": 504.2, "t_data_s": 0.0, "t_rollout_s": 36.2, "t_step_s": 71.9, "t_refresh_s": 0.3, "mem_gb": 10.5}
+{"step": 66, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.2769135721528282, "tokens": 120000, "cumulative_loss_tokens": 7920000, "grad_norm": 1.3515625, "lr": 3e-05, "finish_rate": 0.105, "comp_len": 504.2, "t_data_s": 0.0, "t_rollout_s": 35.6, "t_step_s": 71.3, "t_refresh_s": 0.3, "mem_gb": 10.53}
+{"step": 67, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.3056252569064498, "tokens": 120000, "cumulative_loss_tokens": 8040000, "grad_norm": 1.4375, "lr": 3e-05, "finish_rate": 0.181, "comp_len": 493.8, "t_data_s": 0.0, "t_rollout_s": 33.5, "t_step_s": 69.5, "t_refresh_s": 0.3, "mem_gb": 10.38}
+{"step": 68, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.30779550124146043, "tokens": 120000, "cumulative_loss_tokens": 8160000, "grad_norm": 1.5, "lr": 3e-05, "finish_rate": 0.129, "comp_len": 497.9, "t_data_s": 0.0, "t_rollout_s": 34.9, "t_step_s": 71.2, "t_refresh_s": 0.3, "mem_gb": 10.42}
+{"step": 69, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.2875990556370467, "tokens": 120000, "cumulative_loss_tokens": 8280000, "grad_norm": 1.4609375, "lr": 3e-05, "finish_rate": 0.132, "comp_len": 495.9, "t_data_s": 0.0, "t_rollout_s": 33.8, "t_step_s": 69.7, "t_refresh_s": 0.3, "mem_gb": 10.38}
+{"step": 70, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.28901881557305653, "tokens": 120000, "cumulative_loss_tokens": 8400000, "grad_norm": 1.3828125, "lr": 3e-05, "finish_rate": 0.108, "comp_len": 500.0, "t_data_s": 0.0, "t_rollout_s": 36.2, "t_step_s": 73.7, "t_refresh_s": 0.3, "mem_gb": 10.54}
+[eval step 70] sample: 'To solve this problem, we need to determine the maximum number of maples that can be planted along the alley given the constraints:\n\n1. The total number of trees is 75.\n2. There are no two maples betw'
+{"step": 70, "gsm8k_n": 64, "gsm8k_quick_chat": 0.28125, "t_eval_s": 44.7}
+{"step": 71, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.281591778382659, "tokens": 120000, "cumulative_loss_tokens": 8520000, "grad_norm": 1.53125, "lr": 3e-05, "finish_rate": 0.144, "comp_len": 493.8, "t_data_s": 0.0, "t_rollout_s": 33.2, "t_step_s": 69.5, "t_refresh_s": 0.3, "mem_gb": 12.44}
+{"step": 72, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.27427094464078544, "tokens": 120000, "cumulative_loss_tokens": 8640000, "grad_norm": 1.4375, "lr": 3e-05, "finish_rate": 0.107, "comp_len": 495.9, "t_data_s": 0.0, "t_rollout_s": 35.8, "t_step_s": 73.1, "t_refresh_s": 0.3, "mem_gb": 10.47}
+{"step": 73, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.25415447360488275, "tokens": 120000, "cumulative_loss_tokens": 8760000, "grad_norm": 1.3359375, "lr": 3e-05, "finish_rate": 0.116, "comp_len": 497.9, "t_data_s": 0.0, "t_rollout_s": 34.4, "t_step_s": 71.6, "t_refresh_s": 0.3, "mem_gb": 10.38}
+{"step": 74, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.2598834279651443, "tokens": 120000, "cumulative_loss_tokens": 8880000, "grad_norm": 1.1875, "lr": 3e-05, "finish_rate": 0.109, "comp_len": 502.1, "t_data_s": 0.0, "t_rollout_s": 35.8, "t_step_s": 72.9, "t_refresh_s": 0.3, "mem_gb": 10.46}
+{"step": 75, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.3036417054095616, "tokens": 120000, "cumulative_loss_tokens": 9000000, "grad_norm": 1.40625, "lr": 3e-05, "finish_rate": 0.133, "comp_len": 497.9, "t_data_s": 0.0, "t_rollout_s": 34.9, "t_step_s": 71.1, "t_refresh_s": 0.3, "mem_gb": 10.41}
+{"step": 76, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.28468160825570427, "tokens": 120000, "cumulative_loss_tokens": 9120000, "grad_norm": 1.40625, "lr": 3e-05, "finish_rate": 0.124, "comp_len": 497.9, "t_data_s": 0.0, "t_rollout_s": 34.7, "t_step_s": 71.7, "t_refresh_s": 0.3, "mem_gb": 10.45}
+{"step": 77, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.26907755986650783, "tokens": 120000, "cumulative_loss_tokens": 9240000, "grad_norm": 1.2109375, "lr": 3e-05, "finish_rate": 0.137, "comp_len": 497.9, "t_data_s": 0.0, "t_rollout_s": 34.0, "t_step_s": 70.4, "t_refresh_s": 0.3, "mem_gb": 10.46}
+{"step": 78, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.23572471514294546, "tokens": 120000, "cumulative_loss_tokens": 9360000, "grad_norm": 1.3125, "lr": 3e-05, "finish_rate": 0.113, "comp_len": 504.2, "t_data_s": 0.0, "t_rollout_s": 36.0, "t_step_s": 72.0, "t_refresh_s": 0.3, "mem_gb": 10.45}
+{"step": 79, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.2453982294137279, "tokens": 120000, "cumulative_loss_tokens": 9480000, "grad_norm": 1.296875, "lr": 3e-05, "finish_rate": 0.156, "comp_len": 491.8, "t_data_s": 0.0, "t_rollout_s": 33.0, "t_step_s": 69.6, "t_refresh_s": 0.3, "mem_gb": 10.43}
+{"step": 80, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.2668868999974181, "tokens": 120000, "cumulative_loss_tokens": 9600000, "grad_norm": 2.328125, "lr": 3e-05, "finish_rate": 0.149, "comp_len": 497.9, "t_data_s": 0.0, "t_rollout_s": 34.2, "t_step_s": 70.1, "t_refresh_s": 0.3, "mem_gb": 10.4}
+[eval step 80] sample: 'To solve this problem, we need to determine the maximum number of maples that can be planted along the alley given the constraints:\n\n1. The total number of trees is 75.\n2. There are no two maples betw'
+{"step": 80, "gsm8k_n": 64, "gsm8k_quick_chat": 0.390625, "t_eval_s": 44.8}
+{"step": 81, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.27245304357651623, "tokens": 120000, "cumulative_loss_tokens": 9720000, "grad_norm": 1.46875, "lr": 3e-05, "finish_rate": 0.097, "comp_len": 504.2, "t_data_s": 0.0, "t_rollout_s": 35.7, "t_step_s": 71.9, "t_refresh_s": 0.3, "mem_gb": 12.44}
+{"step": 82, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.26285181757311027, "tokens": 120000, "cumulative_loss_tokens": 9840000, "grad_norm": 1.1875, "lr": 3e-05, "finish_rate": 0.117, "comp_len": 500.0, "t_data_s": 0.0, "t_rollout_s": 35.5, "t_step_s": 71.6, "t_refresh_s": 0.3, "mem_gb": 10.42}
+{"step": 83, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.2504500327867145, "tokens": 120000, "cumulative_loss_tokens": 9960000, "grad_norm": 1.390625, "lr": 3e-05, "finish_rate": 0.157, "comp_len": 495.9, "t_data_s": 0.0, "t_rollout_s": 34.4, "t_step_s": 71.2, "t_refresh_s": 0.3, "mem_gb": 10.46}
+{"step": 84, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.25419983249952394, "tokens": 120000, "cumulative_loss_tokens": 10080000, "grad_norm": 1.640625, "lr": 3e-05, "finish_rate": 0.165, "comp_len": 495.9, "t_data_s": 0.0, "t_rollout_s": 33.9, "t_step_s": 70.4, "t_refresh_s": 0.3, "mem_gb": 10.43}
+{"step": 85, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.24969746678496402, "tokens": 120000, "cumulative_loss_tokens": 10200000, "grad_norm": 1.3203125, "lr": 3e-05, "finish_rate": 0.117, "comp_len": 502.1, "t_data_s": 0.0, "t_rollout_s": 35.8, "t_step_s": 73.1, "t_refresh_s": 0.3, "mem_gb": 10.47}
+{"step": 86, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.26418633902855215, "tokens": 120000, "cumulative_loss_tokens": 10320000, "grad_norm": 1.3984375, "lr": 3e-05, "finish_rate": 0.172, "comp_len": 491.8, "t_data_s": 0.0, "t_rollout_s": 34.2, "t_step_s": 71.4, "t_refresh_s": 0.3, "mem_gb": 10.48}
+{"step": 87, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.25853364124981065, "tokens": 120000, "cumulative_loss_tokens": 10440000, "grad_norm": 1.3515625, "lr": 3e-05, "finish_rate": 0.137, "comp_len": 497.9, "t_data_s": 0.0, "t_rollout_s": 34.3, "t_step_s": 70.2, "t_refresh_s": 0.3, "mem_gb": 10.4}
+{"step": 88, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.26424587182166676, "tokens": 120000, "cumulative_loss_tokens": 10560000, "grad_norm": 1.328125, "lr": 3e-05, "finish_rate": 0.132, "comp_len": 495.9, "t_data_s": 0.0, "t_rollout_s": 34.2, "t_step_s": 70.9, "t_refresh_s": 0.3, "mem_gb": 10.48}
+{"step": 89, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.247395279224962, "tokens": 120000, "cumulative_loss_tokens": 10680000, "grad_norm": 1.2109375, "lr": 3e-05, "finish_rate": 0.12, "comp_len": 497.9, "t_data_s": 0.0, "t_rollout_s": 34.5, "t_step_s": 70.4, "t_refresh_s": 0.3, "mem_gb": 10.37}
+{"step": 90, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.27036167939615746, "tokens": 120000, "cumulative_loss_tokens": 10800000, "grad_norm": 1.3359375, "lr": 3e-05, "finish_rate": 0.112, "comp_len": 500.0, "t_data_s": 0.0, "t_rollout_s": 34.6, "t_step_s": 71.2, "t_refresh_s": 0.3, "mem_gb": 10.53}
+[eval step 90] sample: "To solve this problem, we need to maximize the number of maples \\( M \\) such that the total number of trees \\( T \\) is 75, and there are no two maples between which there are exactly 5 trees.\n\nLet's b"
+{"step": 90, "gsm8k_n": 64, "gsm8k_quick_chat": 0.359375, "t_eval_s": 44.7}
+{"step": 91, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.2520888489575436, "tokens": 120000, "cumulative_loss_tokens": 10920000, "grad_norm": 1.203125, "lr": 3e-05, "finish_rate": 0.216, "comp_len": 489.8, "t_data_s": 0.0, "t_rollout_s": 33.6, "t_step_s": 70.4, "t_refresh_s": 0.3, "mem_gb": 12.44}
+{"step": 92, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.23066797492839397, "tokens": 120000, "cumulative_loss_tokens": 11040000, "grad_norm": 1.2421875, "lr": 3e-05, "finish_rate": 0.088, "comp_len": 504.2, "t_data_s": 0.0, "t_rollout_s": 34.6, "t_step_s": 70.4, "t_refresh_s": 0.3, "mem_gb": 10.43}
+{"step": 93, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.23549754046387972, "tokens": 120000, "cumulative_loss_tokens": 11160000, "grad_norm": 1.3125, "lr": 3e-05, "finish_rate": 0.088, "comp_len": 502.1, "t_data_s": 0.0, "t_rollout_s": 37.7, "t_step_s": 74.4, "t_refresh_s": 0.3, "mem_gb": 10.54}
+{"step": 94, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.2587053025153776, "tokens": 120000, "cumulative_loss_tokens": 11280000, "grad_norm": 1.40625, "lr": 3e-05, "finish_rate": 0.126, "comp_len": 502.1, "t_data_s": 0.0, "t_rollout_s": 35.1, "t_step_s": 71.5, "t_refresh_s": 0.3, "mem_gb": 10.45}
+{"step": 95, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.25772884955219927, "tokens": 120000, "cumulative_loss_tokens": 11400000, "grad_norm": 1.2265625, "lr": 3e-05, "finish_rate": 0.059, "comp_len": 506.3, "t_data_s": 0.0, "t_rollout_s": 37.7, "t_step_s": 74.7, "t_refresh_s": 0.3, "mem_gb": 10.52}
+{"step": 96, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.2583792527654519, "tokens": 120000, "cumulative_loss_tokens": 11520000, "grad_norm": 1.296875, "lr": 3e-05, "finish_rate": 0.145, "comp_len": 497.9, "t_data_s": 0.0, "t_rollout_s": 33.8, "t_step_s": 70.6, "t_refresh_s": 0.3, "mem_gb": 10.47}
+{"step": 97, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.2516133760511875, "tokens": 120000, "cumulative_loss_tokens": 11640000, "grad_norm": 1.3984375, "lr": 3e-05, "finish_rate": 0.174, "comp_len": 495.9, "t_data_s": 0.0, "t_rollout_s": 33.8, "t_step_s": 71.0, "t_refresh_s": 0.3, "mem_gb": 10.56}
+{"step": 98, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.2511387197598815, "tokens": 120000, "cumulative_loss_tokens": 11760000, "grad_norm": 1.25, "lr": 3e-05, "finish_rate": 0.083, "comp_len": 500.0, "t_data_s": 0.0, "t_rollout_s": 36.0, "t_step_s": 72.5, "t_refresh_s": 0.3, "mem_gb": 10.5}
+{"step": 99, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.25845928863280765, "tokens": 120000, "cumulative_loss_tokens": 11880000, "grad_norm": 1.328125, "lr": 3e-05, "finish_rate": 0.121, "comp_len": 500.0, "t_data_s": 0.0, "t_rollout_s": 35.0, "t_step_s": 71.0, "t_refresh_s": 0.3, "mem_gb": 10.4}
+{"step": 100, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.2377076770828416, "tokens": 120000, "cumulative_loss_tokens": 12000000, "grad_norm": 1.203125, "lr": 3e-05, "finish_rate": 0.177, "comp_len": 493.8, "t_data_s": 0.0, "t_rollout_s": 32.8, "t_step_s": 69.4, "t_refresh_s": 0.3, "mem_gb": 10.38}
+[eval step 100] sample: 'To solve this problem, we need to maximize the number of maples (M) that can be planted along the alley such that the total number of trees (T) is 75, and there are no two maples between which there a'
+{"step": 100, "gsm8k_n": 64, "gsm8k_quick_chat": 0.328125, "t_eval_s": 44.8}
+checkpoint snapshot queued -> outputs/healed/healing_breadth/glean_math_keep25_seed1224_long/step0100
+{"step": 101, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.2684837412368506, "tokens": 120000, "cumulative_loss_tokens": 12120000, "grad_norm": 1.296875, "lr": 3e-05, "finish_rate": 0.196, "comp_len": 489.8, "t_data_s": 0.0, "t_rollout_s": 33.0, "t_step_s": 69.4, "t_refresh_s": 0.3, "mem_gb": 12.44}
+{"step": 102, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.24094757018235202, "tokens": 120000, "cumulative_loss_tokens": 12240000, "grad_norm": 1.3046875, "lr": 3e-05, "finish_rate": 0.141, "comp_len": 497.9, "t_data_s": 0.0, "t_rollout_s": 34.0, "t_step_s": 70.5, "t_refresh_s": 0.3, "mem_gb": 10.38}
+{"step": 103, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.2634418107945472, "tokens": 120000, "cumulative_loss_tokens": 12360000, "grad_norm": 1.2265625, "lr": 3e-05, "finish_rate": 0.181, "comp_len": 493.8, "t_data_s": 0.0, "t_rollout_s": 33.4, "t_step_s": 70.0, "t_refresh_s": 0.3, "mem_gb": 10.4}
+{"step": 104, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.22899044912308456, "tokens": 120000, "cumulative_loss_tokens": 12480000, "grad_norm": 1.171875, "lr": 3e-05, "finish_rate": 0.136, "comp_len": 495.9, "t_data_s": 0.0, "t_rollout_s": 33.8, "t_step_s": 70.1, "t_refresh_s": 0.3, "mem_gb": 10.38}
+{"step": 105, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.23377793765577176, "tokens": 120000, "cumulative_loss_tokens": 12600000, "grad_norm": 1.203125, "lr": 3e-05, "finish_rate": 0.128, "comp_len": 495.9, "t_data_s": 0.0, "t_rollout_s": 34.9, "t_step_s": 72.2, "t_refresh_s": 0.3, "mem_gb": 10.52}
+{"step": 106, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.23661676298119128, "tokens": 120000, "cumulative_loss_tokens": 12720000, "grad_norm": 1.1484375, "lr": 3e-05, "finish_rate": 0.121, "comp_len": 500.0, "t_data_s": 0.0, "t_rollout_s": 35.6, "t_step_s": 72.4, "t_refresh_s": 0.3, "mem_gb": 10.46}
+{"step": 107, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.22338223065932591, "tokens": 120000, "cumulative_loss_tokens": 12840000, "grad_norm": 1.1328125, "lr": 3e-05, "finish_rate": 0.145, "comp_len": 495.9, "t_data_s": 0.0, "t_rollout_s": 34.3, "t_step_s": 71.2, "t_refresh_s": 0.3, "mem_gb": 10.49}
+{"step": 108, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.24060181738672157, "tokens": 120000, "cumulative_loss_tokens": 12960000, "grad_norm": 1.34375, "lr": 3e-05, "finish_rate": 0.168, "comp_len": 491.8, "t_data_s": 0.0, "t_rollout_s": 33.4, "t_step_s": 70.5, "t_refresh_s": 0.3, "mem_gb": 10.54}
+{"step": 109, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.22999099592032532, "tokens": 120000, "cumulative_loss_tokens": 13080000, "grad_norm": 1.1796875, "lr": 3e-05, "finish_rate": 0.161, "comp_len": 495.9, "t_data_s": 0.0, "t_rollout_s": 33.9, "t_step_s": 70.3, "t_refresh_s": 0.3, "mem_gb": 10.37}
+{"step": 110, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.24095743914954365, "tokens": 120000, "cumulative_loss_tokens": 13200000, "grad_norm": 1.578125, "lr": 3e-05, "finish_rate": 0.112, "comp_len": 497.9, "t_data_s": 0.0, "t_rollout_s": 35.2, "t_step_s": 71.6, "t_refresh_s": 0.3, "mem_gb": 10.48}
+[eval step 110] sample: 'To solve this problem, we need to determine the maximum number of maples (\\(M\\)) that can be planted along the alley such that the total number of trees (\\(T\\)) is 75, and there are no two maples betw'
+{"step": 110, "gsm8k_n": 64, "gsm8k_quick_chat": 0.390625, "t_eval_s": 45.3}
+{"step": 111, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.21752187181736032, "tokens": 120000, "cumulative_loss_tokens": 13320000, "grad_norm": 1.0859375, "lr": 3e-05, "finish_rate": 0.217, "comp_len": 481.9, "t_data_s": 0.0, "t_rollout_s": 30.9, "t_step_s": 68.1, "t_refresh_s": 0.3, "mem_gb": 12.44}
+{"step": 112, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.2314429316678395, "tokens": 120000, "cumulative_loss_tokens": 13440000, "grad_norm": 1.3359375, "lr": 3e-05, "finish_rate": 0.12, "comp_len": 497.9, "t_data_s": 0.0, "t_rollout_s": 34.8, "t_step_s": 71.5, "t_refresh_s": 0.3, "mem_gb": 10.52}
+{"step": 113, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.2235619456699739, "tokens": 120000, "cumulative_loss_tokens": 13560000, "grad_norm": 1.1484375, "lr": 3e-05, "finish_rate": 0.137, "comp_len": 497.9, "t_data_s": 0.0, "t_rollout_s": 33.9, "t_step_s": 71.1, "t_refresh_s": 0.3, "mem_gb": 10.55}
+{"step": 114, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.23131392294329903, "tokens": 120000, "cumulative_loss_tokens": 13680000, "grad_norm": 1.234375, "lr": 3e-05, "finish_rate": 0.124, "comp_len": 495.9, "t_data_s": 0.0, "t_rollout_s": 34.3, "t_step_s": 71.2, "t_refresh_s": 0.3, "mem_gb": 10.38}
+{"step": 115, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.22863815203296642, "tokens": 120000, "cumulative_loss_tokens": 13800000, "grad_norm": 1.2578125, "lr": 3e-05, "finish_rate": 0.104, "comp_len": 497.9, "t_data_s": 0.0, "t_rollout_s": 34.9, "t_step_s": 71.7, "t_refresh_s": 0.3, "mem_gb": 10.4}
+{"step": 116, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.22345312169020376, "tokens": 120000, "cumulative_loss_tokens": 13920000, "grad_norm": 1.3203125, "lr": 3e-05, "finish_rate": 0.125, "comp_len": 500.0, "t_data_s": 0.0, "t_rollout_s": 35.2, "t_step_s": 72.0, "t_refresh_s": 0.3, "mem_gb": 10.48}
+{"step": 117, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.22978670983004074, "tokens": 120000, "cumulative_loss_tokens": 14040000, "grad_norm": 1.1328125, "lr": 3e-05, "finish_rate": 0.108, "comp_len": 497.9, "t_data_s": 0.0, "t_rollout_s": 35.8, "t_step_s": 73.0, "t_refresh_s": 0.3, "mem_gb": 10.52}
+{"step": 118, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.2331024925402055, "tokens": 120000, "cumulative_loss_tokens": 14160000, "grad_norm": 1.28125, "lr": 3e-05, "finish_rate": 0.076, "comp_len": 504.2, "t_data_s": 0.0, "t_rollout_s": 37.3, "t_step_s": 73.9, "t_refresh_s": 0.3, "mem_gb": 10.46}
+{"step": 119, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.22393636475646247, "tokens": 120000, "cumulative_loss_tokens": 14280000, "grad_norm": 1.234375, "lr": 3e-05, "finish_rate": 0.13, "comp_len": 502.1, "t_data_s": 0.0, "t_rollout_s": 35.4, "t_step_s": 71.5, "t_refresh_s": 0.3, "mem_gb": 10.44}
+{"step": 120, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.2355032395929719, "tokens": 120000, "cumulative_loss_tokens": 14400000, "grad_norm": 1.203125, "lr": 3e-05, "finish_rate": 0.16, "comp_len": 491.8, "t_data_s": 0.0, "t_rollout_s": 33.1, "t_step_s": 68.8, "t_refresh_s": 0.3, "mem_gb": 10.36}
+[eval step 120] sample: 'To solve this problem, we need to determine the maximum number of maples that can be planted along the alley given the constraints:\n\n1. The total number of trees is 75.\n2. There are no two maples betw'
+{"step": 120, "gsm8k_n": 64, "gsm8k_quick_chat": 0.34375, "t_eval_s": 44.9}
+{"step": 121, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.24267969185064237, "tokens": 120000, "cumulative_loss_tokens": 14520000, "grad_norm": 1.171875, "lr": 3e-05, "finish_rate": 0.113, "comp_len": 502.1, "t_data_s": 0.0, "t_rollout_s": 34.9, "t_step_s": 71.1, "t_refresh_s": 0.3, "mem_gb": 12.43}
+{"step": 122, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.23425059191025793, "tokens": 120000, "cumulative_loss_tokens": 14640000, "grad_norm": 1.1640625, "lr": 3e-05, "finish_rate": 0.093, "comp_len": 506.3, "t_data_s": 0.0, "t_rollout_s": 37.0, "t_step_s": 74.2, "t_refresh_s": 0.3, "mem_gb": 10.53}
+{"step": 123, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.26844764651320874, "tokens": 120000, "cumulative_loss_tokens": 14760000, "grad_norm": 1.359375, "lr": 3e-05, "finish_rate": 0.096, "comp_len": 500.0, "t_data_s": 0.0, "t_rollout_s": 35.0, "t_step_s": 71.1, "t_refresh_s": 0.3, "mem_gb": 10.47}
+{"step": 124, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.21723629867484173, "tokens": 120000, "cumulative_loss_tokens": 14880000, "grad_norm": 1.2578125, "lr": 3e-05, "finish_rate": 0.116, "comp_len": 497.9, "t_data_s": 0.0, "t_rollout_s": 33.6, "t_step_s": 69.0, "t_refresh_s": 0.3, "mem_gb": 10.37}
+{"step": 125, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.21943003199926267, "tokens": 120000, "cumulative_loss_tokens": 15000000, "grad_norm": 1.3515625, "lr": 3e-05, "finish_rate": 0.14, "comp_len": 495.9, "t_data_s": 0.0, "t_rollout_s": 34.5, "t_step_s": 70.8, "t_refresh_s": 0.3, "mem_gb": 10.49}
+{"step": 126, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.214982470301042, "tokens": 120000, "cumulative_loss_tokens": 15120000, "grad_norm": 1.171875, "lr": 3e-05, "finish_rate": 0.13, "comp_len": 502.1, "t_data_s": 0.0, "t_rollout_s": 34.9, "t_step_s": 70.3, "t_refresh_s": 0.3, "mem_gb": 10.37}
+{"step": 127, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.22647308003318806, "tokens": 120000, "cumulative_loss_tokens": 15240000, "grad_norm": 1.34375, "lr": 3e-05, "finish_rate": 0.105, "comp_len": 502.1, "t_data_s": 0.0, "t_rollout_s": 35.0, "t_step_s": 71.0, "t_refresh_s": 0.3, "mem_gb": 10.38}
+{"step": 128, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.23893264475651085, "tokens": 120000, "cumulative_loss_tokens": 15360000, "grad_norm": 1.4140625, "lr": 3e-05, "finish_rate": 0.104, "comp_len": 500.0, "t_data_s": 0.0, "t_rollout_s": 35.3, "t_step_s": 71.4, "t_refresh_s": 0.3, "mem_gb": 10.5}
+{"step": 129, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.23907007528046767, "tokens": 120000, "cumulative_loss_tokens": 15480000, "grad_norm": 1.1796875, "lr": 3e-05, "finish_rate": 0.127, "comp_len": 491.8, "t_data_s": 0.0, "t_rollout_s": 31.7, "t_step_s": 69.0, "t_refresh_s": 0.3, "mem_gb": 10.47}
+{"step": 130, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.23370486757767697, "tokens": 120000, "cumulative_loss_tokens": 15600000, "grad_norm": 1.2421875, "lr": 3e-05, "finish_rate": 0.139, "comp_len": 489.8, "t_data_s": 0.0, "t_rollout_s": 32.2, "t_step_s": 68.9, "t_refresh_s": 0.3, "mem_gb": 10.43}
+[eval step 130] sample: 'To solve this problem, we need to determine the maximum number of maples that can be planted along the alley given the constraints:\n\n1. The total number of trees is 75.\n2. There are no two maples betw'
+{"step": 130, "gsm8k_n": 64, "gsm8k_quick_chat": 0.375, "t_eval_s": 45.3}
+{"step": 131, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.22123020526878537, "tokens": 120000, "cumulative_loss_tokens": 15720000, "grad_norm": 1.2421875, "lr": 3e-05, "finish_rate": 0.088, "comp_len": 504.2, "t_data_s": 0.0, "t_rollout_s": 36.5, "t_step_s": 73.1, "t_refresh_s": 0.3, "mem_gb": 12.44}
+{"step": 132, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.22262640608406314, "tokens": 120000, "cumulative_loss_tokens": 15840000, "grad_norm": 1.203125, "lr": 3e-05, "finish_rate": 0.129, "comp_len": 500.0, "t_data_s": 0.0, "t_rollout_s": 33.3, "t_step_s": 69.3, "t_refresh_s": 0.3, "mem_gb": 10.38}
+{"step": 133, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.23642245450820773, "tokens": 120000, "cumulative_loss_tokens": 15960000, "grad_norm": 1.3125, "lr": 3e-05, "finish_rate": 0.141, "comp_len": 497.9, "t_data_s": 0.0, "t_rollout_s": 33.9, "t_step_s": 70.1, "t_refresh_s": 0.3, "mem_gb": 10.5}
+{"step": 134, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.22181539568305014, "tokens": 120000, "cumulative_loss_tokens": 16080000, "grad_norm": 1.1484375, "lr": 3e-05, "finish_rate": 0.124, "comp_len": 497.9, "t_data_s": 0.0, "t_rollout_s": 35.1, "t_step_s": 71.9, "t_refresh_s": 0.3, "mem_gb": 10.49}
+{"step": 135, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.21886845049690457, "tokens": 120000, "cumulative_loss_tokens": 16200000, "grad_norm": 1.265625, "lr": 3e-05, "finish_rate": 0.088, "comp_len": 502.1, "t_data_s": 0.0, "t_rollout_s": 35.4, "t_step_s": 71.3, "t_refresh_s": 0.3, "mem_gb": 10.43}
+{"step": 136, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.23528965907134117, "tokens": 120000, "cumulative_loss_tokens": 16320000, "grad_norm": 1.2421875, "lr": 3e-05, "finish_rate": 0.117, "comp_len": 500.0, "t_data_s": 0.0, "t_rollout_s": 35.0, "t_step_s": 71.1, "t_refresh_s": 0.3, "mem_gb": 10.45}
+{"step": 137, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.22028850772418082, "tokens": 120000, "cumulative_loss_tokens": 16440000, "grad_norm": 1.3828125, "lr": 3e-05, "finish_rate": 0.076, "comp_len": 504.2, "t_data_s": 0.0, "t_rollout_s": 36.0, "t_step_s": 71.8, "t_refresh_s": 0.3, "mem_gb": 10.47}
+{"step": 138, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.22388356751191119, "tokens": 120000, "cumulative_loss_tokens": 16560000, "grad_norm": 1.171875, "lr": 3e-05, "finish_rate": 0.123, "comp_len": 493.8, "t_data_s": 0.0, "t_rollout_s": 33.4, "t_step_s": 69.5, "t_refresh_s": 0.3, "mem_gb": 10.38}
+{"step": 139, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.2186046968769282, "tokens": 120000, "cumulative_loss_tokens": 16680000, "grad_norm": 1.1171875, "lr": 3e-05, "finish_rate": 0.084, "comp_len": 504.2, "t_data_s": 0.0, "t_rollout_s": 36.2, "t_step_s": 72.8, "t_refresh_s": 0.3, "mem_gb": 10.46}
+{"step": 140, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.21518618610985576, "tokens": 120000, "cumulative_loss_tokens": 16800000, "grad_norm": 1.1171875, "lr": 3e-05, "finish_rate": 0.121, "comp_len": 500.0, "t_data_s": 0.0, "t_rollout_s": 33.9, "t_step_s": 69.8, "t_refresh_s": 0.3, "mem_gb": 10.44}
+[eval step 140] sample: 'To solve this problem, we need to carefully analyze the constraints given:\n\n1. **Total number of trees:** 75 trees.\n2. **No two maples between which there are exactly 5 trees:** This means that no two'
+{"step": 140, "gsm8k_n": 64, "gsm8k_quick_chat": 0.359375, "t_eval_s": 44.8}
+{"step": 141, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.21726824556073795, "tokens": 120000, "cumulative_loss_tokens": 16920000, "grad_norm": 1.1953125, "lr": 3e-05, "finish_rate": 0.072, "comp_len": 506.3, "t_data_s": 0.0, "t_rollout_s": 37.1, "t_step_s": 73.1, "t_refresh_s": 0.3, "mem_gb": 12.43}
+{"step": 142, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.21776566527485847, "tokens": 120000, "cumulative_loss_tokens": 17040000, "grad_norm": 1.2109375, "lr": 3e-05, "finish_rate": 0.104, "comp_len": 500.0, "t_data_s": 0.0, "t_rollout_s": 33.9, "t_step_s": 70.1, "t_refresh_s": 0.3, "mem_gb": 10.4}
+{"step": 143, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.21226181217512738, "tokens": 120000, "cumulative_loss_tokens": 17160000, "grad_norm": 1.234375, "lr": 3e-05, "finish_rate": 0.128, "comp_len": 493.8, "t_data_s": 0.0, "t_rollout_s": 33.6, "t_step_s": 70.3, "t_refresh_s": 0.3, "mem_gb": 10.47}
+{"step": 144, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.20745828753014406, "tokens": 120000, "cumulative_loss_tokens": 17280000, "grad_norm": 1.09375, "lr": 3e-05, "finish_rate": 0.141, "comp_len": 497.9, "t_data_s": 0.0, "t_rollout_s": 35.2, "t_step_s": 71.5, "t_refresh_s": 0.3, "mem_gb": 10.46}
+{"step": 145, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.20095169744286687, "tokens": 120000, "cumulative_loss_tokens": 17400000, "grad_norm": 1.1796875, "lr": 3e-05, "finish_rate": 0.067, "comp_len": 504.2, "t_data_s": 0.0, "t_rollout_s": 37.0, "t_step_s": 72.8, "t_refresh_s": 0.3, "mem_gb": 10.5}
+{"step": 146, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.21430944070120653, "tokens": 120000, "cumulative_loss_tokens": 17520000, "grad_norm": 1.15625, "lr": 3e-05, "finish_rate": 0.092, "comp_len": 502.1, "t_data_s": 0.0, "t_rollout_s": 35.2, "t_step_s": 71.4, "t_refresh_s": 0.3, "mem_gb": 10.41}
+{"step": 147, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.20074411552796761, "tokens": 120000, "cumulative_loss_tokens": 17640000, "grad_norm": 1.1015625, "lr": 3e-05, "finish_rate": 0.092, "comp_len": 502.1, "t_data_s": 0.0, "t_rollout_s": 35.3, "t_step_s": 71.3, "t_refresh_s": 0.3, "mem_gb": 10.51}
+{"step": 148, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.2208368306187292, "tokens": 120000, "cumulative_loss_tokens": 17760000, "grad_norm": 1.1328125, "lr": 3e-05, "finish_rate": 0.148, "comp_len": 493.8, "t_data_s": 0.0, "t_rollout_s": 33.4, "t_step_s": 69.9, "t_refresh_s": 0.3, "mem_gb": 10.4}
+{"step": 149, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.22132619506145518, "tokens": 120000, "cumulative_loss_tokens": 17880000, "grad_norm": 1.265625, "lr": 3e-05, "finish_rate": 0.108, "comp_len": 500.0, "t_data_s": 0.0, "t_rollout_s": 36.8, "t_step_s": 74.1, "t_refresh_s": 0.3, "mem_gb": 10.47}
+{"step": 150, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.2195085643796871, "tokens": 120000, "cumulative_loss_tokens": 18000000, "grad_norm": 1.1328125, "lr": 3e-05, "finish_rate": 0.112, "comp_len": 500.0, "t_data_s": 0.0, "t_rollout_s": 36.1, "t_step_s": 73.3, "t_refresh_s": 0.3, "mem_gb": 10.53}
+[eval step 150] sample: "To solve this problem, we need to determine the maximum number of maples that can be placed along the alley such that there are no two maples between which there are exactly 5 trees.\n\nLet's break down"
+{"step": 150, "gsm8k_n": 64, "gsm8k_quick_chat": 0.3125, "t_eval_s": 44.9}
+checkpoint snapshot queued -> outputs/healed/healing_breadth/glean_math_keep25_seed1224_long/step0150
+{"step": 151, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.21052663061742982, "tokens": 120000, "cumulative_loss_tokens": 18120000, "grad_norm": 1.234375, "lr": 3e-05, "finish_rate": 0.1, "comp_len": 502.1, "t_data_s": 0.0, "t_rollout_s": 35.0, "t_step_s": 70.8, "t_refresh_s": 0.3, "mem_gb": 12.43}
+{"step": 152, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.23573231500076752, "tokens": 120000, "cumulative_loss_tokens": 18240000, "grad_norm": 1.140625, "lr": 3e-05, "finish_rate": 0.076, "comp_len": 504.2, "t_data_s": 0.0, "t_rollout_s": 37.6, "t_step_s": 74.7, "t_refresh_s": 0.3, "mem_gb": 10.54}
+{"step": 153, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.21262744003695747, "tokens": 120000, "cumulative_loss_tokens": 18360000, "grad_norm": 1.125, "lr": 3e-05, "finish_rate": 0.152, "comp_len": 493.8, "t_data_s": 0.0, "t_rollout_s": 33.7, "t_step_s": 70.2, "t_refresh_s": 0.3, "mem_gb": 10.46}
+{"step": 154, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.20737704386965683, "tokens": 120000, "cumulative_loss_tokens": 18480000, "grad_norm": 1.125, "lr": 3e-05, "finish_rate": 0.155, "comp_len": 489.8, "t_data_s": 0.0, "t_rollout_s": 32.1, "t_step_s": 69.1, "t_refresh_s": 0.3, "mem_gb": 10.4}
+{"step": 155, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.20515705520591387, "tokens": 120000, "cumulative_loss_tokens": 18600000, "grad_norm": 1.125, "lr": 3e-05, "finish_rate": 0.125, "comp_len": 500.0, "t_data_s": 0.0, "t_rollout_s": 34.8, "t_step_s": 72.9, "t_refresh_s": 0.3, "mem_gb": 10.47}
+{"step": 156, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.227416008350874, "tokens": 120000, "cumulative_loss_tokens": 18720000, "grad_norm": 1.1640625, "lr": 3e-05, "finish_rate": 0.143, "comp_len": 491.8, "t_data_s": 0.0, "t_rollout_s": 33.7, "t_step_s": 70.4, "t_refresh_s": 0.3, "mem_gb": 10.46}
+{"step": 157, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.2179569326767077, "tokens": 120000, "cumulative_loss_tokens": 18840000, "grad_norm": 1.1953125, "lr": 3e-05, "finish_rate": 0.116, "comp_len": 497.9, "t_data_s": 0.0, "t_rollout_s": 34.7, "t_step_s": 70.6, "t_refresh_s": 0.3, "mem_gb": 10.37}
+{"step": 158, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.21790637281499803, "tokens": 120000, "cumulative_loss_tokens": 18960000, "grad_norm": 1.1953125, "lr": 3e-05, "finish_rate": 0.12, "comp_len": 495.9, "t_data_s": 0.0, "t_rollout_s": 33.5, "t_step_s": 69.6, "t_refresh_s": 0.3, "mem_gb": 10.37}
+{"step": 159, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.21037111974650374, "tokens": 120000, "cumulative_loss_tokens": 19080000, "grad_norm": 1.25, "lr": 3e-05, "finish_rate": 0.124, "comp_len": 497.9, "t_data_s": 0.0, "t_rollout_s": 34.3, "t_step_s": 70.5, "t_refresh_s": 0.3, "mem_gb": 10.39}
+{"step": 160, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.22135015249016385, "tokens": 120000, "cumulative_loss_tokens": 19200000, "grad_norm": 1.234375, "lr": 3e-05, "finish_rate": 0.1, "comp_len": 502.1, "t_data_s": 0.0, "t_rollout_s": 35.2, "t_step_s": 72.0, "t_refresh_s": 0.3, "mem_gb": 10.44}
+[eval step 160] sample: "To solve this problem, we need to determine the maximum number of maples that can be placed along the alley such that there are no two maples between which there are exactly 5 trees.\n\nLet's break down"
+{"step": 160, "gsm8k_n": 64, "gsm8k_quick_chat": 0.375, "t_eval_s": 44.8}
+{"step": 161, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.19312054982806245, "tokens": 120000, "cumulative_loss_tokens": 19320000, "grad_norm": 1.078125, "lr": 3e-05, "finish_rate": 0.076, "comp_len": 506.3, "t_data_s": 0.0, "t_rollout_s": 37.0, "t_step_s": 73.0, "t_refresh_s": 0.3, "mem_gb": 12.43}
+{"step": 162, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.21525976048012574, "tokens": 120000, "cumulative_loss_tokens": 19440000, "grad_norm": 1.2734375, "lr": 3e-05, "finish_rate": 0.177, "comp_len": 493.8, "t_data_s": 0.0, "t_rollout_s": 33.4, "t_step_s": 69.6, "t_refresh_s": 0.3, "mem_gb": 10.38}
+{"step": 163, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.21099251879341901, "tokens": 120000, "cumulative_loss_tokens": 19560000, "grad_norm": 1.171875, "lr": 3e-05, "finish_rate": 0.108, "comp_len": 500.0, "t_data_s": 0.0, "t_rollout_s": 35.3, "t_step_s": 71.9, "t_refresh_s": 0.3, "mem_gb": 10.41}
+{"step": 164, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.2188319661433498, "tokens": 120000, "cumulative_loss_tokens": 19680000, "grad_norm": 1.1953125, "lr": 3e-05, "finish_rate": 0.136, "comp_len": 495.9, "t_data_s": 0.0, "t_rollout_s": 33.7, "t_step_s": 70.1, "t_refresh_s": 0.3, "mem_gb": 10.47}
+{"step": 165, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.21699987841347854, "tokens": 120000, "cumulative_loss_tokens": 19800000, "grad_norm": 1.1640625, "lr": 3e-05, "finish_rate": 0.137, "comp_len": 497.9, "t_data_s": 0.0, "t_rollout_s": 34.7, "t_step_s": 72.0, "t_refresh_s": 0.3, "mem_gb": 10.54}
+{"step": 166, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.20422210597346227, "tokens": 120000, "cumulative_loss_tokens": 19920000, "grad_norm": 1.1875, "lr": 3e-05, "finish_rate": 0.088, "comp_len": 502.1, "t_data_s": 0.0, "t_rollout_s": 34.4, "t_step_s": 69.9, "t_refresh_s": 0.3, "mem_gb": 10.36}
+{"step": 167, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.21616086491048336, "tokens": 120000, "cumulative_loss_tokens": 20040000, "grad_norm": 1.109375, "lr": 3e-05, "finish_rate": 0.108, "comp_len": 500.0, "t_data_s": 0.0, "t_rollout_s": 35.9, "t_step_s": 72.8, "t_refresh_s": 0.3, "mem_gb": 10.49}
+{"step": 168, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.2041427251155178, "tokens": 120000, "cumulative_loss_tokens": 20160000, "grad_norm": 1.1796875, "lr": 3e-05, "finish_rate": 0.117, "comp_len": 500.0, "t_data_s": 0.0, "t_rollout_s": 35.2, "t_step_s": 71.1, "t_refresh_s": 0.3, "mem_gb": 10.39}
+{"step": 169, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.2007420184203113, "tokens": 120000, "cumulative_loss_tokens": 20280000, "grad_norm": 1.0546875, "lr": 3e-05, "finish_rate": 0.113, "comp_len": 504.2, "t_data_s": 0.0, "t_rollout_s": 36.2, "t_step_s": 72.4, "t_refresh_s": 0.3, "mem_gb": 10.51}
+{"step": 170, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.22800080033155778, "tokens": 120000, "cumulative_loss_tokens": 20400000, "grad_norm": 1.40625, "lr": 3e-05, "finish_rate": 0.153, "comp_len": 495.9, "t_data_s": 0.0, "t_rollout_s": 33.6, "t_step_s": 69.9, "t_refresh_s": 0.3, "mem_gb": 10.42}
+[eval step 170] sample: 'To solve this problem, we need to carefully analyze the constraints and use reasoning to determine the maximum number of maples that can be planted.\n\n### Problem Breakdown:\n\n1. **Total Number of Trees'
+{"step": 170, "gsm8k_n": 64, "gsm8k_quick_chat": 0.34375, "t_eval_s": 44.8}
+{"step": 171, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.20762226390354335, "tokens": 120000, "cumulative_loss_tokens": 20520000, "grad_norm": 1.2421875, "lr": 3e-05, "finish_rate": 0.112, "comp_len": 497.9, "t_data_s": 0.0, "t_rollout_s": 34.9, "t_step_s": 72.1, "t_refresh_s": 0.3, "mem_gb": 12.43}
+{"step": 172, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.19745559071643898, "tokens": 120000, "cumulative_loss_tokens": 20640000, "grad_norm": 1.1875, "lr": 3e-05, "finish_rate": 0.145, "comp_len": 495.9, "t_data_s": 0.0, "t_rollout_s": 33.4, "t_step_s": 69.5, "t_refresh_s": 0.3, "mem_gb": 10.38}

healed/healing_breadth/keep25_long.queue.log

@@ -0,0 +1 @@
+2026-07-15T06:33:26-07:00 resuming keep25 at step 50; target step 500 (60000000 cumulative tokens)

healed/healing_breadth/reap_math_keep75_seed1224.console.log

@@ -0,0 +1,113 @@
+/home/henry/Documents/PythonProjects/variable-reap/.venv/lib/python3.12/site-packages/megablocks/grouped_gemm_util.py:10: UserWarning: Grouped GEMM not available.
+  warnings.warn('Grouped GEMM not available.')
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from /home/henry/.netrc.
+wandb: Currently logged in as: hbfreed to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: Tracking run with wandb version 0.28.0
+wandb: Run data is saved locally in outputs/healed/healing_breadth/reap_math_keep75_seed1224/wandb/run-20260714_215914-wghixdgk
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run heal-reap_math_keep75-seed1224-run1
+wandb: ⭐️ View project at https://wandb.ai/hbfreed/glean-heal
+wandb: 🚀 View run at https://wandb.ai/hbfreed/glean-heal/runs/wghixdgk
+
+
+starting vllm rollout server on GPU GPU-864c54df-0130-7780-e271-8a5551d1733f (port 8377) ...
+vllm server healthy in 24s
+dataset_source filter ['omega', 'polaris', 'orz_math', 'mathsub', 'dapo-math'] -> 63998 prompts
+
+difficulty <= 4 -> 63998 prompts
+WARNING: difficulty filter removed nothing — the slice likely carries difficulty=None (Dolci math sources do), so the teacher-competence guard is NOT in effect
+
+
+63977 prompts | 940 steps/epoch | 50 total steps | student params 5.31B | teacher overlap=True
+{"step": 1, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.22007248604670168, "tokens": 120000, "cumulative_loss_tokens": 120000, "grad_norm": 4.15625, "lr": 6e-06, "finish_rate": 0.079, "comp_len": 502.1, "t_data_s": 0.0, "t_rollout_s": 31.9, "t_step_s": 123.1, "t_refresh_s": 2.7, "mem_gb": 21.1}
+The attention mask is not set and cannot be inferred from input because pad token is same as eos token. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.
+[eval step 1] sample: 'To solve this problem, we need to maximize the number of maples \\( m \\) while ensuring that no two maples are exactly 5 trees apart.\n\nGiven:\n1. Total number of trees \\( m + l = 75 \\)\n2. No two maples '
+{"step": 2, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.19900615173100183, "tokens": 120000, "cumulative_loss_tokens": 240000, "grad_norm": 3.890625, "lr": 9e-06, "finish_rate": 0.1, "comp_len": 500.0, "t_data_s": 0.0, "t_rollout_s": 32.2, "t_step_s": 105.8, "t_refresh_s": 2.7, "mem_gb": 21.25}
+{"step": 3, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.20272269249390812, "tokens": 120000, "cumulative_loss_tokens": 360000, "grad_norm": 3.609375, "lr": 1.2e-05, "finish_rate": 0.112, "comp_len": 497.9, "t_data_s": 0.0, "t_rollout_s": 32.7, "t_step_s": 106.0, "t_refresh_s": 2.7, "mem_gb": 21.29}
+{"step": 4, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.16433406947404147, "tokens": 120000, "cumulative_loss_tokens": 480000, "grad_norm": 2.453125, "lr": 1.5e-05, "finish_rate": 0.156, "comp_len": 493.8, "t_data_s": 0.0, "t_rollout_s": 31.4, "t_step_s": 102.7, "t_refresh_s": 2.7, "mem_gb": 21.27}
+{"step": 5, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.13169451138923566, "tokens": 120000, "cumulative_loss_tokens": 600000, "grad_norm": 1.84375, "lr": 1.8e-05, "finish_rate": 0.129, "comp_len": 500.0, "t_data_s": 0.0, "t_rollout_s": 31.5, "t_step_s": 101.5, "t_refresh_s": 2.7, "mem_gb": 21.26}
+{"step": 6, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.11568032757478455, "tokens": 120000, "cumulative_loss_tokens": 720000, "grad_norm": 2.140625, "lr": 2.1e-05, "finish_rate": 0.176, "comp_len": 489.8, "t_data_s": 0.0, "t_rollout_s": 31.3, "t_step_s": 103.0, "t_refresh_s": 2.7, "mem_gb": 21.21}
+{"step": 7, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.09576526515452811, "tokens": 120000, "cumulative_loss_tokens": 840000, "grad_norm": 1.296875, "lr": 2.4e-05, "finish_rate": 0.23, "comp_len": 491.8, "t_data_s": 0.0, "t_rollout_s": 30.8, "t_step_s": 101.4, "t_refresh_s": 2.7, "mem_gb": 21.21}
+{"step": 8, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.08849306978667155, "tokens": 120000, "cumulative_loss_tokens": 960000, "grad_norm": 1.671875, "lr": 2.7000000000000002e-05, "finish_rate": 0.199, "comp_len": 487.8, "t_data_s": 0.0, "t_rollout_s": 31.2, "t_step_s": 103.4, "t_refresh_s": 2.7, "mem_gb": 21.25}
+{"step": 9, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.08143095947749292, "tokens": 120000, "cumulative_loss_tokens": 1080000, "grad_norm": 1.1953125, "lr": 3e-05, "finish_rate": 0.202, "comp_len": 485.8, "t_data_s": 0.0, "t_rollout_s": 31.0, "t_step_s": 103.3, "t_refresh_s": 2.7, "mem_gb": 21.21}
+{"step": 10, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.0782286991564557, "tokens": 120000, "cumulative_loss_tokens": 1200000, "grad_norm": 0.94140625, "lr": 3e-05, "finish_rate": 0.143, "comp_len": 489.8, "t_data_s": 0.0, "t_rollout_s": 31.6, "t_step_s": 102.8, "t_refresh_s": 2.7, "mem_gb": 21.29}
+[eval step 10] sample: "To solve this problem, we need to maximize the number of maples \\( m \\) such that there are no two maples exactly 5 trees apart. Let's denote the positions of the maples as \\( x_1, x_2, \\ldots, x_m \\)"
+{"step": 11, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.08126618308176597, "tokens": 120000, "cumulative_loss_tokens": 1320000, "grad_norm": 0.9453125, "lr": 3e-05, "finish_rate": 0.196, "comp_len": 480.0, "t_data_s": 0.0, "t_rollout_s": 30.9, "t_step_s": 103.0, "t_refresh_s": 2.7, "mem_gb": 21.21}
+{"step": 12, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.07858114459690017, "tokens": 120000, "cumulative_loss_tokens": 1440000, "grad_norm": 0.8828125, "lr": 3e-05, "finish_rate": 0.184, "comp_len": 480.0, "t_data_s": 0.0, "t_rollout_s": 30.9, "t_step_s": 103.6, "t_refresh_s": 2.7, "mem_gb": 21.28}
+{"step": 13, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.06576858285398533, "tokens": 120000, "cumulative_loss_tokens": 1560000, "grad_norm": 0.7890625, "lr": 3e-05, "finish_rate": 0.164, "comp_len": 491.8, "t_data_s": 0.0, "t_rollout_s": 31.4, "t_step_s": 102.8, "t_refresh_s": 2.7, "mem_gb": 21.23}
+{"step": 14, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.07604408309549714, "tokens": 120000, "cumulative_loss_tokens": 1680000, "grad_norm": 0.96484375, "lr": 3e-05, "finish_rate": 0.213, "comp_len": 491.8, "t_data_s": 0.0, "t_rollout_s": 31.2, "t_step_s": 101.5, "t_refresh_s": 2.7, "mem_gb": 21.25}
+{"step": 15, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.0675725906633772, "tokens": 120000, "cumulative_loss_tokens": 1800000, "grad_norm": 0.8125, "lr": 3e-05, "finish_rate": 0.163, "comp_len": 489.8, "t_data_s": 0.0, "t_rollout_s": 31.7, "t_step_s": 103.9, "t_refresh_s": 2.7, "mem_gb": 21.26}
+{"step": 16, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.06715507488114139, "tokens": 120000, "cumulative_loss_tokens": 1920000, "grad_norm": 0.984375, "lr": 3e-05, "finish_rate": 0.158, "comp_len": 485.8, "t_data_s": 0.0, "t_rollout_s": 31.4, "t_step_s": 103.5, "t_refresh_s": 2.7, "mem_gb": 21.24}
+{"step": 17, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.06346533598719786, "tokens": 120000, "cumulative_loss_tokens": 2040000, "grad_norm": 1.046875, "lr": 3e-05, "finish_rate": 0.183, "comp_len": 487.8, "t_data_s": 0.0, "t_rollout_s": 31.0, "t_step_s": 101.8, "t_refresh_s": 2.7, "mem_gb": 21.25}
+{"step": 18, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.0625511735424865, "tokens": 120000, "cumulative_loss_tokens": 2160000, "grad_norm": 0.8203125, "lr": 3e-05, "finish_rate": 0.169, "comp_len": 493.8, "t_data_s": 0.0, "t_rollout_s": 31.4, "t_step_s": 102.1, "t_refresh_s": 2.7, "mem_gb": 21.28}
+{"step": 19, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.05953531045786416, "tokens": 120000, "cumulative_loss_tokens": 2280000, "grad_norm": 0.73046875, "lr": 3e-05, "finish_rate": 0.148, "comp_len": 493.8, "t_data_s": 0.0, "t_rollout_s": 31.9, "t_step_s": 103.6, "t_refresh_s": 2.7, "mem_gb": 21.26}
+{"step": 20, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.0648774014983947, "tokens": 120000, "cumulative_loss_tokens": 2400000, "grad_norm": 0.859375, "lr": 3e-05, "finish_rate": 0.163, "comp_len": 489.8, "t_data_s": 0.0, "t_rollout_s": 31.6, "t_step_s": 103.0, "t_refresh_s": 2.7, "mem_gb": 21.27}
+[eval step 20] sample: "To solve this problem, we need to maximize the number of maples \\( m \\) while ensuring that there are no two maples that are exactly 5 trees apart. \n\nLet's break down the problem:\n\n1. **Define Variabl"
+{"step": 21, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.057967641365962726, "tokens": 120000, "cumulative_loss_tokens": 2520000, "grad_norm": 1.0625, "lr": 3e-05, "finish_rate": 0.259, "comp_len": 478.1, "t_data_s": 0.0, "t_rollout_s": 30.2, "t_step_s": 102.4, "t_refresh_s": 2.7, "mem_gb": 21.21}
+{"step": 22, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.058268615744728594, "tokens": 120000, "cumulative_loss_tokens": 2640000, "grad_norm": 0.6796875, "lr": 3e-05, "finish_rate": 0.186, "comp_len": 485.8, "t_data_s": 0.0, "t_rollout_s": 31.3, "t_step_s": 103.7, "t_refresh_s": 2.7, "mem_gb": 21.24}
+{"step": 23, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.052534118128793, "tokens": 120000, "cumulative_loss_tokens": 2760000, "grad_norm": 0.6484375, "lr": 3e-05, "finish_rate": 0.153, "comp_len": 495.9, "t_data_s": 0.0, "t_rollout_s": 31.3, "t_step_s": 101.6, "t_refresh_s": 2.7, "mem_gb": 21.24}
+{"step": 24, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.06487227020380087, "tokens": 120000, "cumulative_loss_tokens": 2880000, "grad_norm": 0.8203125, "lr": 3e-05, "finish_rate": 0.129, "comp_len": 497.9, "t_data_s": 0.0, "t_rollout_s": 31.9, "t_step_s": 103.5, "t_refresh_s": 2.7, "mem_gb": 21.26}
+{"step": 25, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.06092730059704433, "tokens": 120000, "cumulative_loss_tokens": 3000000, "grad_norm": 0.75, "lr": 3e-05, "finish_rate": 0.184, "comp_len": 491.8, "t_data_s": 0.0, "t_rollout_s": 31.9, "t_step_s": 103.7, "t_refresh_s": 2.7, "mem_gb": 21.29}
+checkpoint snapshot queued -> outputs/healed/healing_breadth/reap_math_keep75_seed1224/step0025
+{"step": 26, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.062316528513189404, "tokens": 120000, "cumulative_loss_tokens": 3120000, "grad_norm": 1.015625, "lr": 3e-05, "finish_rate": 0.17, "comp_len": 485.8, "t_data_s": 0.0, "t_rollout_s": 31.6, "t_step_s": 104.3, "t_refresh_s": 2.7, "mem_gb": 21.29}
+{"step": 27, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.05730062984156733, "tokens": 120000, "cumulative_loss_tokens": 3240000, "grad_norm": 0.75, "lr": 3e-05, "finish_rate": 0.14, "comp_len": 495.9, "t_data_s": 0.0, "t_rollout_s": 32.3, "t_step_s": 105.0, "t_refresh_s": 2.7, "mem_gb": 21.32}
+{"step": 28, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.05523895585861367, "tokens": 120000, "cumulative_loss_tokens": 3360000, "grad_norm": 0.734375, "lr": 3e-05, "finish_rate": 0.129, "comp_len": 497.9, "t_data_s": 0.0, "t_rollout_s": 31.7, "t_step_s": 102.8, "t_refresh_s": 2.7, "mem_gb": 21.27}
+{"step": 29, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.05680957672783794, "tokens": 120000, "cumulative_loss_tokens": 3480000, "grad_norm": 0.7265625, "lr": 3e-05, "finish_rate": 0.127, "comp_len": 491.8, "t_data_s": 0.0, "t_rollout_s": 32.0, "t_step_s": 104.9, "t_refresh_s": 2.7, "mem_gb": 21.29}
+{"step": 30, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.059886360732543595, "tokens": 120000, "cumulative_loss_tokens": 3600000, "grad_norm": 0.68359375, "lr": 3e-05, "finish_rate": 0.227, "comp_len": 485.8, "t_data_s": 0.0, "t_rollout_s": 30.8, "t_step_s": 102.0, "t_refresh_s": 2.7, "mem_gb": 21.25}
+[eval step 30] sample: "To solve this problem, we need to determine the maximum number of maples (\\(M\\)) that can be planted along the alley such that there are no two maples exactly 5 trees apart.\n\nLet's break down the prob"
+{"step": 31, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.05055032962113619, "tokens": 120000, "cumulative_loss_tokens": 3720000, "grad_norm": 0.640625, "lr": 3e-05, "finish_rate": 0.213, "comp_len": 491.8, "t_data_s": 0.0, "t_rollout_s": 31.0, "t_step_s": 102.2, "t_refresh_s": 2.7, "mem_gb": 21.34}
+{"step": 32, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.05529377434644848, "tokens": 120000, "cumulative_loss_tokens": 3840000, "grad_norm": 0.6796875, "lr": 3e-05, "finish_rate": 0.167, "comp_len": 487.8, "t_data_s": 0.0, "t_rollout_s": 31.4, "t_step_s": 103.0, "t_refresh_s": 2.7, "mem_gb": 21.26}
+{"step": 33, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.058952488194188725, "tokens": 120000, "cumulative_loss_tokens": 3960000, "grad_norm": 0.76171875, "lr": 3e-05, "finish_rate": 0.195, "comp_len": 487.8, "t_data_s": 0.0, "t_rollout_s": 31.4, "t_step_s": 103.7, "t_refresh_s": 2.7, "mem_gb": 21.29}
+{"step": 34, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.05145253275079498, "tokens": 120000, "cumulative_loss_tokens": 4080000, "grad_norm": 0.81640625, "lr": 3e-05, "finish_rate": 0.24, "comp_len": 487.8, "t_data_s": 0.0, "t_rollout_s": 31.1, "t_step_s": 103.0, "t_refresh_s": 2.7, "mem_gb": 21.28}
+{"step": 35, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.05340613210907516, "tokens": 120000, "cumulative_loss_tokens": 4200000, "grad_norm": 0.7578125, "lr": 3e-05, "finish_rate": 0.253, "comp_len": 474.3, "t_data_s": 0.0, "t_rollout_s": 31.0, "t_step_s": 105.5, "t_refresh_s": 2.7, "mem_gb": 21.27}
+{"step": 36, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.04817226686875026, "tokens": 120000, "cumulative_loss_tokens": 4320000, "grad_norm": 0.7890625, "lr": 3e-05, "finish_rate": 0.18, "comp_len": 491.8, "t_data_s": 0.0, "t_rollout_s": 32.0, "t_step_s": 104.0, "t_refresh_s": 2.7, "mem_gb": 21.27}
+{"step": 37, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.054132809651192895, "tokens": 120000, "cumulative_loss_tokens": 4440000, "grad_norm": 0.66015625, "lr": 3e-05, "finish_rate": 0.159, "comp_len": 489.8, "t_data_s": 0.0, "t_rollout_s": 31.5, "t_step_s": 104.0, "t_refresh_s": 2.7, "mem_gb": 21.24}
+{"step": 38, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.05171422024030859, "tokens": 120000, "cumulative_loss_tokens": 4560000, "grad_norm": 1.1484375, "lr": 3e-05, "finish_rate": 0.178, "comp_len": 495.9, "t_data_s": 0.0, "t_rollout_s": 31.4, "t_step_s": 101.7, "t_refresh_s": 2.7, "mem_gb": 21.27}
+{"step": 39, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.05344836845624571, "tokens": 120000, "cumulative_loss_tokens": 4680000, "grad_norm": 0.640625, "lr": 3e-05, "finish_rate": 0.16, "comp_len": 491.8, "t_data_s": 0.0, "t_rollout_s": 31.3, "t_step_s": 102.8, "t_refresh_s": 2.7, "mem_gb": 21.26}
+{"step": 40, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.050235971879477925, "tokens": 120000, "cumulative_loss_tokens": 4800000, "grad_norm": 0.62890625, "lr": 3e-05, "finish_rate": 0.165, "comp_len": 493.8, "t_data_s": 0.0, "t_rollout_s": 31.9, "t_step_s": 103.1, "t_refresh_s": 2.7, "mem_gb": 21.26}
+[eval step 40] sample: "To solve this problem, we need to maximize the number of maples \\( m \\) such that there are no two maples exactly 5 trees apart. Let's denote the number of larches by \\( l \\). According to the problem"
+{"step": 41, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.04683446280825883, "tokens": 120000, "cumulative_loss_tokens": 4920000, "grad_norm": 0.79296875, "lr": 3e-05, "finish_rate": 0.181, "comp_len": 481.9, "t_data_s": 0.0, "t_rollout_s": 31.3, "t_step_s": 104.6, "t_refresh_s": 2.7, "mem_gb": 21.33}
+{"step": 42, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.051344527876703065, "tokens": 120000, "cumulative_loss_tokens": 5040000, "grad_norm": 0.671875, "lr": 3e-05, "finish_rate": 0.147, "comp_len": 489.8, "t_data_s": 0.0, "t_rollout_s": 31.5, "t_step_s": 103.0, "t_refresh_s": 2.7, "mem_gb": 21.25}
+{"step": 43, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.0487479245382982, "tokens": 120000, "cumulative_loss_tokens": 5160000, "grad_norm": 0.84375, "lr": 3e-05, "finish_rate": 0.172, "comp_len": 491.8, "t_data_s": 0.0, "t_rollout_s": 31.1, "t_step_s": 101.6, "t_refresh_s": 2.7, "mem_gb": 21.22}
+{"step": 44, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.04652415793016553, "tokens": 120000, "cumulative_loss_tokens": 5280000, "grad_norm": 0.69140625, "lr": 3e-05, "finish_rate": 0.197, "comp_len": 481.9, "t_data_s": 0.0, "t_rollout_s": 30.9, "t_step_s": 102.8, "t_refresh_s": 2.7, "mem_gb": 21.28}
+{"step": 45, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.052038926035289965, "tokens": 120000, "cumulative_loss_tokens": 5400000, "grad_norm": 0.87109375, "lr": 3e-05, "finish_rate": 0.152, "comp_len": 493.8, "t_data_s": 0.0, "t_rollout_s": 31.1, "t_step_s": 101.6, "t_refresh_s": 2.7, "mem_gb": 21.2}
+{"step": 46, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.051263596718541034, "tokens": 120000, "cumulative_loss_tokens": 5520000, "grad_norm": 0.65234375, "lr": 3e-05, "finish_rate": 0.173, "comp_len": 493.8, "t_data_s": 0.0, "t_rollout_s": 31.5, "t_step_s": 103.1, "t_refresh_s": 2.7, "mem_gb": 21.24}
+{"step": 47, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.051921346824189335, "tokens": 120000, "cumulative_loss_tokens": 5640000, "grad_norm": 0.74609375, "lr": 3e-05, "finish_rate": 0.219, "comp_len": 478.1, "t_data_s": 0.0, "t_rollout_s": 30.8, "t_step_s": 103.4, "t_refresh_s": 2.7, "mem_gb": 21.25}
+{"step": 48, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.05030671559885765, "tokens": 120000, "cumulative_loss_tokens": 5760000, "grad_norm": 0.65625, "lr": 3e-05, "finish_rate": 0.136, "comp_len": 495.9, "t_data_s": 0.0, "t_rollout_s": 31.9, "t_step_s": 103.4, "t_refresh_s": 2.7, "mem_gb": 21.24}
+{"step": 49, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.048079004754545164, "tokens": 120000, "cumulative_loss_tokens": 5880000, "grad_norm": 0.59375, "lr": 3e-05, "finish_rate": 0.095, "comp_len": 497.9, "t_data_s": 0.0, "t_rollout_s": 31.6, "t_step_s": 102.4, "t_refresh_s": 2.7, "mem_gb": 21.28}
+{"step": 50, "epoch": 0, "training_mode": "on-policy", "reverse_kl": 0.051015459259475274, "tokens": 120000, "cumulative_loss_tokens": 6000000, "grad_norm": 0.78515625, "lr": 3e-05, "finish_rate": 0.136, "comp_len": 493.8, "t_data_s": 0.0, "t_rollout_s": 32.2, "t_step_s": 84.7, "t_refresh_s": 0.0, "mem_gb": 21.29}
+[eval step 50] sample: "To solve this problem, we need to maximize the number of maples \\( m \\) such that there are no two maples exactly 5 trees apart. Let's break down the problem step-by-step:\n\n1. **Understand the Constra"
+checkpoint snapshot queued -> outputs/healed/healing_breadth/reap_math_keep75_seed1224/step0050
+wandb: updating run metadata
+wandb: uploading config.yaml; uploading output.log; uploading wandb-summary.json
+wandb: 
+wandb: Run history:
+wandb:               comp_len █▇▆█▅▄▃▅▂▂▅▅▃▄▆▅▁▃▇▇▃▇▇▅▃▄▄▄▅▅▅▆▂▅▅▆▆▁▇▆
+wandb: cumulative_loss_tokens ▁▁▁▁▂▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███
+wandb:                  epoch ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁
+wandb:            finish_rate ▁▂▂▃▅▆▆▃▆▅▄▄▅▄▄█▅▄▃▅▃▃▃▇▆▆▇█▅▄▄▄▅▄▅▄▅▆▃▃
+wandb:              grad_norm █▇▇▅▃▂▃▂▂▂▁▂▁▂▂▁▁▂▁▁▁▂▁▁▁▁▁▁▁▁▂▁▁▁▁▁▁▁▁▁
+wandb:                     lr ▁▂▃▄▅▆▇█████████████████████████████████
+wandb:                 mem_gb ▁▅▇▆▆▄▅▄▄▆▅▆▅▅▆▆▄▅▅▆▇▇▆▇█▇▆▆▆▅▆▆█▅▄▄▅▅▅▇
+wandb:             reverse_kl █▇▇▆▄▃▃▂▂▂▂▂▂▂▂▂▁▁▁▂▂▁▁▁▂▁▂▁▁▁▁▁▁▁▁▁▁▁▁▁
+wandb:                   step ▁▁▁▁▂▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▅▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇███
+wandb:               t_data_s ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁
+wandb:                     +4 ...
+wandb: 
+wandb: Run summary:
+wandb:               comp_len 493.8
+wandb: cumulative_loss_tokens 6000000
+wandb:                  epoch 0
+wandb:            finish_rate 0.136
+wandb:              grad_norm 0.78516
+wandb:                     lr 3e-05
+wandb:                 mem_gb 21.29
+wandb:             reverse_kl 0.05102
+wandb:                   step 50
+wandb:               t_data_s 0
+wandb:                     +5 ...
+wandb: 
+wandb: 🚀 View run heal-reap_math_keep75-seed1224-run1 at: https://wandb.ai/hbfreed/glean-heal/runs/wghixdgk
+wandb: ⭐️ View project at: https://wandb.ai/hbfreed/glean-heal
+wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)
+wandb: Find logs at: outputs/healed/healing_breadth/reap_math_keep75_seed1224/wandb/run-20260714_215914-wghixdgk/logs

healed/healing_breadth/traj_sweep.log

@@ -0,0 +1,88 @@
+2026-07-15T19:40:43-07:00 eval step0025 chat
+{
+  "correct": 407,
+  "accuracy": 0.30856709628506446,
+  "finished": 1008,
+  "finish_rate": 0.7642153146322972,
+  "mean_completion_tokens": 351.8074298711145
+}
+saved item-level results -> outputs/evals/healing_breadth/keep25_traj/step0025_chat.json
+2026-07-15T19:44:18-07:00 eval step0050 chat
+{
+  "correct": 344,
+  "accuracy": 0.2608036391205459,
+  "finished": 755,
+  "finish_rate": 0.5724033358605004,
+  "mean_completion_tokens": 369.57088703563306
+}
+saved item-level results -> outputs/evals/healing_breadth/keep25_traj/step0050_chat.json
+2026-07-15T19:47:57-07:00 eval step0100 chat
+{
+  "correct": 564,
+  "accuracy": 0.4275966641394996,
+  "finished": 1235,
+  "finish_rate": 0.9363153904473086,
+  "mean_completion_tokens": 271.9120545868082
+}
+saved item-level results -> outputs/evals/healing_breadth/keep25_traj/step0100_chat.json
+2026-07-15T19:50:53-07:00 eval step0150 chat
+{
+  "correct": 607,
+  "accuracy": 0.46019711902956784,
+  "finished": 1250,
+  "finish_rate": 0.9476876421531463,
+  "mean_completion_tokens": 256.28127369219106
+}
+saved item-level results -> outputs/evals/healing_breadth/keep25_traj/step0150_chat.json
+2026-07-15T19:53:43-07:00 eval step0200 chat
+{
+  "correct": 606,
+  "accuracy": 0.45943896891584535,
+  "finished": 1251,
+  "finish_rate": 0.9484457922668689,
+  "mean_completion_tokens": 245.95754359363153
+}
+saved item-level results -> outputs/evals/healing_breadth/keep25_traj/step0200_chat.json
+2026-07-15T19:56:25-07:00 eval step0250 chat
+{
+  "correct": 610,
+  "accuracy": 0.4624715693707354,
+  "finished": 1225,
+  "finish_rate": 0.9287338893100834,
+  "mean_completion_tokens": 256.76648976497347
+}
+saved item-level results -> outputs/evals/healing_breadth/keep25_traj/step0250_chat.json
+2026-07-15T19:59:11-07:00 eval step0300 chat
+{
+  "correct": 625,
+  "accuracy": 0.47384382107657314,
+  "finished": 1240,
+  "finish_rate": 0.9401061410159212,
+  "mean_completion_tokens": 243.75208491281273
+}
+saved item-level results -> outputs/evals/healing_breadth/keep25_traj/step0300_chat.json
+2026-07-15T20:01:52-07:00 step0350 exists, skip
+2026-07-15T20:01:52-07:00 step0400 exists, skip
+2026-07-15T20:01:52-07:00 step0450 exists, skip
+2026-07-15T20:01:52-07:00 eval step0500 chat
+{
+  "correct": 631,
+  "accuracy": 0.4783927217589083,
+  "finished": 1231,
+  "finish_rate": 0.9332827899924185,
+  "mean_completion_tokens": 265.1554207733131
+}
+saved item-level results -> outputs/evals/healing_breadth/keep25_traj/step0500_chat.json
+2026-07-15T20:04:42-07:00 === keep25 chat healing trajectory (full 1319) ===
+2026-07-15T20:04:42-07:00 step 0025: acc=0.30856709628506446 finish=0.7642153146322972
+2026-07-15T20:04:42-07:00 step 0050: acc=0.2608036391205459 finish=0.5724033358605004
+2026-07-15T20:04:42-07:00 step 0100: acc=0.4275966641394996 finish=0.9363153904473086
+2026-07-15T20:04:42-07:00 step 0150: acc=0.46019711902956784 finish=0.9476876421531463
+2026-07-15T20:04:43-07:00 step 0200: acc=0.45943896891584535 finish=0.9484457922668689
+2026-07-15T20:04:43-07:00 step 0250: acc=0.4624715693707354 finish=0.9287338893100834
+2026-07-15T20:04:43-07:00 step 0300: acc=0.47384382107657314 finish=0.9401061410159212
+2026-07-15T20:04:43-07:00 step 0350: acc=0.48218347232752085 finish=0.956027293404094
+2026-07-15T20:04:43-07:00 step 0400: acc=0.4981046247156937 finish=0.9514783927217589
+2026-07-15T20:04:43-07:00 step 0450: acc=0.48597422289613346 finish=0.9446550416982562
+2026-07-15T20:04:43-07:00 step 0500: acc=0.4783927217589083 finish=0.9332827899924185
+2026-07-15T20:04:43-07:00 TRAJECTORY SWEEP COMPLETE

healed/healing_breadth/traj_sweep2.log

@@ -0,0 +1,37 @@
+2026-07-15T19:51:42-07:00 [w2] eval step0350 chat (gpu 8ca7, port 8378)
+{
+  "correct": 636,
+  "accuracy": 0.48218347232752085,
+  "finished": 1261,
+  "finish_rate": 0.956027293404094,
+  "mean_completion_tokens": 249.04094010614102
+}
+saved item-level results -> outputs/evals/healing_breadth/keep25_traj/step0350_chat.json
+2026-07-15T19:54:25-07:00 [w2] eval step0400 chat (gpu 8ca7, port 8378)
+{
+  "correct": 657,
+  "accuracy": 0.4981046247156937,
+  "finished": 1255,
+  "finish_rate": 0.9514783927217589,
+  "mean_completion_tokens": 249.73692191053829
+}
+saved item-level results -> outputs/evals/healing_breadth/keep25_traj/step0400_chat.json
+2026-07-15T19:57:08-07:00 [w2] eval step0450 chat (gpu 8ca7, port 8378)
+{
+  "correct": 641,
+  "accuracy": 0.48597422289613346,
+  "finished": 1246,
+  "finish_rate": 0.9446550416982562,
+  "mean_completion_tokens": 256.36618650492795
+}
+saved item-level results -> outputs/evals/healing_breadth/keep25_traj/step0450_chat.json
+2026-07-15T19:59:54-07:00 [w2] eval step0500 chat (gpu 8ca7, port 8378)
+{
+  "correct": 627,
+  "accuracy": 0.47536012130401817,
+  "finished": 1239,
+  "finish_rate": 0.9393479909021987,
+  "mean_completion_tokens": 264.5200909780136
+}
+saved item-level results -> outputs/evals/healing_breadth/keep25_traj/step0500_chat.json
+2026-07-15T20:02:42-07:00 [w2] worker2 tail complete

healed/keep50_warmup_fixed_s1224/args.json

@@ -0,0 +1,72 @@
+{
+  "student": "outputs/pruned/glean-0125inst-math-keep50",
+  "teacher": "allenai/OLMoE-1B-7B-0125-Instruct",
+  "training_mode": "off-policy",
+  "kl_direction": "forward",
+  "dataset": "allenai/RLVR-MATH",
+  "dataset_sources": null,
+  "max_difficulty": null,
+  "trajectories": "outputs/teacher_trajectories/dolci_math_curated.jsonl",
+  "trajectory_dataset": "allenai/Dolci-Instruct-RL",
+  "off_policy_frames": "chat",
+  "off_policy_max_seq_len": 2048,
+  "topk_targets": "outputs/teacher_trajectories/dolci_math_curated_opd_top128",
+  "max_loss_tokens": null,
+  "loss_tokens_per_step": 120000,
+  "teacher_device": "cuda:0",
+  "student_device": "cuda:0",
+  "lr": 3e-05,
+  "optimizer": "adamw8bit",
+  "weight_decay": 0.1,
+  "epochs": 3,
+  "prompts_per_step": 256,
+  "group_size": 1,
+  "rollout_batch": 64,
+  "micro_batch": 3,
+  "max_new_tokens": 256,
+  "max_prompt_len": 1024,
+  "warmup_steps": 10,
+  "max_grad_norm": 1.0,
+  "eval_every": 10,
+  "gsm8k_every": 0,
+  "gsm8k_n": 256,
+  "gsm8k_batch": 16,
+  "gsm8k_max_new_tokens": 512,
+  "gsm8k_frames": "chat",
+  "save_every": 50,
+  "out_dir": "outputs/healed/keep50_warmup_fixed_s1224",
+  "sweep": 150,
+  "wandb": true,
+  "wandb_project": "glean-heal",
+  "wandb_run_name": "warmup-fixed-keep50-s1224",
+  "wandb_run_id": null,
+  "wandb_resume": null,
+  "wandb_mode": "online",
+  "no_wandb_sync": false,
+  "debug": false,
+  "resume_from": null,
+  "start_step": 0,
+  "no_grad_checkpointing": false,
+  "seed": 1224,
+  "no_teacher_overlap": false,
+  "sync_checkpoints": false,
+  "rollout_engine": "hf",
+  "vllm_gpu": null,
+  "vllm_port": 8377,
+  "vllm_refresh_every": 5,
+  "vllm_serve_bin": "vllm-plugin/.venv/bin/python",
+  "vllm_gpu_mem_util": 0.85,
+  "liger_loss": false,
+  "gold_mix_lambda": 0.0,
+  "gold_topk_targets": null,
+  "gold_loss": "ce",
+  "gold_mix_decay": 0.0,
+  "fast_teacher": false,
+  "fast_student": false,
+  "reference_kl_beta": 0.0,
+  "drop_truncated_rollouts": false,
+  "vllm_max_model_len": null,
+  "vllm_refresh_mode": "reload",
+  "vllm_live_dir": null,
+  "resolved_kl_direction": "forward"
+}

healed/keep50_warmup_fixed_s1224/step0150/modeling_pruned_olmoe.py

@@ -0,0 +1,66 @@
+"""GLEAN-pruned OLMoE: HF-loadable model with ragged (variable-width) experts.
+
+Pattern follows hbfreed/variable-flex-olmo's PrunedFlexOlmoForCausalLM
+(docs/recon/prior-work-hbfreed.md), generalized from one scalar width to a
+per-(layer, expert) width table: ``super().__init__`` builds the uniform
+architecture from the config, then every MoE block is rebuilt to its pruned
+shape — surviving experts only, each at its own width, router sliced to
+match — so the state dict aligns exactly with what
+``glean.prune.prune_channels_global`` leaves behind.
+
+Caveat: ``output_router_logits=True`` (the load-balancing aux loss) assumes a
+uniform ``config.num_experts`` and is unsupported on ragged models.
+"""
+
+import torch.nn as nn
+from transformers.activations import ACT2FN
+from transformers.models.olmoe.modeling_olmoe import OlmoeForCausalLM
+
+from .configuration_pruned_olmoe import PrunedOlmoeConfig
+
+
+class RaggedOlmoeMLP(nn.Module):
+    """OlmoeMLP with an explicit intermediate width (SwiGLU, no biases)."""
+
+    def __init__(self, hidden_size: int, intermediate_size: int, hidden_act: str):
+        super().__init__()
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.gate_proj = nn.Linear(hidden_size, intermediate_size, bias=False)
+        self.up_proj = nn.Linear(hidden_size, intermediate_size, bias=False)
+        self.down_proj = nn.Linear(intermediate_size, hidden_size, bias=False)
+        self.act_fn = ACT2FN[hidden_act]
+
+    def forward(self, x):
+        return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
+
+
+class PrunedOlmoeForCausalLM(OlmoeForCausalLM):
+    """OLMoE with per-layer surviving-expert lists at per-expert widths."""
+
+    config_class = PrunedOlmoeConfig
+
+    def __init__(self, config: PrunedOlmoeConfig):
+        super().__init__(config)
+        widths_table = getattr(config, "expert_widths", None)
+        if widths_table is None:
+            return  # unpruned: plain OLMoE
+        if len(widths_table) != len(self.model.layers):
+            raise ValueError(
+                f"expert_widths has {len(widths_table)} rows but the model has "
+                f"{len(self.model.layers)} decoder layers"
+            )
+        for layer, widths in zip(self.model.layers, widths_table):
+            if any(w <= 0 for w in widths):
+                raise ValueError("expert_widths must list surviving experts only (>0)")
+            block = layer.mlp
+            if len(widths) < block.top_k:
+                raise ValueError(
+                    f"a layer keeps {len(widths)} experts < top_k={block.top_k}"
+                )
+            block.num_experts = len(widths)
+            block.gate = nn.Linear(config.hidden_size, len(widths), bias=False)
+            block.experts = nn.ModuleList(
+                RaggedOlmoeMLP(config.hidden_size, w, config.hidden_act)
+                for w in widths
+            )

healed/keep50_warmup_fixed_s1224/train_log.jsonl

@@ -0,0 +1,150 @@
+{"step": 1, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.21896971870896717, "tokens": 120000, "cumulative_loss_tokens": 120000, "grad_norm": 4.6875, "lr": 6e-06, "finish_rate": 0.907, "comp_len": 508.5, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.337, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 99.1, "frames": {"chat": 236}, "mem_gb": 15.78, "mem_gb_teacher": 15.78}
+{"step": 2, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.27215903437460465, "tokens": 120000, "cumulative_loss_tokens": 240000, "grad_norm": 4.84375, "lr": 9e-06, "finish_rate": 0.781, "comp_len": 558.1, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.474, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 90.6, "frames": {"chat": 215}, "mem_gb": 16.05, "mem_gb_teacher": 16.05}
+{"step": 3, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.2780314308715363, "tokens": 120000, "cumulative_loss_tokens": 360000, "grad_norm": 4.09375, "lr": 1.2e-05, "finish_rate": 0.825, "comp_len": 553.0, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.376, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 90.6, "frames": {"chat": 217}, "mem_gb": 15.93, "mem_gb_teacher": 15.93}
+{"step": 4, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.21216022065331538, "tokens": 120000, "cumulative_loss_tokens": 480000, "grad_norm": 2.546875, "lr": 1.5e-05, "finish_rate": 0.8, "comp_len": 585.4, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.366, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 86.0, "frames": {"chat": 205}, "mem_gb": 15.99, "mem_gb_teacher": 15.99}
+{"step": 5, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.13948054732351253, "tokens": 120000, "cumulative_loss_tokens": 600000, "grad_norm": 1.6484375, "lr": 1.8e-05, "finish_rate": 0.834, "comp_len": 524.0, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.318, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 94.5, "frames": {"chat": 229}, "mem_gb": 15.96, "mem_gb_teacher": 15.96}
+{"step": 6, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.2525411546646928, "tokens": 120000, "cumulative_loss_tokens": 720000, "grad_norm": 2.578125, "lr": 2.1e-05, "finish_rate": 0.812, "comp_len": 538.1, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.336, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 92.0, "frames": {"chat": 223}, "mem_gb": 16.03, "mem_gb_teacher": 16.03}
+{"step": 7, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.1408244575532774, "tokens": 120000, "cumulative_loss_tokens": 840000, "grad_norm": 1.171875, "lr": 2.4e-05, "finish_rate": 0.708, "comp_len": 594.1, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.42, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 86.3, "frames": {"chat": 202}, "mem_gb": 16.07, "mem_gb_teacher": 16.07}
+{"step": 8, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.16170320059585697, "tokens": 120000, "cumulative_loss_tokens": 960000, "grad_norm": 1.171875, "lr": 2.7000000000000002e-05, "finish_rate": 0.77, "comp_len": 574.2, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.453, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 87.6, "frames": {"chat": 209}, "mem_gb": 16.04, "mem_gb_teacher": 16.04}
+{"step": 9, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.1060452919805112, "tokens": 120000, "cumulative_loss_tokens": 1080000, "grad_norm": 0.703125, "lr": 3e-05, "finish_rate": 0.885, "comp_len": 528.6, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.397, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 93.7, "frames": {"chat": 227}, "mem_gb": 16.02, "mem_gb_teacher": 16.02}
+{"step": 10, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.11166963671234746, "tokens": 120000, "cumulative_loss_tokens": 1200000, "grad_norm": 0.66796875, "lr": 3e-05, "finish_rate": 0.848, "comp_len": 521.7, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.546, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 95.5, "frames": {"chat": 230}, "mem_gb": 16.09, "mem_gb_teacher": 16.09}
+{"step": 11, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.09883926218959192, "tokens": 120000, "cumulative_loss_tokens": 1320000, "grad_norm": 0.671875, "lr": 3e-05, "finish_rate": 0.879, "comp_len": 519.5, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.426, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 94.7, "frames": {"chat": 231}, "mem_gb": 15.94, "mem_gb_teacher": 15.94}
+{"step": 12, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.09610702018613616, "tokens": 120000, "cumulative_loss_tokens": 1440000, "grad_norm": 0.55078125, "lr": 3e-05, "finish_rate": 0.882, "comp_len": 489.8, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.246, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 99.3, "frames": {"chat": 245}, "mem_gb": 16.02, "mem_gb_teacher": 16.02}
+{"step": 13, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.09665392311389248, "tokens": 120000, "cumulative_loss_tokens": 1560000, "grad_norm": 0.5703125, "lr": 3e-05, "finish_rate": 0.81, "comp_len": 571.4, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.375, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 87.0, "frames": {"chat": 210}, "mem_gb": 16.02, "mem_gb_teacher": 16.02}
+{"step": 14, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.1129512736000431, "tokens": 120000, "cumulative_loss_tokens": 1680000, "grad_norm": 0.578125, "lr": 3e-05, "finish_rate": 0.758, "comp_len": 568.7, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.312, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 89.4, "frames": {"chat": 211}, "mem_gb": 16.02, "mem_gb_teacher": 16.02}
+{"step": 15, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.09338119786353782, "tokens": 120000, "cumulative_loss_tokens": 1800000, "grad_norm": 0.453125, "lr": 3e-05, "finish_rate": 0.819, "comp_len": 543.0, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.413, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 92.0, "frames": {"chat": 221}, "mem_gb": 16.08, "mem_gb_teacher": 16.08}
+{"step": 16, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.08123978671409811, "tokens": 120000, "cumulative_loss_tokens": 1920000, "grad_norm": 0.490234375, "lr": 3e-05, "finish_rate": 0.912, "comp_len": 480.0, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.366, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 101.9, "frames": {"chat": 250}, "mem_gb": 15.89, "mem_gb_teacher": 15.89}
+{"step": 17, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.08966803371421993, "tokens": 120000, "cumulative_loss_tokens": 2040000, "grad_norm": 0.49609375, "lr": 3e-05, "finish_rate": 0.79, "comp_len": 524.0, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.402, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 95.6, "frames": {"chat": 229}, "mem_gb": 16.06, "mem_gb_teacher": 16.06}
+{"step": 18, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.06860631760672356, "tokens": 120000, "cumulative_loss_tokens": 2160000, "grad_norm": 0.408203125, "lr": 3e-05, "finish_rate": 0.888, "comp_len": 480.0, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.479, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 101.4, "frames": {"chat": 250}, "mem_gb": 16.04, "mem_gb_teacher": 16.04}
+{"step": 19, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.08370754941549773, "tokens": 120000, "cumulative_loss_tokens": 2280000, "grad_norm": 0.421875, "lr": 3e-05, "finish_rate": 0.844, "comp_len": 519.5, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.363, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 95.2, "frames": {"chat": 231}, "mem_gb": 15.91, "mem_gb_teacher": 15.91}
+{"step": 20, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.07473204600410537, "tokens": 120000, "cumulative_loss_tokens": 2400000, "grad_norm": 0.41796875, "lr": 3e-05, "finish_rate": 0.844, "comp_len": 535.7, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.433, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 92.5, "frames": {"chat": 224}, "mem_gb": 15.96, "mem_gb_teacher": 15.96}
+{"step": 21, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.08024157820896556, "tokens": 120000, "cumulative_loss_tokens": 2520000, "grad_norm": 0.486328125, "lr": 3e-05, "finish_rate": 0.802, "comp_len": 566.0, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.329, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 88.4, "frames": {"chat": 212}, "mem_gb": 16.0, "mem_gb_teacher": 16.0}
+{"step": 22, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.0721184289892825, "tokens": 120000, "cumulative_loss_tokens": 2640000, "grad_norm": 0.435546875, "lr": 3e-05, "finish_rate": 0.87, "comp_len": 504.2, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.376, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 97.3, "frames": {"chat": 238}, "mem_gb": 15.95, "mem_gb_teacher": 15.95}
+{"step": 23, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.07248173923180438, "tokens": 120000, "cumulative_loss_tokens": 2760000, "grad_norm": 0.443359375, "lr": 3e-05, "finish_rate": 0.903, "comp_len": 466.9, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.445, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 103.5, "frames": {"chat": 257}, "mem_gb": 15.83, "mem_gb_teacher": 15.83}
+{"step": 24, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.0655906916304181, "tokens": 120000, "cumulative_loss_tokens": 2880000, "grad_norm": 0.36328125, "lr": 3e-05, "finish_rate": 0.868, "comp_len": 528.6, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.405, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 93.3, "frames": {"chat": 227}, "mem_gb": 16.02, "mem_gb_teacher": 16.02}
+{"step": 25, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.07533252080177578, "tokens": 120000, "cumulative_loss_tokens": 3000000, "grad_norm": 0.380859375, "lr": 3e-05, "finish_rate": 0.838, "comp_len": 526.3, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.359, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 93.6, "frames": {"chat": 228}, "mem_gb": 16.05, "mem_gb_teacher": 16.05}
+{"step": 26, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.07692283792655605, "tokens": 120000, "cumulative_loss_tokens": 3120000, "grad_norm": 0.40625, "lr": 3e-05, "finish_rate": 0.803, "comp_len": 515.0, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.542, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 97.0, "frames": {"chat": 233}, "mem_gb": 16.04, "mem_gb_teacher": 16.04}
+{"step": 27, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.06803224476923546, "tokens": 120000, "cumulative_loss_tokens": 3240000, "grad_norm": 0.3828125, "lr": 3e-05, "finish_rate": 0.863, "comp_len": 515.0, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.351, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 96.2, "frames": {"chat": 233}, "mem_gb": 16.04, "mem_gb_teacher": 16.04}
+{"step": 28, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.09600000411309302, "tokens": 120000, "cumulative_loss_tokens": 3360000, "grad_norm": 0.46484375, "lr": 3e-05, "finish_rate": 0.731, "comp_len": 609.1, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.388, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 85.0, "frames": {"chat": 197}, "mem_gb": 16.13, "mem_gb_teacher": 16.13}
+{"step": 29, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.0838949200007754, "tokens": 120000, "cumulative_loss_tokens": 3480000, "grad_norm": 0.404296875, "lr": 3e-05, "finish_rate": 0.862, "comp_len": 502.1, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.384, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 98.3, "frames": {"chat": 239}, "mem_gb": 15.88, "mem_gb_teacher": 15.88}
+{"step": 30, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.08550032369385784, "tokens": 120000, "cumulative_loss_tokens": 3600000, "grad_norm": 0.640625, "lr": 3e-05, "finish_rate": 0.83, "comp_len": 535.7, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.302, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 91.8, "frames": {"chat": 224}, "mem_gb": 15.93, "mem_gb_teacher": 15.93}
+{"step": 31, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.06893708595448794, "tokens": 120000, "cumulative_loss_tokens": 3720000, "grad_norm": 0.3828125, "lr": 3e-05, "finish_rate": 0.788, "comp_len": 553.0, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.388, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 90.4, "frames": {"chat": 217}, "mem_gb": 16.04, "mem_gb_teacher": 16.04}
+{"step": 32, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.0662639382578743, "tokens": 120000, "cumulative_loss_tokens": 3840000, "grad_norm": 0.376953125, "lr": 3e-05, "finish_rate": 0.876, "comp_len": 497.9, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.259, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 98.4, "frames": {"chat": 241}, "mem_gb": 16.04, "mem_gb_teacher": 16.04}
+{"step": 33, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.06439438750580885, "tokens": 120000, "cumulative_loss_tokens": 3960000, "grad_norm": 0.39453125, "lr": 3e-05, "finish_rate": 0.835, "comp_len": 550.5, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.407, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 89.8, "frames": {"chat": 218}, "mem_gb": 16.02, "mem_gb_teacher": 16.02}
+{"step": 34, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.06970151182319968, "tokens": 120000, "cumulative_loss_tokens": 4080000, "grad_norm": 0.353515625, "lr": 3e-05, "finish_rate": 0.767, "comp_len": 582.5, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.439, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 86.2, "frames": {"chat": 206}, "mem_gb": 16.03, "mem_gb_teacher": 16.03}
+{"step": 35, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.06962556957538861, "tokens": 120000, "cumulative_loss_tokens": 4200000, "grad_norm": 0.37890625, "lr": 3e-05, "finish_rate": 0.845, "comp_len": 517.2, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.512, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 96.4, "frames": {"chat": 232}, "mem_gb": 16.07, "mem_gb_teacher": 16.07}
+{"step": 36, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.07808773103132843, "tokens": 120000, "cumulative_loss_tokens": 4320000, "grad_norm": 0.419921875, "lr": 3e-05, "finish_rate": 0.771, "comp_len": 550.5, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.432, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 92.0, "frames": {"chat": 218}, "mem_gb": 16.09, "mem_gb_teacher": 16.09}
+{"step": 37, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.07417992857682208, "tokens": 120000, "cumulative_loss_tokens": 4440000, "grad_norm": 0.376953125, "lr": 3e-05, "finish_rate": 0.779, "comp_len": 563.4, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.428, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 88.9, "frames": {"chat": 213}, "mem_gb": 16.05, "mem_gb_teacher": 16.05}
+{"step": 38, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.07201246263841167, "tokens": 120000, "cumulative_loss_tokens": 4560000, "grad_norm": 0.369140625, "lr": 3e-05, "finish_rate": 0.887, "comp_len": 483.9, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.527, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 100.9, "frames": {"chat": 248}, "mem_gb": 16.02, "mem_gb_teacher": 16.02}
+{"step": 39, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.05988815330729509, "tokens": 120000, "cumulative_loss_tokens": 4680000, "grad_norm": 0.373046875, "lr": 3e-05, "finish_rate": 0.803, "comp_len": 550.5, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.333, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 90.1, "frames": {"chat": 218}, "mem_gb": 16.08, "mem_gb_teacher": 16.08}
+{"step": 40, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.05963528015368308, "tokens": 120000, "cumulative_loss_tokens": 4800000, "grad_norm": 0.3359375, "lr": 3e-05, "finish_rate": 0.851, "comp_len": 543.0, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.401, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 91.5, "frames": {"chat": 221}, "mem_gb": 16.04, "mem_gb_teacher": 16.04}
+{"step": 41, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.05816531496203194, "tokens": 120000, "cumulative_loss_tokens": 4920000, "grad_norm": 0.345703125, "lr": 3e-05, "finish_rate": 0.894, "comp_len": 508.5, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.396, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 96.1, "frames": {"chat": 236}, "mem_gb": 15.97, "mem_gb_teacher": 15.97}
+{"step": 42, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.06005277933338657, "tokens": 120000, "cumulative_loss_tokens": 5040000, "grad_norm": 0.326171875, "lr": 3e-05, "finish_rate": 0.858, "comp_len": 487.8, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.338, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 100.6, "frames": {"chat": 246}, "mem_gb": 15.89, "mem_gb_teacher": 15.89}
+{"step": 43, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.06171362210111692, "tokens": 120000, "cumulative_loss_tokens": 5160000, "grad_norm": 0.35546875, "lr": 3e-05, "finish_rate": 0.838, "comp_len": 512.8, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.48, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 96.2, "frames": {"chat": 234}, "mem_gb": 16.14, "mem_gb_teacher": 16.14}
+{"step": 44, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.055975045030827945, "tokens": 120000, "cumulative_loss_tokens": 5280000, "grad_norm": 0.3515625, "lr": 3e-05, "finish_rate": 0.748, "comp_len": 594.1, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.474, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 85.4, "frames": {"chat": 202}, "mem_gb": 16.03, "mem_gb_teacher": 16.03}
+{"step": 45, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.06047188884726105, "tokens": 120000, "cumulative_loss_tokens": 5400000, "grad_norm": 0.3359375, "lr": 3e-05, "finish_rate": 0.811, "comp_len": 553.0, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.36, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 90.3, "frames": {"chat": 217}, "mem_gb": 16.04, "mem_gb_teacher": 16.04}
+{"step": 46, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.05655016646341731, "tokens": 120000, "cumulative_loss_tokens": 5520000, "grad_norm": 0.357421875, "lr": 3e-05, "finish_rate": 0.866, "comp_len": 535.7, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.47, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 92.3, "frames": {"chat": 224}, "mem_gb": 16.04, "mem_gb_teacher": 16.04}
+{"step": 47, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.0790889122961089, "tokens": 120000, "cumulative_loss_tokens": 5640000, "grad_norm": 0.828125, "lr": 3e-05, "finish_rate": 0.753, "comp_len": 558.1, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.285, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 89.5, "frames": {"chat": 215}, "mem_gb": 16.05, "mem_gb_teacher": 16.05}
+{"step": 48, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.05377363468687981, "tokens": 120000, "cumulative_loss_tokens": 5760000, "grad_norm": 0.353515625, "lr": 3e-05, "finish_rate": 0.884, "comp_len": 463.3, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.356, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 104.3, "frames": {"chat": 259}, "mem_gb": 15.97, "mem_gb_teacher": 15.97}
+{"step": 49, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.06014201375305032, "tokens": 120000, "cumulative_loss_tokens": 5880000, "grad_norm": 0.3359375, "lr": 3e-05, "finish_rate": 0.829, "comp_len": 571.4, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.475, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 86.9, "frames": {"chat": 210}, "mem_gb": 16.04, "mem_gb_teacher": 16.04}
+{"step": 50, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.07395202046850076, "tokens": 120000, "cumulative_loss_tokens": 6000000, "grad_norm": 0.416015625, "lr": 3e-05, "finish_rate": 0.77, "comp_len": 563.4, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.304, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 89.3, "frames": {"chat": 213}, "mem_gb": 16.09, "mem_gb_teacher": 16.09}
+{"step": 51, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.05859705059945894, "tokens": 120000, "cumulative_loss_tokens": 6120000, "grad_norm": 0.34765625, "lr": 3e-05, "finish_rate": 0.815, "comp_len": 540.5, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.453, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 90.8, "frames": {"chat": 222}, "mem_gb": 16.0, "mem_gb_teacher": 16.0}
+{"step": 52, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.0539097297622667, "tokens": 120000, "cumulative_loss_tokens": 6240000, "grad_norm": 0.353515625, "lr": 3e-05, "finish_rate": 0.889, "comp_len": 510.6, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.439, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 96.1, "frames": {"chat": 235}, "mem_gb": 16.05, "mem_gb_teacher": 16.05}
+{"step": 53, "epoch": 0, "training_mode": "off-policy", "forward_topk_kl": 0.0594748610290233, "tokens": 120000, "cumulative_loss_tokens": 6360000, "grad_norm": 0.328125, "lr": 3e-05, "finish_rate": 0.798, "comp_len": 576.9, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.417, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 86.8, "frames": {"chat": 208}, "mem_gb": 16.01, "mem_gb_teacher": 16.01}
+{"step": 54, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.045099887475029875, "tokens": 120000, "cumulative_loss_tokens": 6480000, "grad_norm": 0.275390625, "lr": 3e-05, "finish_rate": 0.733, "comp_len": 628.3, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.413, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 80.2, "frames": {"chat": 191}, "mem_gb": 16.05, "mem_gb_teacher": 16.05}
+{"step": 55, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.03569839329215077, "tokens": 120000, "cumulative_loss_tokens": 6600000, "grad_norm": 0.2451171875, "lr": 3e-05, "finish_rate": 0.845, "comp_len": 547.9, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.523, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 90.0, "frames": {"chat": 219}, "mem_gb": 16.04, "mem_gb_teacher": 16.04}
+{"step": 56, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.04039377661425, "tokens": 120000, "cumulative_loss_tokens": 6720000, "grad_norm": 0.302734375, "lr": 3e-05, "finish_rate": 0.778, "comp_len": 579.7, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.459, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 86.0, "frames": {"chat": 207}, "mem_gb": 16.05, "mem_gb_teacher": 16.05}
+{"step": 57, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.055602129854184265, "tokens": 120000, "cumulative_loss_tokens": 6840000, "grad_norm": 0.330078125, "lr": 3e-05, "finish_rate": 0.755, "comp_len": 576.9, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.449, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 88.0, "frames": {"chat": 208}, "mem_gb": 16.01, "mem_gb_teacher": 16.01}
+{"step": 58, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.035877808295966436, "tokens": 120000, "cumulative_loss_tokens": 6960000, "grad_norm": 0.279296875, "lr": 3e-05, "finish_rate": 0.799, "comp_len": 547.9, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.288, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 90.8, "frames": {"chat": 219}, "mem_gb": 16.04, "mem_gb_teacher": 16.04}
+{"step": 59, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.03478414489501932, "tokens": 120000, "cumulative_loss_tokens": 7080000, "grad_norm": 0.2578125, "lr": 3e-05, "finish_rate": 0.915, "comp_len": 487.8, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.329, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 99.7, "frames": {"chat": 246}, "mem_gb": 15.92, "mem_gb_teacher": 15.92}
+{"step": 60, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.04739725781680706, "tokens": 120000, "cumulative_loss_tokens": 7200000, "grad_norm": 0.330078125, "lr": 3e-05, "finish_rate": 0.704, "comp_len": 582.5, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.437, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 87.5, "frames": {"chat": 206}, "mem_gb": 16.07, "mem_gb_teacher": 16.07}
+{"step": 61, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.03820183755345643, "tokens": 120000, "cumulative_loss_tokens": 7320000, "grad_norm": 0.275390625, "lr": 3e-05, "finish_rate": 0.876, "comp_len": 515.0, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.375, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 95.5, "frames": {"chat": 233}, "mem_gb": 16.05, "mem_gb_teacher": 16.05}
+{"step": 62, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.04388709897000032, "tokens": 120000, "cumulative_loss_tokens": 7440000, "grad_norm": 0.275390625, "lr": 3e-05, "finish_rate": 0.847, "comp_len": 524.0, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.363, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 93.8, "frames": {"chat": 229}, "mem_gb": 15.92, "mem_gb_teacher": 15.92}
+{"step": 63, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.032643947703313705, "tokens": 120000, "cumulative_loss_tokens": 7560000, "grad_norm": 0.236328125, "lr": 3e-05, "finish_rate": 0.864, "comp_len": 508.5, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.285, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 96.4, "frames": {"chat": 236}, "mem_gb": 15.95, "mem_gb_teacher": 15.95}
+{"step": 64, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.04086883016227123, "tokens": 120000, "cumulative_loss_tokens": 7680000, "grad_norm": 0.2451171875, "lr": 3e-05, "finish_rate": 0.87, "comp_len": 502.1, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.37, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 98.3, "frames": {"chat": 239}, "mem_gb": 15.84, "mem_gb_teacher": 15.84}
+{"step": 65, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.03435394472128246, "tokens": 120000, "cumulative_loss_tokens": 7800000, "grad_norm": 0.2177734375, "lr": 3e-05, "finish_rate": 0.867, "comp_len": 497.9, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.349, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 99.1, "frames": {"chat": 241}, "mem_gb": 15.96, "mem_gb_teacher": 15.96}
+{"step": 66, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.036759108127855385, "tokens": 120000, "cumulative_loss_tokens": 7920000, "grad_norm": 0.2294921875, "lr": 3e-05, "finish_rate": 0.863, "comp_len": 531.0, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.44, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 93.4, "frames": {"chat": 226}, "mem_gb": 15.92, "mem_gb_teacher": 15.92}
+{"step": 67, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.031785604377323765, "tokens": 120000, "cumulative_loss_tokens": 8040000, "grad_norm": 0.224609375, "lr": 3e-05, "finish_rate": 0.893, "comp_len": 512.8, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.32, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 95.3, "frames": {"chat": 234}, "mem_gb": 16.05, "mem_gb_teacher": 16.05}
+{"step": 68, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.03207274362700991, "tokens": 120000, "cumulative_loss_tokens": 8160000, "grad_norm": 0.22265625, "lr": 3e-05, "finish_rate": 0.914, "comp_len": 466.9, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.401, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 102.7, "frames": {"chat": 257}, "mem_gb": 16.04, "mem_gb_teacher": 16.04}
+{"step": 69, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.04908341010484534, "tokens": 120000, "cumulative_loss_tokens": 8280000, "grad_norm": 0.2890625, "lr": 3e-05, "finish_rate": 0.76, "comp_len": 576.9, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.523, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 89.3, "frames": {"chat": 208}, "mem_gb": 16.1, "mem_gb_teacher": 16.1}
+{"step": 70, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.043880132612407516, "tokens": 120000, "cumulative_loss_tokens": 8400000, "grad_norm": 0.267578125, "lr": 3e-05, "finish_rate": 0.763, "comp_len": 568.7, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.522, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 89.2, "frames": {"chat": 211}, "mem_gb": 16.07, "mem_gb_teacher": 16.07}
+{"step": 71, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.04658220293604148, "tokens": 120000, "cumulative_loss_tokens": 8520000, "grad_norm": 0.302734375, "lr": 3e-05, "finish_rate": 0.806, "comp_len": 528.6, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.407, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 93.8, "frames": {"chat": 227}, "mem_gb": 16.05, "mem_gb_teacher": 16.05}
+{"step": 72, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.04197514075435077, "tokens": 120000, "cumulative_loss_tokens": 8640000, "grad_norm": 0.275390625, "lr": 3e-05, "finish_rate": 0.796, "comp_len": 568.7, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.434, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 89.1, "frames": {"chat": 211}, "mem_gb": 16.03, "mem_gb_teacher": 16.03}
+{"step": 73, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.03433373855294194, "tokens": 120000, "cumulative_loss_tokens": 8760000, "grad_norm": 0.248046875, "lr": 3e-05, "finish_rate": 0.861, "comp_len": 504.2, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.403, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 98.0, "frames": {"chat": 238}, "mem_gb": 16.04, "mem_gb_teacher": 16.04}
+{"step": 74, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.035312130354298275, "tokens": 120000, "cumulative_loss_tokens": 8880000, "grad_norm": 0.2255859375, "lr": 3e-05, "finish_rate": 0.835, "comp_len": 506.3, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.517, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 98.2, "frames": {"chat": 237}, "mem_gb": 16.08, "mem_gb_teacher": 16.08}
+{"step": 75, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.04418459653495035, "tokens": 120000, "cumulative_loss_tokens": 9000000, "grad_norm": 0.265625, "lr": 3e-05, "finish_rate": 0.721, "comp_len": 576.9, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.339, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 88.6, "frames": {"chat": 208}, "mem_gb": 16.08, "mem_gb_teacher": 16.08}
+{"step": 76, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.03462567985369048, "tokens": 120000, "cumulative_loss_tokens": 9120000, "grad_norm": 0.212890625, "lr": 3e-05, "finish_rate": 0.801, "comp_len": 543.0, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.588, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 91.3, "frames": {"chat": 221}, "mem_gb": 16.17, "mem_gb_teacher": 16.17}
+{"step": 77, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.0355419263230792, "tokens": 120000, "cumulative_loss_tokens": 9240000, "grad_norm": 0.244140625, "lr": 3e-05, "finish_rate": 0.853, "comp_len": 517.2, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.479, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 96.6, "frames": {"chat": 232}, "mem_gb": 16.01, "mem_gb_teacher": 16.01}
+{"step": 78, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.03847126886160113, "tokens": 120000, "cumulative_loss_tokens": 9360000, "grad_norm": 0.25, "lr": 3e-05, "finish_rate": 0.764, "comp_len": 576.9, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.43, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 87.7, "frames": {"chat": 208}, "mem_gb": 16.04, "mem_gb_teacher": 16.04}
+{"step": 79, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.03187043257508582, "tokens": 120000, "cumulative_loss_tokens": 9480000, "grad_norm": 0.232421875, "lr": 3e-05, "finish_rate": 0.837, "comp_len": 528.6, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.356, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 93.2, "frames": {"chat": 227}, "mem_gb": 15.96, "mem_gb_teacher": 15.96}
+{"step": 80, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.03652716889477645, "tokens": 120000, "cumulative_loss_tokens": 9600000, "grad_norm": 0.251953125, "lr": 3e-05, "finish_rate": 0.824, "comp_len": 543.0, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.387, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 91.2, "frames": {"chat": 221}, "mem_gb": 15.99, "mem_gb_teacher": 15.99}
+{"step": 81, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.03261745297779174, "tokens": 120000, "cumulative_loss_tokens": 9720000, "grad_norm": 0.2138671875, "lr": 3e-05, "finish_rate": 0.815, "comp_len": 517.2, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.457, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 95.5, "frames": {"chat": 232}, "mem_gb": 16.05, "mem_gb_teacher": 16.05}
+{"step": 82, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.03819663266551991, "tokens": 120000, "cumulative_loss_tokens": 9840000, "grad_norm": 0.255859375, "lr": 3e-05, "finish_rate": 0.822, "comp_len": 547.9, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.423, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 90.2, "frames": {"chat": 219}, "mem_gb": 16.05, "mem_gb_teacher": 16.05}
+{"step": 83, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.03785421463410991, "tokens": 120000, "cumulative_loss_tokens": 9960000, "grad_norm": 0.2265625, "lr": 3e-05, "finish_rate": 0.713, "comp_len": 615.4, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.399, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 82.7, "frames": {"chat": 195}, "mem_gb": 16.14, "mem_gb_teacher": 16.14}
+{"step": 84, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.03711687127229913, "tokens": 120000, "cumulative_loss_tokens": 10080000, "grad_norm": 0.244140625, "lr": 3e-05, "finish_rate": 0.833, "comp_len": 555.6, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.503, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 88.9, "frames": {"chat": 216}, "mem_gb": 16.05, "mem_gb_teacher": 16.05}
+{"step": 85, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.042023429202785095, "tokens": 120000, "cumulative_loss_tokens": 10200000, "grad_norm": 0.275390625, "lr": 3e-05, "finish_rate": 0.788, "comp_len": 576.9, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.37, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 87.7, "frames": {"chat": 208}, "mem_gb": 15.93, "mem_gb_teacher": 15.93}
+{"step": 86, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.03288566896258077, "tokens": 120000, "cumulative_loss_tokens": 10320000, "grad_norm": 0.2265625, "lr": 3e-05, "finish_rate": 0.919, "comp_len": 510.6, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.407, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 95.6, "frames": {"chat": 235}, "mem_gb": 15.93, "mem_gb_teacher": 15.93}
+{"step": 87, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.033694713056855834, "tokens": 120000, "cumulative_loss_tokens": 10440000, "grad_norm": 0.2578125, "lr": 3e-05, "finish_rate": 0.853, "comp_len": 533.3, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.391, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 91.6, "frames": {"chat": 225}, "mem_gb": 16.04, "mem_gb_teacher": 16.04}
+{"step": 88, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.04422600561644261, "tokens": 120000, "cumulative_loss_tokens": 10560000, "grad_norm": 0.2451171875, "lr": 3e-05, "finish_rate": 0.77, "comp_len": 563.4, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.483, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 89.2, "frames": {"chat": 213}, "mem_gb": 16.13, "mem_gb_teacher": 16.13}
+{"step": 89, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.029030231156169126, "tokens": 120000, "cumulative_loss_tokens": 10680000, "grad_norm": 0.2255859375, "lr": 3e-05, "finish_rate": 0.922, "comp_len": 466.9, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.372, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 104.0, "frames": {"chat": 257}, "mem_gb": 15.8, "mem_gb_teacher": 15.8}
+{"step": 90, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.043001266495510934, "tokens": 120000, "cumulative_loss_tokens": 10800000, "grad_norm": 0.2578125, "lr": 3e-05, "finish_rate": 0.792, "comp_len": 566.0, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.497, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 89.1, "frames": {"chat": 212}, "mem_gb": 16.07, "mem_gb_teacher": 16.07}
+{"step": 91, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.03456530287990657, "tokens": 120000, "cumulative_loss_tokens": 10920000, "grad_norm": 0.263671875, "lr": 3e-05, "finish_rate": 0.833, "comp_len": 543.0, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.338, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 90.9, "frames": {"chat": 221}, "mem_gb": 16.05, "mem_gb_teacher": 16.05}
+{"step": 92, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.030914492412268495, "tokens": 120000, "cumulative_loss_tokens": 11040000, "grad_norm": 0.203125, "lr": 3e-05, "finish_rate": 0.868, "comp_len": 495.9, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.442, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 98.2, "frames": {"chat": 242}, "mem_gb": 16.04, "mem_gb_teacher": 16.04}
+{"step": 93, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.0329031198489829, "tokens": 120000, "cumulative_loss_tokens": 11160000, "grad_norm": 0.2216796875, "lr": 3e-05, "finish_rate": 0.836, "comp_len": 545.5, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.352, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 91.5, "frames": {"chat": 220}, "mem_gb": 16.01, "mem_gb_teacher": 16.01}
+{"step": 94, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.029550562638859263, "tokens": 120000, "cumulative_loss_tokens": 11280000, "grad_norm": 0.2197265625, "lr": 3e-05, "finish_rate": 0.896, "comp_len": 500.0, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.291, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 96.4, "frames": {"chat": 240}, "mem_gb": 15.9, "mem_gb_teacher": 15.9}
+{"step": 95, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.0336006456746875, "tokens": 120000, "cumulative_loss_tokens": 11400000, "grad_norm": 0.23046875, "lr": 3e-05, "finish_rate": 0.728, "comp_len": 582.5, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.359, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 87.0, "frames": {"chat": 206}, "mem_gb": 16.03, "mem_gb_teacher": 16.03}
+{"step": 96, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.04226834708025368, "tokens": 120000, "cumulative_loss_tokens": 11520000, "grad_norm": 0.255859375, "lr": 3e-05, "finish_rate": 0.867, "comp_len": 531.0, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.402, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 92.9, "frames": {"chat": 226}, "mem_gb": 16.05, "mem_gb_teacher": 16.05}
+{"step": 97, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.054896473065484314, "tokens": 120000, "cumulative_loss_tokens": 11640000, "grad_norm": 0.3046875, "lr": 3e-05, "finish_rate": 0.877, "comp_len": 491.8, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.382, "t_data_s": 0.1, "t_rollout_s": 0.0, "t_step_s": 99.8, "frames": {"chat": 244}, "mem_gb": 15.83, "mem_gb_teacher": 15.83}
+{"step": 98, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.040829892701484884, "tokens": 120000, "cumulative_loss_tokens": 11760000, "grad_norm": 0.271484375, "lr": 3e-05, "finish_rate": 0.804, "comp_len": 535.7, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.406, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 92.6, "frames": {"chat": 224}, "mem_gb": 16.05, "mem_gb_teacher": 16.05}
+{"step": 99, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.031381215056039705, "tokens": 120000, "cumulative_loss_tokens": 11880000, "grad_norm": 0.240234375, "lr": 3e-05, "finish_rate": 0.923, "comp_len": 442.8, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.319, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 109.0, "frames": {"chat": 271}, "mem_gb": 15.78, "mem_gb_teacher": 15.78}
+{"step": 100, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.036837965581729075, "tokens": 120000, "cumulative_loss_tokens": 12000000, "grad_norm": 0.265625, "lr": 3e-05, "finish_rate": 0.856, "comp_len": 508.5, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.362, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 96.5, "frames": {"chat": 236}, "mem_gb": 16.06, "mem_gb_teacher": 16.06}
+{"step": 101, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.03681235469069021, "tokens": 120000, "cumulative_loss_tokens": 12120000, "grad_norm": 0.244140625, "lr": 3e-05, "finish_rate": 0.841, "comp_len": 517.2, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.306, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 96.0, "frames": {"chat": 232}, "mem_gb": 15.93, "mem_gb_teacher": 15.93}
+{"step": 102, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.03435125927827321, "tokens": 120000, "cumulative_loss_tokens": 12240000, "grad_norm": 0.251953125, "lr": 3e-05, "finish_rate": 0.79, "comp_len": 571.4, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.392, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 87.6, "frames": {"chat": 210}, "mem_gb": 15.98, "mem_gb_teacher": 15.98}
+{"step": 103, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.033812837561887375, "tokens": 120000, "cumulative_loss_tokens": 12360000, "grad_norm": 0.275390625, "lr": 3e-05, "finish_rate": 0.811, "comp_len": 553.0, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.431, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 89.9, "frames": {"chat": 217}, "mem_gb": 15.95, "mem_gb_teacher": 15.95}
+{"step": 104, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.03568466838332825, "tokens": 120000, "cumulative_loss_tokens": 12480000, "grad_norm": 0.25390625, "lr": 3e-05, "finish_rate": 0.839, "comp_len": 535.7, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.38, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 92.7, "frames": {"chat": 224}, "mem_gb": 16.07, "mem_gb_teacher": 16.07}
+{"step": 105, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.04380085541328881, "tokens": 120000, "cumulative_loss_tokens": 12600000, "grad_norm": 0.287109375, "lr": 3e-05, "finish_rate": 0.749, "comp_len": 591.1, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.481, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 86.5, "frames": {"chat": 203}, "mem_gb": 15.92, "mem_gb_teacher": 15.92}
+{"step": 106, "epoch": 1, "training_mode": "off-policy", "forward_topk_kl": 0.0331037232719129, "tokens": 120000, "cumulative_loss_tokens": 12720000, "grad_norm": 0.2197265625, "lr": 3e-05, "finish_rate": 0.887, "comp_len": 502.1, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.326, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 97.2, "frames": {"chat": 239}, "mem_gb": 16.02, "mem_gb_teacher": 16.02}
+{"step": 107, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.022136363052297384, "tokens": 120000, "cumulative_loss_tokens": 12840000, "grad_norm": 0.1875, "lr": 3e-05, "finish_rate": 0.902, "comp_len": 472.4, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.301, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 102.8, "frames": {"chat": 254}, "mem_gb": 15.93, "mem_gb_teacher": 15.93}
+{"step": 108, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.02345696447007358, "tokens": 120000, "cumulative_loss_tokens": 12960000, "grad_norm": 0.21875, "lr": 3e-05, "finish_rate": 0.876, "comp_len": 497.9, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.421, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 98.5, "frames": {"chat": 241}, "mem_gb": 16.03, "mem_gb_teacher": 16.03}
+{"step": 109, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.03220615579979494, "tokens": 120000, "cumulative_loss_tokens": 13080000, "grad_norm": 0.2294921875, "lr": 3e-05, "finish_rate": 0.746, "comp_len": 563.4, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.419, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 89.1, "frames": {"chat": 213}, "mem_gb": 16.05, "mem_gb_teacher": 16.05}
+{"step": 110, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.028369275209781095, "tokens": 120000, "cumulative_loss_tokens": 13200000, "grad_norm": 0.205078125, "lr": 3e-05, "finish_rate": 0.864, "comp_len": 543.0, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.641, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 91.4, "frames": {"chat": 221}, "mem_gb": 16.1, "mem_gb_teacher": 16.1}
+{"step": 111, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.030552633555675855, "tokens": 120000, "cumulative_loss_tokens": 13320000, "grad_norm": 0.201171875, "lr": 3e-05, "finish_rate": 0.745, "comp_len": 612.2, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.334, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 83.1, "frames": {"chat": 196}, "mem_gb": 16.06, "mem_gb_teacher": 16.06}
+{"step": 112, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.02257079966662762, "tokens": 120000, "cumulative_loss_tokens": 13440000, "grad_norm": 0.1689453125, "lr": 3e-05, "finish_rate": 0.926, "comp_len": 444.4, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.427, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 108.7, "frames": {"chat": 270}, "mem_gb": 15.86, "mem_gb_teacher": 15.86}
+{"step": 113, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.024793952927171875, "tokens": 120000, "cumulative_loss_tokens": 13560000, "grad_norm": 0.2041015625, "lr": 3e-05, "finish_rate": 0.815, "comp_len": 555.6, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.327, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 89.5, "frames": {"chat": 216}, "mem_gb": 16.04, "mem_gb_teacher": 16.04}
+{"step": 114, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.02779970950682958, "tokens": 120000, "cumulative_loss_tokens": 13680000, "grad_norm": 0.1923828125, "lr": 3e-05, "finish_rate": 0.775, "comp_len": 600.0, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.314, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 83.9, "frames": {"chat": 200}, "mem_gb": 16.01, "mem_gb_teacher": 16.01}
+{"step": 115, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.025435146312182768, "tokens": 120000, "cumulative_loss_tokens": 13800000, "grad_norm": 0.19921875, "lr": 3e-05, "finish_rate": 0.767, "comp_len": 582.5, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.406, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 86.4, "frames": {"chat": 206}, "mem_gb": 15.96, "mem_gb_teacher": 15.96}
+{"step": 116, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.021958818318624982, "tokens": 120000, "cumulative_loss_tokens": 13920000, "grad_norm": 0.1650390625, "lr": 3e-05, "finish_rate": 0.902, "comp_len": 512.8, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.5, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 94.8, "frames": {"chat": 234}, "mem_gb": 15.99, "mem_gb_teacher": 15.99}
+{"step": 117, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.024690080278979926, "tokens": 120000, "cumulative_loss_tokens": 14040000, "grad_norm": 0.171875, "lr": 3e-05, "finish_rate": 0.823, "comp_len": 558.1, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.325, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 88.4, "frames": {"chat": 215}, "mem_gb": 16.01, "mem_gb_teacher": 16.01}
+{"step": 118, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.02221767870101612, "tokens": 120000, "cumulative_loss_tokens": 14160000, "grad_norm": 0.2021484375, "lr": 3e-05, "finish_rate": 0.922, "comp_len": 470.6, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.447, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 101.5, "frames": {"chat": 255}, "mem_gb": 15.99, "mem_gb_teacher": 15.99}
+{"step": 119, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.021592201069470806, "tokens": 120000, "cumulative_loss_tokens": 14280000, "grad_norm": 0.1708984375, "lr": 3e-05, "finish_rate": 0.892, "comp_len": 480.0, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.377, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 101.8, "frames": {"chat": 250}, "mem_gb": 15.87, "mem_gb_teacher": 15.87}
+{"step": 120, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.022036931684900386, "tokens": 120000, "cumulative_loss_tokens": 14400000, "grad_norm": 0.1826171875, "lr": 3e-05, "finish_rate": 0.884, "comp_len": 495.9, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.525, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 97.8, "frames": {"chat": 242}, "mem_gb": 16.04, "mem_gb_teacher": 16.04}
+{"step": 121, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.027385966173838823, "tokens": 120000, "cumulative_loss_tokens": 14520000, "grad_norm": 0.177734375, "lr": 3e-05, "finish_rate": 0.729, "comp_len": 603.0, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.517, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 85.1, "frames": {"chat": 199}, "mem_gb": 16.05, "mem_gb_teacher": 16.05}
+{"step": 122, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.037581453007894255, "tokens": 120000, "cumulative_loss_tokens": 14640000, "grad_norm": 0.2138671875, "lr": 3e-05, "finish_rate": 0.784, "comp_len": 576.9, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.386, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 88.8, "frames": {"chat": 208}, "mem_gb": 16.08, "mem_gb_teacher": 16.08}
+{"step": 123, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.02500725610067602, "tokens": 120000, "cumulative_loss_tokens": 14760000, "grad_norm": 0.212890625, "lr": 3e-05, "finish_rate": 0.764, "comp_len": 576.9, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.304, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 86.8, "frames": {"chat": 208}, "mem_gb": 16.02, "mem_gb_teacher": 16.02}
+{"step": 124, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.029994825281358013, "tokens": 120000, "cumulative_loss_tokens": 14880000, "grad_norm": 0.2041015625, "lr": 3e-05, "finish_rate": 0.732, "comp_len": 574.2, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.473, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 89.1, "frames": {"chat": 209}, "mem_gb": 16.17, "mem_gb_teacher": 16.17}
+{"step": 125, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.021332296466493667, "tokens": 120000, "cumulative_loss_tokens": 15000000, "grad_norm": 0.1728515625, "lr": 3e-05, "finish_rate": 0.855, "comp_len": 510.6, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.321, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 97.2, "frames": {"chat": 235}, "mem_gb": 16.0, "mem_gb_teacher": 16.0}
+{"step": 126, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.02423879373523717, "tokens": 120000, "cumulative_loss_tokens": 15120000, "grad_norm": 0.2138671875, "lr": 3e-05, "finish_rate": 0.74, "comp_len": 588.2, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.503, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 85.5, "frames": {"chat": 204}, "mem_gb": 16.0, "mem_gb_teacher": 16.0}
+{"step": 127, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.03143483543585365, "tokens": 120000, "cumulative_loss_tokens": 15240000, "grad_norm": 0.24609375, "lr": 3e-05, "finish_rate": 0.745, "comp_len": 576.9, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.498, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 88.5, "frames": {"chat": 208}, "mem_gb": 16.05, "mem_gb_teacher": 16.05}
+{"step": 128, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.022677327596287555, "tokens": 120000, "cumulative_loss_tokens": 15360000, "grad_norm": 0.1748046875, "lr": 3e-05, "finish_rate": 0.825, "comp_len": 500.0, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.456, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 98.6, "frames": {"chat": 240}, "mem_gb": 16.05, "mem_gb_teacher": 16.05}
+{"step": 129, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.02317165856284555, "tokens": 120000, "cumulative_loss_tokens": 15480000, "grad_norm": 0.18359375, "lr": 3e-05, "finish_rate": 0.89, "comp_len": 487.8, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.425, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 99.3, "frames": {"chat": 246}, "mem_gb": 16.04, "mem_gb_teacher": 16.04}
+{"step": 130, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.02553549518882452, "tokens": 120000, "cumulative_loss_tokens": 15600000, "grad_norm": 0.16796875, "lr": 3e-05, "finish_rate": 0.909, "comp_len": 493.8, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.299, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 98.1, "frames": {"chat": 243}, "mem_gb": 15.86, "mem_gb_teacher": 15.86}
+{"step": 131, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.03025520235665608, "tokens": 120000, "cumulative_loss_tokens": 15720000, "grad_norm": 0.193359375, "lr": 3e-05, "finish_rate": 0.745, "comp_len": 576.9, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.449, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 87.0, "frames": {"chat": 208}, "mem_gb": 16.06, "mem_gb_teacher": 16.06}
+{"step": 132, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.033440696114464666, "tokens": 120000, "cumulative_loss_tokens": 15840000, "grad_norm": 0.20703125, "lr": 3e-05, "finish_rate": 0.817, "comp_len": 547.9, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.352, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 90.4, "frames": {"chat": 219}, "mem_gb": 16.05, "mem_gb_teacher": 16.05}
+{"step": 133, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.030961406842339785, "tokens": 120000, "cumulative_loss_tokens": 15960000, "grad_norm": 0.205078125, "lr": 3e-05, "finish_rate": 0.782, "comp_len": 568.7, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.447, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 89.4, "frames": {"chat": 211}, "mem_gb": 16.06, "mem_gb_teacher": 16.06}
+{"step": 134, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.030365196120288845, "tokens": 120000, "cumulative_loss_tokens": 16080000, "grad_norm": 0.25, "lr": 3e-05, "finish_rate": 0.862, "comp_len": 517.2, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.283, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 95.3, "frames": {"chat": 232}, "mem_gb": 16.02, "mem_gb_teacher": 16.02}
+{"step": 135, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.03637079153128434, "tokens": 120000, "cumulative_loss_tokens": 16200000, "grad_norm": 0.201171875, "lr": 3e-05, "finish_rate": 0.804, "comp_len": 560.7, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.403, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 90.4, "frames": {"chat": 214}, "mem_gb": 16.05, "mem_gb_teacher": 16.05}
+{"step": 136, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.024788761290698312, "tokens": 120000, "cumulative_loss_tokens": 16320000, "grad_norm": 0.173828125, "lr": 3e-05, "finish_rate": 0.819, "comp_len": 531.0, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.395, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 94.4, "frames": {"chat": 226}, "mem_gb": 15.94, "mem_gb_teacher": 15.94}
+{"step": 137, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.02415202263732596, "tokens": 120000, "cumulative_loss_tokens": 16440000, "grad_norm": 0.171875, "lr": 3e-05, "finish_rate": 0.819, "comp_len": 571.4, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.404, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 88.7, "frames": {"chat": 210}, "mem_gb": 16.06, "mem_gb_teacher": 16.06}
+{"step": 138, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.023906889412474507, "tokens": 120000, "cumulative_loss_tokens": 16560000, "grad_norm": 0.1884765625, "lr": 3e-05, "finish_rate": 0.858, "comp_len": 550.5, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.436, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 91.6, "frames": {"chat": 218}, "mem_gb": 15.88, "mem_gb_teacher": 15.88}
+{"step": 139, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.02281373731412459, "tokens": 120000, "cumulative_loss_tokens": 16680000, "grad_norm": 0.185546875, "lr": 3e-05, "finish_rate": 0.858, "comp_len": 515.0, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.354, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 96.0, "frames": {"chat": 233}, "mem_gb": 16.04, "mem_gb_teacher": 16.04}
+{"step": 140, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.029389094779423128, "tokens": 120000, "cumulative_loss_tokens": 16800000, "grad_norm": 0.1962890625, "lr": 3e-05, "finish_rate": 0.786, "comp_len": 558.1, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.221, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 91.1, "frames": {"chat": 215}, "mem_gb": 16.05, "mem_gb_teacher": 16.05}
+{"step": 141, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.02496639485837271, "tokens": 120000, "cumulative_loss_tokens": 16920000, "grad_norm": 0.1826171875, "lr": 3e-05, "finish_rate": 0.845, "comp_len": 515.0, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.428, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 95.0, "frames": {"chat": 233}, "mem_gb": 16.04, "mem_gb_teacher": 16.04}
+{"step": 142, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.024896287856064736, "tokens": 120000, "cumulative_loss_tokens": 17040000, "grad_norm": 0.2138671875, "lr": 3e-05, "finish_rate": 0.766, "comp_len": 574.2, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.353, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 87.5, "frames": {"chat": 209}, "mem_gb": 15.99, "mem_gb_teacher": 15.99}
+{"step": 143, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.0202058710468933, "tokens": 120000, "cumulative_loss_tokens": 17160000, "grad_norm": 0.166015625, "lr": 3e-05, "finish_rate": 0.908, "comp_len": 458.0, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.308, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 105.8, "frames": {"chat": 262}, "mem_gb": 15.92, "mem_gb_teacher": 15.92}
+{"step": 144, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.0228134301078273, "tokens": 120000, "cumulative_loss_tokens": 17280000, "grad_norm": 0.171875, "lr": 3e-05, "finish_rate": 0.9, "comp_len": 481.9, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.241, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 101.0, "frames": {"chat": 249}, "mem_gb": 16.01, "mem_gb_teacher": 16.01}
+{"step": 145, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.030403959429240787, "tokens": 120000, "cumulative_loss_tokens": 17400000, "grad_norm": 0.18359375, "lr": 3e-05, "finish_rate": 0.819, "comp_len": 528.6, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.364, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 94.0, "frames": {"chat": 227}, "mem_gb": 16.04, "mem_gb_teacher": 16.04}
+{"step": 146, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.022969909678919553, "tokens": 120000, "cumulative_loss_tokens": 17520000, "grad_norm": 0.1796875, "lr": 3e-05, "finish_rate": 0.814, "comp_len": 543.0, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.356, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 91.2, "frames": {"chat": 221}, "mem_gb": 16.04, "mem_gb_teacher": 16.04}
+{"step": 147, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.02228280872052225, "tokens": 120000, "cumulative_loss_tokens": 17640000, "grad_norm": 0.171875, "lr": 3e-05, "finish_rate": 0.859, "comp_len": 512.8, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.473, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 95.0, "frames": {"chat": 234}, "mem_gb": 16.06, "mem_gb_teacher": 16.06}
+{"step": 148, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.023852225604599032, "tokens": 120000, "cumulative_loss_tokens": 17760000, "grad_norm": 0.212890625, "lr": 3e-05, "finish_rate": 0.817, "comp_len": 563.4, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.29, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 87.8, "frames": {"chat": 213}, "mem_gb": 16.0, "mem_gb_teacher": 16.0}
+{"step": 149, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.022865247969034438, "tokens": 120000, "cumulative_loss_tokens": 17880000, "grad_norm": 0.1748046875, "lr": 3e-05, "finish_rate": 0.836, "comp_len": 563.4, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.455, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 88.1, "frames": {"chat": 213}, "mem_gb": 15.94, "mem_gb_teacher": 15.94}
+{"step": 150, "epoch": 2, "training_mode": "off-policy", "forward_topk_kl": 0.021284172641811892, "tokens": 120000, "cumulative_loss_tokens": 18000000, "grad_norm": 0.1669921875, "lr": 3e-05, "finish_rate": 0.906, "comp_len": 512.8, "dropped_truncated": 0, "gold_loss": null, "gold_lambda": null, "rep_ratio": 2.392, "t_data_s": 0.0, "t_rollout_s": 0.0, "t_step_s": 95.1, "frames": {"chat": 234}, "mem_gb": 15.97, "mem_gb_teacher": 15.97}

healed/opd_warm_unleashed/args.json

@@ -0,0 +1,71 @@
+{
+  "student": "outputs/healed/keep50_warmup_fixed_s1224/step0150",
+  "teacher": "allenai/OLMoE-1B-7B-0125-Instruct",
+  "training_mode": "on-policy",
+  "kl_direction": "reverse",
+  "dataset": "allenai/Dolci-Instruct-RL",
+  "dataset_sources": null,
+  "max_difficulty": null,
+  "trajectories": "outputs/teacher_trajectories/dolci_math_curated.jsonl",
+  "trajectory_dataset": "allenai/Dolci-Instruct-RL",
+  "off_policy_frames": "chat",
+  "off_policy_max_seq_len": 2048,
+  "topk_targets": null,
+  "max_loss_tokens": null,
+  "loss_tokens_per_step": null,
+  "teacher_device": "cuda:0",
+  "student_device": "cuda:1",
+  "lr": 1e-05,
+  "optimizer": "adamw8bit",
+  "weight_decay": 0.1,
+  "epochs": 2,
+  "prompts_per_step": 256,
+  "group_size": 4,
+  "rollout_batch": 64,
+  "micro_batch": 4,
+  "max_new_tokens": 2048,
+  "max_prompt_len": 1024,
+  "warmup_steps": 10,
+  "max_grad_norm": 1.0,
+  "eval_every": 10,
+  "gsm8k_every": 50,
+  "gsm8k_n": 256,
+  "gsm8k_batch": 16,
+  "gsm8k_max_new_tokens": 1024,
+  "gsm8k_frames": "chat",
+  "save_every": 50,
+  "out_dir": "outputs/healed/opd_warm_unleashed",
+  "sweep": 500,
+  "wandb": true,
+  "wandb_project": "glean-heal",
+  "wandb_run_name": "opd-warm-unleashed2-keep50-s1223",
+  "wandb_run_id": null,
+  "wandb_resume": null,
+  "wandb_mode": "offline",
+  "no_wandb_sync": false,
+  "debug": false,
+  "resume_from": "outputs/healed/opd_warm_unleashed/step0300",
+  "start_step": 300,
+  "no_grad_checkpointing": false,
+  "seed": 1223,
+  "no_teacher_overlap": false,
+  "sync_checkpoints": false,
+  "rollout_engine": "vllm",
+  "vllm_gpu": "2",
+  "vllm_port": 8377,
+  "vllm_refresh_every": 1,
+  "vllm_serve_bin": "vllm-plugin/.venv/bin/python",
+  "vllm_gpu_mem_util": 0.85,
+  "liger_loss": true,
+  "gold_mix_lambda": 0.25,
+  "gold_topk_targets": "outputs/teacher_trajectories/dolci_combined_top128",
+  "gold_loss": "ce",
+  "gold_mix_decay": 0.005,
+  "fast_teacher": false,
+  "reference_kl_beta": 0.0,
+  "drop_truncated_rollouts": false,
+  "vllm_max_model_len": null,
+  "vllm_refresh_mode": "reload",
+  "vllm_live_dir": null,
+  "resolved_kl_direction": "reverse"
+}

healed/opd_warm_unleashed/step0450/chat_template.jinja

@@ -0,0 +1,9 @@
+{{ bos_token }}{% for message in messages %}{% if message['role'] == 'system' %}{{ '<|system|>
+' + message['content'] + '
+' }}{% elif message['role'] == 'user' %}{{ '<|user|>
+' + message['content'] + '
+' }}{% elif message['role'] == 'assistant' %}{% if not loop.last %}{{ '<|assistant|>
+'  + message['content'] + eos_token + '
+' }}{% else %}{{ '<|assistant|>
+'  + message['content'] + eos_token }}{% endif %}{% endif %}{% if loop.last and add_generation_prompt %}{{ '<|assistant|>
+' }}{% endif %}{% endfor %}

healed/opd_warm_unleashed/step0450/config.json

@@ -0,0 +1,887 @@
+{
+  "architectures": [
+    "PrunedOlmoeForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoConfig": "configuration_pruned_olmoe.PrunedOlmoeConfig",
+    "AutoModelForCausalLM": "modeling_pruned_olmoe.PrunedOlmoeForCausalLM"
+  },
+  "clip_qkv": null,
+  "dtype": "bfloat16",
+  "eos_token_id": 50279,
+  "expert_widths": [
+    [
+      1024,
+      384,
+      256,
+      256,
+      768,
+      1024,
+      128,
+      768,
+      384,
+      128,
+      1024,
+      384,
+      384,
+      128,
+      640,
+      896,
+      896,
+      256,
+      768,
+      128,
+      768,
+      768,
+      768,
+      768,
+      640,
+      896,
+      256,
+      128,
+      384,
+      896,
+      256,
+      768,
+      512,
+      640,
+      768,
+      896,
+      1024,
+      384,
+      384,
+      512,
+      384,
+      256,
+      768,
+      640,
+      896,
+      896,
+      512,
+      768,
+      512,
+      384,
+      1024,
+      896,
+      896,
+      768,
+      128
+    ],
+    [
+      768,
+      896,
+      896,
+      256,
+      896,
+      384,
+      768,
+      768,
+      1024,
+      384,
+      512,
+      1024,
+      768,
+      768,
+      1024,
+      640,
+      128,
+      768,
+      384,
+      768,
+      128,
+      256,
+      512,
+      384,
+      896,
+      256,
+      1024,
+      1024,
+      896,
+      1024,
+      896,
+      768,
+      256,
+      768,
+      128,
+      1024,
+      768,
+      768,
+      640,
+      384,
+      1024,
+      384,
+      640,
+      896,
+      1024,
+      512,
+      512,
+      512,
+      640
+    ],
+    [
+      768,
+      1024,
+      512,
+      384,
+      384,
+      768,
+      256,
+      384,
+      1024,
+      896,
+      640,
+      256,
+      1024,
+      1024,
+      1024,
+      640,
+      1024,
+      896,
+      512,
+      768,
+      128,
+      768,
+      1024,
+      768,
+      256,
+      1024,
+      896,
+      1024,
+      896,
+      512,
+      640,
+      384,
+      896,
+      384,
+      512,
+      640,
+      256,
+      768,
+      256,
+      768,
+      768,
+      640,
+      1024,
+      640,
+      896,
+      768,
+      768,
+      256
+    ],
+    [
+      640,
+      640,
+      896,
+      512,
+      640,
+      896,
+      896,
+      384,
+      896,
+      896,
+      384,
+      896,
+      640,
+      256,
+      640,
+      1024,
+      1024,
+      768,
+      1024,
+      896,
+      1024,
+      768,
+      896,
+      256,
+      512,
+      768,
+      1024,
+      256,
+      768,
+      512,
+      256,
+      640,
+      1024,
+      1024,
+      512,
+      1024,
+      768,
+      256,
+      1024,
+      384,
+      896,
+      1024,
+      896,
+      1024,
+      384
+    ],
+    [
+      256,
+      640,
+      640,
+      896,
+      768,
+      896,
+      768,
+      768,
+      896,
+      896,
+      1024,
+      256,
+      512,
+      1024,
+      640,
+      896,
+      512,
+      512,
+      384,
+      384,
+      256,
+      384,
+      384,
+      896,
+      896,
+      768,
+      640,
+      896,
+      768,
+      1024,
+      512,
+      640,
+      512,
+      640,
+      896,
+      512,
+      512,
+      384,
+      640,
+      896,
+      896,
+      896,
+      1024,
+      640,
+      1024,
+      640,
+      1024
+    ],
+    [
+      1024,
+      512,
+      1024,
+      1024,
+      640,
+      896,
+      640,
+      1024,
+      896,
+      384,
+      1024,
+      128,
+      896,
+      768,
+      1024,
+      768,
+      640,
+      896,
+      768,
+      640,
+      512,
+      896,
+      512,
+      640,
+      256,
+      768,
+      640,
+      768,
+      384,
+      896,
+      512,
+      512,
+      256,
+      512,
+      896,
+      256,
+      384,
+      640,
+      512,
+      640,
+      896,
+      512,
+      1024,
+      256,
+      768,
+      1024,
+      768,
+      256,
+      256
+    ],
+    [
+      640,
+      896,
+      1024,
+      896,
+      1024,
+      1024,
+      1024,
+      512,
+      256,
+      256,
+      1024,
+      768,
+      512,
+      768,
+      1024,
+      1024,
+      1024,
+      384,
+      512,
+      1024,
+      512,
+      1024,
+      128,
+      640,
+      640,
+      896,
+      768,
+      128,
+      256,
+      256,
+      256,
+      256,
+      896,
+      1024,
+      1024,
+      384,
+      896,
+      256,
+      896,
+      640,
+      1024,
+      384,
+      640,
+      256,
+      1024,
+      1024,
+      1024
+    ],
+    [
+      1024,
+      384,
+      1024,
+      1024,
+      256,
+      128,
+      256,
+      384,
+      256,
+      384,
+      896,
+      768,
+      896,
+      896,
+      512,
+      896,
+      640,
+      384,
+      384,
+      896,
+      768,
+      384,
+      896,
+      768,
+      768,
+      512,
+      896,
+      768,
+      768,
+      896,
+      128,
+      896,
+      512,
+      256,
+      768,
+      128,
+      384,
+      256,
+      896,
+      896,
+      384,
+      768,
+      512,
+      640,
+      256,
+      768,
+      640,
+      896,
+      384,
+      512,
+      1024,
+      768,
+      384
+    ],
+    [
+      512,
+      768,
+      512,
+      256,
+      128,
+      640,
+      384,
+      640,
+      768,
+      896,
+      640,
+      768,
+      256,
+      384,
+      1024,
+      896,
+      256,
+      896,
+      512,
+      256,
+      896,
+      768,
+      256,
+      896,
+      896,
+      384,
+      896,
+      640,
+      768,
+      512,
+      768,
+      768,
+      1024,
+      768,
+      640,
+      768,
+      384,
+      256,
+      512,
+      896,
+      128,
+      384,
+      256,
+      768,
+      384,
+      256,
+      1024,
+      1024,
+      896,
+      256,
+      1024,
+      256,
+      128,
+      896
+    ],
+    [
+      640,
+      640,
+      896,
+      256,
+      1024,
+      512,
+      1024,
+      768,
+      384,
+      512,
+      256,
+      768,
+      896,
+      768,
+      512,
+      768,
+      768,
+      640,
+      384,
+      768,
+      512,
+      768,
+      768,
+      512,
+      768,
+      128,
+      896,
+      512,
+      768,
+      1024,
+      128,
+      384,
+      768,
+      768,
+      768,
+      384,
+      512,
+      640,
+      768,
+      512,
+      768,
+      1024,
+      640,
+      896,
+      256,
+      1024,
+      384,
+      768,
+      768,
+      768
+    ],
+    [
+      896,
+      512,
+      896,
+      768,
+      384,
+      384,
+      768,
+      512,
+      768,
+      512,
+      1024,
+      640,
+      896,
+      896,
+      256,
+      640,
+      1024,
+      256,
+      896,
+      128,
+      128,
+      128,
+      768,
+      896,
+      384,
+      896,
+      512,
+      896,
+      384,
+      256,
+      640,
+      640,
+      896,
+      768,
+      640,
+      256,
+      896,
+      896,
+      512,
+      128,
+      896,
+      256,
+      256,
+      640,
+      896,
+      896,
+      128,
+      1024,
+      256,
+      384,
+      1024,
+      640,
+      896
+    ],
+    [
+      768,
+      384,
+      640,
+      896,
+      256,
+      128,
+      384,
+      896,
+      128,
+      128,
+      896,
+      256,
+      384,
+      896,
+      512,
+      768,
+      768,
+      512,
+      512,
+      768,
+      896,
+      640,
+      768,
+      896,
+      896,
+      896,
+      640,
+      896,
+      640,
+      512,
+      896,
+      256,
+      512,
+      128,
+      512,
+      384,
+      768,
+      768,
+      1024,
+      256,
+      768,
+      256,
+      768,
+      512,
+      640,
+      1024,
+      128,
+      896,
+      896,
+      896,
+      896,
+      1024
+    ],
+    [
+      512,
+      384,
+      768,
+      640,
+      640,
+      768,
+      1024,
+      896,
+      512,
+      256,
+      640,
+      768,
+      640,
+      896,
+      128,
+      256,
+      896,
+      1024,
+      256,
+      640,
+      512,
+      256,
+      128,
+      512,
+      256,
+      640,
+      768,
+      768,
+      128,
+      128,
+      768,
+      640,
+      1024,
+      1024,
+      768,
+      512,
+      896,
+      768,
+      896,
+      1024,
+      896,
+      896,
+      512,
+      640,
+      1024,
+      512,
+      1024,
+      512,
+      512,
+      512,
+      768
+    ],
+    [
+      896,
+      896,
+      1024,
+      1024,
+      896,
+      128,
+      768,
+      256,
+      1024,
+      256,
+      1024,
+      640,
+      384,
+      256,
+      256,
+      512,
+      768,
+      896,
+      512,
+      768,
+      384,
+      1024,
+      896,
+      896,
+      1024,
+      896,
+      768,
+      896,
+      640,
+      1024,
+      512,
+      896,
+      512,
+      1024,
+      512,
+      512,
+      256,
+      256,
+      256,
+      512,
+      768,
+      128,
+      384,
+      512,
+      896,
+      896,
+      1024,
+      256,
+      384,
+      640
+    ],
+    [
+      896,
+      640,
+      384,
+      512,
+      256,
+      640,
+      1024,
+      384,
+      1024,
+      1024,
+      768,
+      256,
+      1024,
+      768,
+      512,
+      896,
+      256,
+      1024,
+      768,
+      768,
+      768,
+      384,
+      384,
+      256,
+      1024,
+      384,
+      384,
+      384,
+      896,
+      768,
+      640,
+      768,
+      512,
+      896,
+      896,
+      896,
+      896,
+      256,
+      384,
+      128,
+      1024,
+      896,
+      256,
+      256,
+      768,
+      640,
+      896,
+      384,
+      768,
+      512,
+      640
+    ],
+    [
+      896,
+      1024,
+      768,
+      1024,
+      896,
+      256,
+      768,
+      128,
+      128,
+      768,
+      512,
+      896,
+      384,
+      768,
+      1024,
+      256,
+      768,
+      768,
+      256,
+      512,
+      512,
+      640,
+      512,
+      256,
+      768,
+      896,
+      384,
+      1024,
+      640,
+      1024,
+      512,
+      512,
+      384,
+      512,
+      512,
+      1024,
+      384,
+      896,
+      768,
+      384,
+      384,
+      128,
+      384,
+      1024,
+      896,
+      640,
+      768,
+      768,
+      256,
+      640,
+      512,
+      640,
+      384
+    ]
+  ],
+  "glean_metadata": {
+    "base_model": "allenai/OLMoE-1B-7B-0125-Instruct",
+    "block_size": 128,
+    "criterion": "reap",
+    "dead_experts": 217,
+    "keep_fraction": 0.5,
+    "min_width": 128,
+    "params": 3697491968,
+    "scores": "outputs/scores_0125inst_dolmino-math/scores.pt"
+  },
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 1024,
+  "max_position_embeddings": 4096,
+  "model_type": "pruned_olmoe",
+  "norm_topk_prob": false,
+  "num_attention_heads": 16,
+  "num_experts": 64,
+  "num_experts_per_tok": 8,
+  "num_hidden_layers": 16,
+  "num_key_value_heads": 16,
+  "output_router_logits": false,
+  "pad_token_id": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "router_aux_loss_coef": 0.01,
+  "tie_word_embeddings": false,
+  "transformers_version": "4.57.6",
+  "use_cache": false,
+  "vocab_size": 50304
+}

healed/opd_warm_unleashed/step0450/configuration_pruned_olmoe.py

@@ -0,0 +1,27 @@
+"""Configuration for GLEAN-pruned OLMoE: variable-width, variable-count experts.
+"""
+
+from transformers.models.olmoe.configuration_olmoe import OlmoeConfig
+
+
+class PrunedOlmoeConfig(OlmoeConfig):
+    """OlmoeConfig plus a per-(layer, expert) width table.
+
+    ``expert_widths[l]`` lists the SwiGLU intermediate width of each surviving
+    expert in decoder layer ``l``, in expert order. Lists are ragged: layers
+    may keep different numbers of experts (deleted experts simply don't
+    appear — the router in layer ``l`` has ``len(expert_widths[l])`` rows),
+    and each width may differ (multiples of the GEMM block size, 128, for
+    variable-MegaBlocks execution). ``None`` means an unpruned model
+    (uniform ``num_experts`` × ``intermediate_size``).
+
+    The inherited ``num_experts`` / ``intermediate_size`` keep their ORIGINAL
+    (pre-pruning) values for provenance; the width table is authoritative for
+    the built architecture.
+    """
+
+    model_type = "pruned_olmoe"
+
+    def __init__(self, expert_widths: list[list[int]] | None = None, **kwargs):
+        super().__init__(**kwargs)
+        self.expert_widths = expert_widths

healed/opd_warm_unleashed/step0450/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "eos_token_id": 50279,
+  "pad_token_id": 1,
+  "transformers_version": "4.57.6"
+}

healed/opd_warm_unleashed/step0450/modeling_pruned_olmoe.py

@@ -0,0 +1,66 @@
+"""GLEAN-pruned OLMoE: HF-loadable model with ragged (variable-width) experts.
+
+Pattern follows hbfreed/variable-flex-olmo's PrunedFlexOlmoForCausalLM
+(docs/recon/prior-work-hbfreed.md), generalized from one scalar width to a
+per-(layer, expert) width table: ``super().__init__`` builds the uniform
+architecture from the config, then every MoE block is rebuilt to its pruned
+shape — surviving experts only, each at its own width, router sliced to
+match — so the state dict aligns exactly with what
+``glean.prune.prune_channels_global`` leaves behind.
+
+Caveat: ``output_router_logits=True`` (the load-balancing aux loss) assumes a
+uniform ``config.num_experts`` and is unsupported on ragged models.
+"""
+
+import torch.nn as nn
+from transformers.activations import ACT2FN
+from transformers.models.olmoe.modeling_olmoe import OlmoeForCausalLM
+
+from .configuration_pruned_olmoe import PrunedOlmoeConfig
+
+
+class RaggedOlmoeMLP(nn.Module):
+    """OlmoeMLP with an explicit intermediate width (SwiGLU, no biases)."""
+
+    def __init__(self, hidden_size: int, intermediate_size: int, hidden_act: str):
+        super().__init__()
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.gate_proj = nn.Linear(hidden_size, intermediate_size, bias=False)
+        self.up_proj = nn.Linear(hidden_size, intermediate_size, bias=False)
+        self.down_proj = nn.Linear(intermediate_size, hidden_size, bias=False)
+        self.act_fn = ACT2FN[hidden_act]
+
+    def forward(self, x):
+        return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
+
+
+class PrunedOlmoeForCausalLM(OlmoeForCausalLM):
+    """OLMoE with per-layer surviving-expert lists at per-expert widths."""
+
+    config_class = PrunedOlmoeConfig
+
+    def __init__(self, config: PrunedOlmoeConfig):
+        super().__init__(config)
+        widths_table = getattr(config, "expert_widths", None)
+        if widths_table is None:
+            return  # unpruned: plain OLMoE
+        if len(widths_table) != len(self.model.layers):
+            raise ValueError(
+                f"expert_widths has {len(widths_table)} rows but the model has "
+                f"{len(self.model.layers)} decoder layers"
+            )
+        for layer, widths in zip(self.model.layers, widths_table):
+            if any(w <= 0 for w in widths):
+                raise ValueError("expert_widths must list surviving experts only (>0)")
+            block = layer.mlp
+            if len(widths) < block.top_k:
+                raise ValueError(
+                    f"a layer keeps {len(widths)} experts < top_k={block.top_k}"
+                )
+            block.num_experts = len(widths)
+            block.gate = nn.Linear(config.hidden_size, len(widths), bias=False)
+            block.experts = nn.ModuleList(
+                RaggedOlmoeMLP(config.hidden_size, w, config.hidden_act)
+                for w in widths
+            )

healed/opd_warm_unleashed/step0450/special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "|||IP_ADDRESS|||",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "|||IP_ADDRESS|||",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

healed/opd_warm_unleashed/step0450/tokenizer_config.json

@@ -0,0 +1,247 @@
+{
+  "add_bos_token": false,
+  "add_eos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<|padding|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50254": {
+      "content": "                        ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50255": {
+      "content": "                       ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50256": {
+      "content": "                      ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50257": {
+      "content": "                     ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50258": {
+      "content": "                    ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50259": {
+      "content": "                   ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50260": {
+      "content": "                  ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50261": {
+      "content": "                 ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50262": {
+      "content": "                ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50263": {
+      "content": "               ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50264": {
+      "content": "              ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50265": {
+      "content": "             ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50266": {
+      "content": "            ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50267": {
+      "content": "           ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50268": {
+      "content": "          ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50269": {
+      "content": "         ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50270": {
+      "content": "        ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50271": {
+      "content": "       ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50272": {
+      "content": "      ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50273": {
+      "content": "     ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50274": {
+      "content": "    ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50275": {
+      "content": "   ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50276": {
+      "content": "  ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50277": {
+      "content": "|||EMAIL_ADDRESS|||",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50278": {
+      "content": "|||PHONE_NUMBER|||",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50279": {
+      "content": "|||IP_ADDRESS|||",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50280": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "|||IP_ADDRESS|||",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "|||IP_ADDRESS|||",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "tokenizer_class": "GPTNeoXTokenizer",
+  "unk_token": null
+}

healed/opd_warm_unleashed/step0500/chat_template.jinja

@@ -0,0 +1,9 @@
+{{ bos_token }}{% for message in messages %}{% if message['role'] == 'system' %}{{ '<|system|>
+' + message['content'] + '
+' }}{% elif message['role'] == 'user' %}{{ '<|user|>
+' + message['content'] + '
+' }}{% elif message['role'] == 'assistant' %}{% if not loop.last %}{{ '<|assistant|>
+'  + message['content'] + eos_token + '
+' }}{% else %}{{ '<|assistant|>
+'  + message['content'] + eos_token }}{% endif %}{% endif %}{% if loop.last and add_generation_prompt %}{{ '<|assistant|>
+' }}{% endif %}{% endfor %}

healed/opd_warm_unleashed/step0500/config.json

@@ -0,0 +1,887 @@
+{
+  "architectures": [
+    "PrunedOlmoeForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoConfig": "configuration_pruned_olmoe.PrunedOlmoeConfig",
+    "AutoModelForCausalLM": "modeling_pruned_olmoe.PrunedOlmoeForCausalLM"
+  },
+  "clip_qkv": null,
+  "dtype": "bfloat16",
+  "eos_token_id": 50279,
+  "expert_widths": [
+    [
+      1024,
+      384,
+      256,
+      256,
+      768,
+      1024,
+      128,
+      768,
+      384,
+      128,
+      1024,
+      384,
+      384,
+      128,
+      640,
+      896,
+      896,
+      256,
+      768,
+      128,
+      768,
+      768,
+      768,
+      768,
+      640,
+      896,
+      256,
+      128,
+      384,
+      896,
+      256,
+      768,
+      512,
+      640,
+      768,
+      896,
+      1024,
+      384,
+      384,
+      512,
+      384,
+      256,
+      768,
+      640,
+      896,
+      896,
+      512,
+      768,
+      512,
+      384,
+      1024,
+      896,
+      896,
+      768,
+      128
+    ],
+    [
+      768,
+      896,
+      896,
+      256,
+      896,
+      384,
+      768,
+      768,
+      1024,
+      384,
+      512,
+      1024,
+      768,
+      768,
+      1024,
+      640,
+      128,
+      768,
+      384,
+      768,
+      128,
+      256,
+      512,
+      384,
+      896,
+      256,
+      1024,
+      1024,
+      896,
+      1024,
+      896,
+      768,
+      256,
+      768,
+      128,
+      1024,
+      768,
+      768,
+      640,
+      384,
+      1024,
+      384,
+      640,
+      896,
+      1024,
+      512,
+      512,
+      512,
+      640
+    ],
+    [
+      768,
+      1024,
+      512,
+      384,
+      384,
+      768,
+      256,
+      384,
+      1024,
+      896,
+      640,
+      256,
+      1024,
+      1024,
+      1024,
+      640,
+      1024,
+      896,
+      512,
+      768,
+      128,
+      768,
+      1024,
+      768,
+      256,
+      1024,
+      896,
+      1024,
+      896,
+      512,
+      640,
+      384,
+      896,
+      384,
+      512,
+      640,
+      256,
+      768,
+      256,
+      768,
+      768,
+      640,
+      1024,
+      640,
+      896,
+      768,
+      768,
+      256
+    ],
+    [
+      640,
+      640,
+      896,
+      512,
+      640,
+      896,
+      896,
+      384,
+      896,
+      896,
+      384,
+      896,
+      640,
+      256,
+      640,
+      1024,
+      1024,
+      768,
+      1024,
+      896,
+      1024,
+      768,
+      896,
+      256,
+      512,
+      768,
+      1024,
+      256,
+      768,
+      512,
+      256,
+      640,
+      1024,
+      1024,
+      512,
+      1024,
+      768,
+      256,
+      1024,
+      384,
+      896,
+      1024,
+      896,
+      1024,
+      384
+    ],
+    [
+      256,
+      640,
+      640,
+      896,
+      768,
+      896,
+      768,
+      768,
+      896,
+      896,
+      1024,
+      256,
+      512,
+      1024,
+      640,
+      896,
+      512,
+      512,
+      384,
+      384,
+      256,
+      384,
+      384,
+      896,
+      896,
+      768,
+      640,
+      896,
+      768,
+      1024,
+      512,
+      640,
+      512,
+      640,
+      896,
+      512,
+      512,
+      384,
+      640,
+      896,
+      896,
+      896,
+      1024,
+      640,
+      1024,
+      640,
+      1024
+    ],
+    [
+      1024,
+      512,
+      1024,
+      1024,
+      640,
+      896,
+      640,
+      1024,
+      896,
+      384,
+      1024,
+      128,
+      896,
+      768,
+      1024,
+      768,
+      640,
+      896,
+      768,
+      640,
+      512,
+      896,
+      512,
+      640,
+      256,
+      768,
+      640,
+      768,
+      384,
+      896,
+      512,
+      512,
+      256,
+      512,
+      896,
+      256,
+      384,
+      640,
+      512,
+      640,
+      896,
+      512,
+      1024,
+      256,
+      768,
+      1024,
+      768,
+      256,
+      256
+    ],
+    [
+      640,
+      896,
+      1024,
+      896,
+      1024,
+      1024,
+      1024,
+      512,
+      256,
+      256,
+      1024,
+      768,
+      512,
+      768,
+      1024,
+      1024,
+      1024,
+      384,
+      512,
+      1024,
+      512,
+      1024,
+      128,
+      640,
+      640,
+      896,
+      768,
+      128,
+      256,
+      256,
+      256,
+      256,
+      896,
+      1024,
+      1024,
+      384,
+      896,
+      256,
+      896,
+      640,
+      1024,
+      384,
+      640,
+      256,
+      1024,
+      1024,
+      1024
+    ],
+    [
+      1024,
+      384,
+      1024,
+      1024,
+      256,
+      128,
+      256,
+      384,
+      256,
+      384,
+      896,
+      768,
+      896,
+      896,
+      512,
+      896,
+      640,
+      384,
+      384,
+      896,
+      768,
+      384,
+      896,
+      768,
+      768,
+      512,
+      896,
+      768,
+      768,
+      896,
+      128,
+      896,
+      512,
+      256,
+      768,
+      128,
+      384,
+      256,
+      896,
+      896,
+      384,
+      768,
+      512,
+      640,
+      256,
+      768,
+      640,
+      896,
+      384,
+      512,
+      1024,
+      768,
+      384
+    ],
+    [
+      512,
+      768,
+      512,
+      256,
+      128,
+      640,
+      384,
+      640,
+      768,
+      896,
+      640,
+      768,
+      256,
+      384,
+      1024,
+      896,
+      256,
+      896,
+      512,
+      256,
+      896,
+      768,
+      256,
+      896,
+      896,
+      384,
+      896,
+      640,
+      768,
+      512,
+      768,
+      768,
+      1024,
+      768,
+      640,
+      768,
+      384,
+      256,
+      512,
+      896,
+      128,
+      384,
+      256,
+      768,
+      384,
+      256,
+      1024,
+      1024,
+      896,
+      256,
+      1024,
+      256,
+      128,
+      896
+    ],
+    [
+      640,
+      640,
+      896,
+      256,
+      1024,
+      512,
+      1024,
+      768,
+      384,
+      512,
+      256,
+      768,
+      896,
+      768,
+      512,
+      768,
+      768,
+      640,
+      384,
+      768,
+      512,
+      768,
+      768,
+      512,
+      768,
+      128,
+      896,
+      512,
+      768,
+      1024,
+      128,
+      384,
+      768,
+      768,
+      768,
+      384,
+      512,
+      640,
+      768,
+      512,
+      768,
+      1024,
+      640,
+      896,
+      256,
+      1024,
+      384,
+      768,
+      768,
+      768
+    ],
+    [
+      896,
+      512,
+      896,
+      768,
+      384,
+      384,
+      768,
+      512,
+      768,
+      512,
+      1024,
+      640,
+      896,
+      896,
+      256,
+      640,
+      1024,
+      256,
+      896,
+      128,
+      128,
+      128,
+      768,
+      896,
+      384,
+      896,
+      512,
+      896,
+      384,
+      256,
+      640,
+      640,
+      896,
+      768,
+      640,
+      256,
+      896,
+      896,
+      512,
+      128,
+      896,
+      256,
+      256,
+      640,
+      896,
+      896,
+      128,
+      1024,
+      256,
+      384,
+      1024,
+      640,
+      896
+    ],
+    [
+      768,
+      384,
+      640,
+      896,
+      256,
+      128,
+      384,
+      896,
+      128,
+      128,
+      896,
+      256,
+      384,
+      896,
+      512,
+      768,
+      768,
+      512,
+      512,
+      768,
+      896,
+      640,
+      768,
+      896,
+      896,
+      896,
+      640,
+      896,
+      640,
+      512,
+      896,
+      256,
+      512,
+      128,
+      512,
+      384,
+      768,
+      768,
+      1024,
+      256,
+      768,
+      256,
+      768,
+      512,
+      640,
+      1024,
+      128,
+      896,
+      896,
+      896,
+      896,
+      1024
+    ],
+    [
+      512,
+      384,
+      768,
+      640,
+      640,
+      768,
+      1024,
+      896,
+      512,
+      256,
+      640,
+      768,
+      640,
+      896,
+      128,
+      256,
+      896,
+      1024,
+      256,
+      640,
+      512,
+      256,
+      128,
+      512,
+      256,
+      640,
+      768,
+      768,
+      128,
+      128,
+      768,
+      640,
+      1024,
+      1024,
+      768,
+      512,
+      896,
+      768,
+      896,
+      1024,
+      896,
+      896,
+      512,
+      640,
+      1024,
+      512,
+      1024,
+      512,
+      512,
+      512,
+      768
+    ],
+    [
+      896,
+      896,
+      1024,
+      1024,
+      896,
+      128,
+      768,
+      256,
+      1024,
+      256,
+      1024,
+      640,
+      384,
+      256,
+      256,
+      512,
+      768,
+      896,
+      512,
+      768,
+      384,
+      1024,
+      896,
+      896,
+      1024,
+      896,
+      768,
+      896,
+      640,
+      1024,
+      512,
+      896,
+      512,
+      1024,
+      512,
+      512,
+      256,
+      256,
+      256,
+      512,
+      768,
+      128,
+      384,
+      512,
+      896,
+      896,
+      1024,
+      256,
+      384,
+      640
+    ],
+    [
+      896,
+      640,
+      384,
+      512,
+      256,
+      640,
+      1024,
+      384,
+      1024,
+      1024,
+      768,
+      256,
+      1024,
+      768,
+      512,
+      896,
+      256,
+      1024,
+      768,
+      768,
+      768,
+      384,
+      384,
+      256,
+      1024,
+      384,
+      384,
+      384,
+      896,
+      768,
+      640,
+      768,
+      512,
+      896,
+      896,
+      896,
+      896,
+      256,
+      384,
+      128,
+      1024,
+      896,
+      256,
+      256,
+      768,
+      640,
+      896,
+      384,
+      768,
+      512,
+      640
+    ],
+    [
+      896,
+      1024,
+      768,
+      1024,
+      896,
+      256,
+      768,
+      128,
+      128,
+      768,
+      512,
+      896,
+      384,
+      768,
+      1024,
+      256,
+      768,
+      768,
+      256,
+      512,
+      512,
+      640,
+      512,
+      256,
+      768,
+      896,
+      384,
+      1024,
+      640,
+      1024,
+      512,
+      512,
+      384,
+      512,
+      512,
+      1024,
+      384,
+      896,
+      768,
+      384,
+      384,
+      128,
+      384,
+      1024,
+      896,
+      640,
+      768,
+      768,
+      256,
+      640,
+      512,
+      640,
+      384
+    ]
+  ],
+  "glean_metadata": {
+    "base_model": "allenai/OLMoE-1B-7B-0125-Instruct",
+    "block_size": 128,
+    "criterion": "reap",
+    "dead_experts": 217,
+    "keep_fraction": 0.5,
+    "min_width": 128,
+    "params": 3697491968,
+    "scores": "outputs/scores_0125inst_dolmino-math/scores.pt"
+  },
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 1024,
+  "max_position_embeddings": 4096,
+  "model_type": "pruned_olmoe",
+  "norm_topk_prob": false,
+  "num_attention_heads": 16,
+  "num_experts": 64,
+  "num_experts_per_tok": 8,
+  "num_hidden_layers": 16,
+  "num_key_value_heads": 16,
+  "output_router_logits": false,
+  "pad_token_id": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "router_aux_loss_coef": 0.01,
+  "tie_word_embeddings": false,
+  "transformers_version": "4.57.6",
+  "use_cache": false,
+  "vocab_size": 50304
+}

healed/opd_warm_unleashed/step0500/configuration_pruned_olmoe.py

@@ -0,0 +1,27 @@
+"""Configuration for GLEAN-pruned OLMoE: variable-width, variable-count experts.
+"""
+
+from transformers.models.olmoe.configuration_olmoe import OlmoeConfig
+
+
+class PrunedOlmoeConfig(OlmoeConfig):
+    """OlmoeConfig plus a per-(layer, expert) width table.
+
+    ``expert_widths[l]`` lists the SwiGLU intermediate width of each surviving
+    expert in decoder layer ``l``, in expert order. Lists are ragged: layers
+    may keep different numbers of experts (deleted experts simply don't
+    appear — the router in layer ``l`` has ``len(expert_widths[l])`` rows),
+    and each width may differ (multiples of the GEMM block size, 128, for
+    variable-MegaBlocks execution). ``None`` means an unpruned model
+    (uniform ``num_experts`` × ``intermediate_size``).
+
+    The inherited ``num_experts`` / ``intermediate_size`` keep their ORIGINAL
+    (pre-pruning) values for provenance; the width table is authoritative for
+    the built architecture.
+    """
+
+    model_type = "pruned_olmoe"
+
+    def __init__(self, expert_widths: list[list[int]] | None = None, **kwargs):
+        super().__init__(**kwargs)
+        self.expert_widths = expert_widths

healed/opd_warm_unleashed/step0500/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "eos_token_id": 50279,
+  "pad_token_id": 1,
+  "transformers_version": "4.57.6"
+}

healed/opd_warm_unleashed/step0500/modeling_pruned_olmoe.py

@@ -0,0 +1,66 @@
+"""GLEAN-pruned OLMoE: HF-loadable model with ragged (variable-width) experts.
+
+Pattern follows hbfreed/variable-flex-olmo's PrunedFlexOlmoForCausalLM
+(docs/recon/prior-work-hbfreed.md), generalized from one scalar width to a
+per-(layer, expert) width table: ``super().__init__`` builds the uniform
+architecture from the config, then every MoE block is rebuilt to its pruned
+shape — surviving experts only, each at its own width, router sliced to
+match — so the state dict aligns exactly with what
+``glean.prune.prune_channels_global`` leaves behind.
+
+Caveat: ``output_router_logits=True`` (the load-balancing aux loss) assumes a
+uniform ``config.num_experts`` and is unsupported on ragged models.
+"""
+
+import torch.nn as nn
+from transformers.activations import ACT2FN
+from transformers.models.olmoe.modeling_olmoe import OlmoeForCausalLM
+
+from .configuration_pruned_olmoe import PrunedOlmoeConfig
+
+
+class RaggedOlmoeMLP(nn.Module):
+    """OlmoeMLP with an explicit intermediate width (SwiGLU, no biases)."""
+
+    def __init__(self, hidden_size: int, intermediate_size: int, hidden_act: str):
+        super().__init__()
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.gate_proj = nn.Linear(hidden_size, intermediate_size, bias=False)
+        self.up_proj = nn.Linear(hidden_size, intermediate_size, bias=False)
+        self.down_proj = nn.Linear(intermediate_size, hidden_size, bias=False)
+        self.act_fn = ACT2FN[hidden_act]
+
+    def forward(self, x):
+        return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
+
+
+class PrunedOlmoeForCausalLM(OlmoeForCausalLM):
+    """OLMoE with per-layer surviving-expert lists at per-expert widths."""
+
+    config_class = PrunedOlmoeConfig
+
+    def __init__(self, config: PrunedOlmoeConfig):
+        super().__init__(config)
+        widths_table = getattr(config, "expert_widths", None)
+        if widths_table is None:
+            return  # unpruned: plain OLMoE
+        if len(widths_table) != len(self.model.layers):
+            raise ValueError(
+                f"expert_widths has {len(widths_table)} rows but the model has "
+                f"{len(self.model.layers)} decoder layers"
+            )
+        for layer, widths in zip(self.model.layers, widths_table):
+            if any(w <= 0 for w in widths):
+                raise ValueError("expert_widths must list surviving experts only (>0)")
+            block = layer.mlp
+            if len(widths) < block.top_k:
+                raise ValueError(
+                    f"a layer keeps {len(widths)} experts < top_k={block.top_k}"
+                )
+            block.num_experts = len(widths)
+            block.gate = nn.Linear(config.hidden_size, len(widths), bias=False)
+            block.experts = nn.ModuleList(
+                RaggedOlmoeMLP(config.hidden_size, w, config.hidden_act)
+                for w in widths
+            )

healed/opd_warm_unleashed/step0500/special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "|||IP_ADDRESS|||",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "|||IP_ADDRESS|||",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

healed/opd_warm_unleashed/step0500/tokenizer_config.json

@@ -0,0 +1,247 @@
+{
+  "add_bos_token": false,
+  "add_eos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<|padding|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50254": {
+      "content": "                        ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50255": {
+      "content": "                       ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50256": {
+      "content": "                      ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50257": {
+      "content": "                     ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50258": {
+      "content": "                    ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50259": {
+      "content": "                   ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50260": {
+      "content": "                  ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50261": {
+      "content": "                 ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50262": {
+      "content": "                ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50263": {
+      "content": "               ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50264": {
+      "content": "              ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50265": {
+      "content": "             ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50266": {
+      "content": "            ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50267": {
+      "content": "           ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50268": {
+      "content": "          ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50269": {
+      "content": "         ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50270": {
+      "content": "        ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50271": {
+      "content": "       ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50272": {
+      "content": "      ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50273": {
+      "content": "     ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50274": {
+      "content": "    ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50275": {
+      "content": "   ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50276": {
+      "content": "  ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50277": {
+      "content": "|||EMAIL_ADDRESS|||",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50278": {
+      "content": "|||PHONE_NUMBER|||",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50279": {
+      "content": "|||IP_ADDRESS|||",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50280": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "|||IP_ADDRESS|||",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "|||IP_ADDRESS|||",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "tokenizer_class": "GPTNeoXTokenizer",
+  "unk_token": null
+}

healed/opd_warm_unleashed/wandb/debug-internal.log

@@ -0,0 +1,16 @@
+{"time":"2026-08-02T20:32:18.760787811-07:00","level":"INFO","msg":"wandb-core"}
+{"time":"2026-08-02T20:32:18.760821712-07:00","level":"INFO","msg":"stream: starting","core version":"0.28.0"}
+{"time":"2026-08-02T20:32:18.874378042-07:00","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-08-02T20:32:18.874396933-07:00","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-08-02T20:32:18.874419894-07:00","level":"INFO","msg":"stream: created new stream","id":"j33a2nw2"}
+{"time":"2026-08-02T20:32:18.874538027-07:00","level":"INFO","msg":"stream: started"}
+{"time":"2026-08-02T20:32:18.874577258-07:00","level":"INFO","msg":"writer: started","stream_id":"j33a2nw2"}
+{"time":"2026-08-02T20:32:18.87461865-07:00","level":"INFO","msg":"sender: started"}
+{"time":"2026-08-02T20:32:18.874662091-07:00","level":"INFO","msg":"handler: started"}
+{"time":"2026-08-02T20:32:18.891498801-07:00","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-08-02T20:32:18.891525461-07:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"}
+{"time":"2026-08-03T03:53:25.789200783-07:00","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2026-08-03T03:53:25.792541472-07:00","level":"INFO","msg":"stream: finishing up"}
+{"time":"2026-08-03T03:53:25.792552172-07:00","level":"INFO","msg":"handler: closed"}
+{"time":"2026-08-03T03:53:25.792600184-07:00","level":"INFO","msg":"sender: closed"}
+{"time":"2026-08-03T03:53:25.792607014-07:00","level":"INFO","msg":"stream: all finished"}