Docs

.gitattributes

@@ -0,0 +1,68 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+LF/data/msrh_rag_test_k3_AfriE5_TV_fewshot_k3.json filter=lfs diff=lfs merge=lfs -text
+LF/data/msrh_rag_test_k3_AfriE5_TV_fewshot_k4.json filter=lfs diff=lfs merge=lfs -text
+LF/data/msrh_rag_test_k3_AfriE5_TV_fewshot_k5.json filter=lfs diff=lfs merge=lfs -text
+LF/data/msrh_rag_test_k3_AfriE5_TV_fewshot_k5_v8.json filter=lfs diff=lfs merge=lfs -text
+LF/data/msrh_rag_test_k3_AfriE5_TV_fewshot_k7.json filter=lfs diff=lfs merge=lfs -text
+LF/data/msrh_rag_test_k3_AfriE5_TV_fewshot_k7_v8.json filter=lfs diff=lfs merge=lfs -text
+LF/data/msrh_rag_train_afrie5_TV_k3.json filter=lfs diff=lfs merge=lfs -text
+LF/data/msrh_rag_train_afrie5_TV_k3_fewshot.json filter=lfs diff=lfs merge=lfs -text
+LF/data/msrh_rag_train_afrie5_TV_k4_fewshot.json filter=lfs diff=lfs merge=lfs -text
+LF/data/msrh_rag_train_afrie5_TV_k5_fewshot.json filter=lfs diff=lfs merge=lfs -text
+LF/data/msrh_rag_train_afrie5_TV_k5_fewshot_v8.json filter=lfs diff=lfs merge=lfs -text
+LF/data/msrh_rag_train_afrie5_TV_k7_fewshot.json filter=lfs diff=lfs merge=lfs -text
+LF/data/msrh_rag_train_afrie5_TV_k7_fewshot_v8.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/Qwen3-32B-3fewshots-bs64-3eps-ckpt-1400/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/Qwen3-32B-5fewshots-bs64-3eps-ckpt-1700/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/Qwen3-32B-7fewshots-bs64-3eps-ckpt-1200/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/Qwen3-32B-7fewshots-bs64-3eps-ckpt-1600/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/Qwen3-32B-NoFewshots-bs64-4eps-ckpt-6500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/Qwen3.5-27B-3fewshots-bs64-3eps-ckpt-1100/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/Qwen3.5-27B-3fewshots-bs64-3eps-ckpt-1200/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/Qwen3.5-27B-3fewshots-bs64-5eps-ckpt-1200/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/Qwen3.5-27B-4fewshots-bs64-3eps-ckpt-1600/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/Qwen3.5-27B-5fewshots-bs64-3eps-v8prompt-ckpt-1500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/Qwen3.5-27B-7fewshots-bs64-3eps-ckpt-1200/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/Qwen3.5-27B-7fewshots-bs64-3eps-ckpt-1600/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/Qwen3.5-27B-NoFewshots-bs64-5eps-ckpt-2800/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/Qwen3.6-27B-3fewshots-bs64-3eps-ckpt-1600/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/Qwen3.6-27B-4fewshots-bs64-3eps-ckpt-1400/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/Qwen3.6-27B-5fewshots-bs64-3eps-ckpt-1000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/Qwen3.6-27B-5fewshots-bs64-3eps-ckpt-1200/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/Qwen3.6-27B-7fewshots-bs64-3eps-ckpt-1600/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/Qwen3.6-27B-NoFewshots-bs64-5eps-ckpt-2600/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+data/Train.csv filter=lfs diff=lfs merge=lfs -text

LF/data/dataset_info.json

@@ -0,0 +1,79 @@
+{
+  "msrh_rag_train_afrie5_TV_k3": {
+    "file_name": "msrh_rag_train_afrie5_TV_k3.json",
+    "formatting": "sharegpt",
+    "columns": {"messages": "messages"},
+    "tags": {
+      "role_tag": "role",
+      "content_tag": "content",
+      "user_tag": "user",
+      "assistant_tag": "assistant"
+    }
+  },
+  "msrh_rag_train_afrie5_TV_k3_fewshot": {
+    "file_name": "msrh_rag_train_afrie5_TV_k3_fewshot.json",
+    "formatting": "sharegpt",
+    "columns": {"messages": "messages"},
+    "tags": {
+      "role_tag": "role",
+      "content_tag": "content",
+      "user_tag": "user",
+      "assistant_tag": "assistant"
+    }
+  },
+  "msrh_rag_train_afrie5_TV_k4_fewshot": {
+    "file_name": "msrh_rag_train_afrie5_TV_k4_fewshot.json",
+    "formatting": "sharegpt",
+    "columns": {"messages": "messages"},
+    "tags": {
+      "role_tag": "role",
+      "content_tag": "content",
+      "user_tag": "user",
+      "assistant_tag": "assistant"
+    }
+  },
+  "msrh_rag_train_afrie5_TV_k5_fewshot": {
+    "file_name": "msrh_rag_train_afrie5_TV_k5_fewshot.json",
+    "formatting": "sharegpt",
+    "columns": {"messages": "messages"},
+    "tags": {
+      "role_tag": "role",
+      "content_tag": "content",
+      "user_tag": "user",
+      "assistant_tag": "assistant"
+    }
+  },
+  "msrh_rag_train_afrie5_TV_k5_fewshot_v8": {
+    "file_name": "msrh_rag_train_afrie5_TV_k5_fewshot_v8.json",
+    "formatting": "sharegpt",
+    "columns": {"messages": "messages"},
+    "tags": {
+      "role_tag": "role",
+      "content_tag": "content",
+      "user_tag": "user",
+      "assistant_tag": "assistant"
+    }
+  },
+  "msrh_rag_train_afrie5_TV_k7_fewshot": {
+    "file_name": "msrh_rag_train_afrie5_TV_k7_fewshot.json",
+    "formatting": "sharegpt",
+    "columns": {"messages": "messages"},
+    "tags": {
+      "role_tag": "role",
+      "content_tag": "content",
+      "user_tag": "user",
+      "assistant_tag": "assistant"
+    }
+  },
+  "msrh_rag_train_afrie5_TV_k7_fewshot_v8": {
+    "file_name": "msrh_rag_train_afrie5_TV_k7_fewshot_v8.json",
+    "formatting": "sharegpt",
+    "columns": {"messages": "messages"},
+    "tags": {
+      "role_tag": "role",
+      "content_tag": "content",
+      "user_tag": "user",
+      "assistant_tag": "assistant"
+    }
+  }
+}

LF/data/msrh_rag_test_k3_AfriE5_TV_fewshot_k3.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e177d64be6a16549044f75038bd13dac09d06620231b55f229c022e6dd90d92e
+size 11500585

LF/data/msrh_rag_test_k3_AfriE5_TV_fewshot_k4.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09df55629206738d16184cc580559a533d1087e95487b08e722f9ca2f2537583
+size 13138184

LF/data/msrh_rag_test_k3_AfriE5_TV_fewshot_k5.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1081a9cda64f33486f402acfd2c84faeb93b4288a31cb2b9ee5a2ab43b19966a
+size 15122201

LF/data/msrh_rag_test_k3_AfriE5_TV_fewshot_k5_v8.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b53f4dec86137c08e905dce9ef4fe99bcf26216a68934c7ac774eb4561ccb08
+size 16901122

LF/data/msrh_rag_test_k3_AfriE5_TV_fewshot_k7.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47801c1b13b99abb097e585bf72c48bc02bb568a6db41d23a571b6d5d17bd747
+size 18531279

LF/data/msrh_rag_test_k3_AfriE5_TV_fewshot_k7_v8.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b124333365bc3c846e55c99ee4d231f040ee5922b5d4f6a7944b1a5b738d2ed3
+size 18654325

LF/data/msrh_rag_train_afrie5_TV_k3.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79e763337b84bb443e209dcf80fdba5373ef110068cf9c8b5cb9410db334885f
+size 93056027

LF/data/msrh_rag_train_afrie5_TV_k3_fewshot.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a584075e87fc617f134011ce44488dcbf9f6b8d6670a476ac8abe2af9786ae30
+size 164071164

LF/data/msrh_rag_train_afrie5_TV_k4_fewshot.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d42d07f88be9a978553d48087fa85b3b4e45b40e1926c8ea303f1b8273937913
+size 186613345

LF/data/msrh_rag_train_afrie5_TV_k5_fewshot.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3dce22556de5dea051c6f5a0c8fdf248870c9f941491be6d6f4cff96eb97c510
+size 210165221

LF/data/msrh_rag_train_afrie5_TV_k5_fewshot_v8.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bcc06431b5eba56ffd75f2f66456a6664f14546bedf7d76e716d1d1dada6a13c
+size 211880768

LF/data/msrh_rag_train_afrie5_TV_k7_fewshot.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e8534b7c468daa3031fc38e9aee3c1cf49f2b64e41418501511c0480ac7a8aa0
+size 255194569

LF/data/msrh_rag_train_afrie5_TV_k7_fewshot_v8.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d8be1de59a6093c2193753ee8ec05a7f53e19f40de9c46ec0ed88dfbac9424a5
+size 256910116

MODEL_CARD.md

@@ -0,0 +1,165 @@
+---
+license: apache-2.0
+tags:
+  - medical-qa
+  - multilingual
+  - low-resource-african
+  - lora
+  - peft
+  - rag
+  - zindi
+  - msrh
+library_name: peft
+pipeline_tag: text-generation
+base_model:
+  - Qwen/Qwen3.5-27B
+  - Qwen/Qwen3.6-27B
+  - Qwen/Qwen3-32B
+---
+
+# Magic — Zindi MSRH Multilingual Health Q&A (Top-1 Solution)
+
+A LoRA-adapter ensemble that scored **private LB = 0.730865** on the Zindi
+[Multilingual Sexual & Reproductive Health Q&A](https://zindi.africa/competitions/multilingual-health-question-answering-in-low-resource-african-languages-challenge)
+challenge. Ships 19 LoRA adapters over 3 Qwen base models plus a per-row
+consensus ensemble builder that regenerates the submitted `go.csv`.
+
+## Model architecture
+
+- **Base models** (3 backbones): `Qwen3.5-27B`, `Qwen3.6-27B`, `Qwen3-32B`.
+- **Adapter type**: LoRA (`peft`), rank `r=128`, `alpha=256`, dropout `0.05`,
+  `target_modules=all` (7 modules: q/k/v/o/gate/up/down proj).
+- **Training**: LlamaFactory + DeepSpeed ZeRO-3, `bf16`, `AdamW`, `lr=2e-4`,
+  cosine schedule with `warmup_ratio=0.05`, 3 epochs, effective batch `=64`
+  (`per_device=2 × grad_accum=4 × 8 GPUs`).
+- **Retrieval**: `McGill-NLP/AfriE5-Large-instruct` top-3 passages (Train+Val
+  as candidate pool, per-subset filtering, self-mask on training queries).
+- **Few-shot demos**: K ∈ {3, 4, 5, 7} same-subset AfriE5-nearest (Q, A)
+  pairs prepended to each prompt.
+- **Prompt variants**: v1 baseline + v8 anchored-extraction (shortened
+  copy-verbatim instruction).
+- **Ensemble**: per-row consensus pick across the 19 adapter predictions.
+- **Private LB**: `0.730865` on the private test set (see `go.csv`).
+
+## Intended use
+
+Answering **maternal, sexual, and reproductive health** questions posed in
+**English** and in four low-resource African languages: **Akan (`Aka_Gha`)**,
+**Amharic (`Amh_Eth`)**, **Luganda (`Lug_Uga`)** and **Swahili (`Swa_Ken`)** —
+together the 8 language×country subsets defined by the competition.
+
+Primary intended users are:
+
+- Research on retrieval-augmented multilingual medical Q&A.
+- Reviewers reproducing the leaderboard result.
+
+Out-of-scope: clinical decision-making, diagnosis, or any use case where
+factual correctness for a specific patient matters. The model has NOT been
+audited for medical safety.
+
+## Dependencies
+
+Pinned versions and install instructions are in `requirements/infer.txt`
+(inference) and `requirements/train.txt` (training).
+
+Hardware: reproduce on any 80GB GPU (H100 / A100). The launcher
+auto-detects visible GPUs and runs up to `min(8, visible)` predicts
+concurrently, so **1 GPU works** (sequential, ~20-30h wall-clock) and
+**8 GPUs is the sweet spot** (~2h wall-clock). No config changes needed.
+
+## Inference / reproduction (one command)
+
+```bash
+bash scripts/run_all.sh
+```
+
+This driver runs the full end-to-end recipe:
+
+1. Loads each of the 19 LoRA adapters onto its base model via vLLM.
+2. Generates predictions on the shipped test JSONLs (`LF/data/`).
+3. Converts each `generated_predictions.jsonl` to a Zindi-format CSV.
+4. Runs `scripts/build_ensemble.py` over the 19 CSVs to regenerate
+   `submission.csv`.
+5. MD5-verifies the regenerated CSV against the shipped `go.csv`.
+
+Step-by-step (if you want to run individually):
+
+```bash
+# 1. Generate 19 per-adapter predictions (writes to predict_out/)
+bash scripts/launch_all_predicts.sh
+
+# 2. (JSONL → CSV conversion runs inline inside run_all.sh; no separate script)
+
+# 3. Ensemble → final CSV (writes submission.csv + md5 check)
+python scripts/build_ensemble.py
+```
+
+Full detail (env setup, LlamaFactory installation, retraining from scratch)
+is in `README.md`.
+
+## Known caveats & setup notes
+
+Before running `scripts/run_all.sh`, be aware of the following (from an
+end-to-end audit of a fresh clone from this repo):
+
+1. **Base models are NOT included** (license reasons). Reviewers must
+   download the three Qwen backbones separately from Hugging Face and
+   place them under `hub/`:
+
+   | Base model | HF link | Local path |
+   |---|---|---|
+   | Qwen3.5-27B | https://huggingface.co/Qwen/Qwen3.5-27B | `hub/Qwen3.5-27B/` |
+   | Qwen3.6-27B | https://huggingface.co/Qwen/Qwen3.6-27B | `hub/Qwen3.6-27B/` |
+   | Qwen3-32B   | https://huggingface.co/Qwen/Qwen3-32B   | `hub/Qwen3-32B/`   |
+
+   Example download:
+   ```bash
+   hf download Qwen/Qwen3.5-27B --local-dir hub/Qwen3.5-27B
+   hf download Qwen/Qwen3.6-27B --local-dir hub/Qwen3.6-27B
+   hf download Qwen/Qwen3-32B   --local-dir hub/Qwen3-32B
+   ```
+   If a repo ID 404s on your side, use a compatible mirror (e.g. an
+   `unsloth/` upload of the same weights).
+
+2. **`base_model_name_or_path` in every `adapter_config.json` points at
+   `/mnt/msrh/Magic_submission/hub/<base>`** — this is a submission-time
+   fake path. Two options:
+   - Extract this repo into `/mnt/msrh/Magic_submission/` (may need `sudo
+     mkdir /mnt/msrh` first) and populate `hub/` there — no code changes.
+   - Or edit `base_model_name_or_path` in each adapter config to point at
+     your local snapshot / HF repo ID.
+
+3. **`scripts/launch_all_predicts.sh` auto-locates its workspace root**
+   from the script path (default: parent dir of `scripts/`). If you want
+   to point at a different location, override the env var:
+   ```bash
+   ROOT=/my/extract/path bash scripts/launch_all_predicts.sh
+   ```
+
+4. **First-run vLLM warm-up is slow** — the FlashInfer GDN prefill kernel
+   is JIT-compiled on the first launch (~1 min extra per GPU). vLLM also
+   suggests `--gdn-prefill-backend triton` as an alternative if you want to
+   skip JIT; not required for correctness.
+
+5. **Regenerated `submission.csv` matches `go.csv` byte-for-byte only on
+   identical hardware / kernel / vLLM state.** vLLM inference is not
+   deterministic across hardware, driver versions, or torch.compile /
+   FlashInfer cache states. On a fresh environment, expect ~60-70% of rows
+   to match `go.csv` byte-for-byte; the remaining rows will be
+   paraphrases of the same underlying answer. **Functional LB equivalence
+   (ROUGE metrics) is what actually matters for evaluation.**
+
+## Citation
+
+If you use this work, please cite the Zindi competition:
+
+```
+Zindi Africa. "Multilingual Health Question Answering in Low-Resource
+African Languages Challenge", 2026. https://zindi.africa/competitions/
+multilingual-health-question-answering-in-low-resource-african-languages-challenge
+```
+
+## License
+
+Apache-2.0 for the adapter weights and code in this repository. The base
+Qwen models carry their own licenses (see the corresponding HF repos).

README.md

@@ -0,0 +1,165 @@
+---
+license: apache-2.0
+tags:
+  - medical-qa
+  - multilingual
+  - low-resource-african
+  - lora
+  - peft
+  - rag
+  - zindi
+  - msrh
+library_name: peft
+pipeline_tag: text-generation
+base_model:
+  - Qwen/Qwen3.5-27B
+  - Qwen/Qwen3.6-27B
+  - Qwen/Qwen3-32B
+---
+
+# Magic — Zindi MSRH Multilingual Health Q&A (Top-1 Solution)
+
+A LoRA-adapter ensemble that scored **private LB = 0.730865** on the Zindi
+[Multilingual Sexual & Reproductive Health Q&A](https://zindi.africa/competitions/multilingual-health-question-answering-in-low-resource-african-languages-challenge)
+challenge. Ships 19 LoRA adapters over 3 Qwen base models plus a per-row
+consensus ensemble builder that regenerates the submitted `go.csv`.
+
+## Model architecture
+
+- **Base models** (3 backbones): `Qwen3.5-27B`, `Qwen3.6-27B`, `Qwen3-32B`.
+- **Adapter type**: LoRA (`peft`), rank `r=128`, `alpha=256`, dropout `0.05`,
+  `target_modules=all` (7 modules: q/k/v/o/gate/up/down proj).
+- **Training**: LlamaFactory + DeepSpeed ZeRO-3, `bf16`, `AdamW`, `lr=2e-4`,
+  cosine schedule with `warmup_ratio=0.05`, 3 epochs, effective batch `=64`
+  (`per_device=2 × grad_accum=4 × 8 GPUs`).
+- **Retrieval**: `McGill-NLP/AfriE5-Large-instruct` top-3 passages (Train+Val
+  as candidate pool, per-subset filtering, self-mask on training queries).
+- **Few-shot demos**: K ∈ {3, 4, 5, 7} same-subset AfriE5-nearest (Q, A)
+  pairs prepended to each prompt.
+- **Prompt variants**: v1 baseline + v8 anchored-extraction (shortened
+  copy-verbatim instruction).
+- **Ensemble**: per-row consensus pick across the 19 adapter predictions.
+- **Private LB**: `0.730865` on the private test set (see `go.csv`).
+
+## Intended use
+
+Answering **maternal, sexual, and reproductive health** questions posed in
+**English** and in four low-resource African languages: **Akan (`Aka_Gha`)**,
+**Amharic (`Amh_Eth`)**, **Luganda (`Lug_Uga`)** and **Swahili (`Swa_Ken`)** —
+together the 8 language×country subsets defined by the competition.
+
+Primary intended users are:
+
+- Research on retrieval-augmented multilingual medical Q&A.
+- Reviewers reproducing the leaderboard result.
+
+Out-of-scope: clinical decision-making, diagnosis, or any use case where
+factual correctness for a specific patient matters. The model has NOT been
+audited for medical safety.
+
+## Dependencies
+
+Pinned versions and install instructions are in `requirements/infer.txt`
+(inference) and `requirements/train.txt` (training).
+
+Hardware: reproduce on any 80GB GPU (H100 / A100). The launcher
+auto-detects visible GPUs and runs up to `min(8, visible)` predicts
+concurrently, so **1 GPU works** (sequential, ~20-30h wall-clock) and
+**8 GPUs is the sweet spot** (~2h wall-clock). No config changes needed.
+
+## Inference / reproduction (one command)
+
+```bash
+bash scripts/run_all.sh
+```
+
+This driver runs the full end-to-end recipe:
+
+1. Loads each of the 19 LoRA adapters onto its base model via vLLM.
+2. Generates predictions on the shipped test JSONLs (`LF/data/`).
+3. Converts each `generated_predictions.jsonl` to a Zindi-format CSV.
+4. Runs `scripts/build_ensemble.py` over the 19 CSVs to regenerate
+   `submission.csv`.
+5. MD5-verifies the regenerated CSV against the shipped `go.csv`.
+
+Step-by-step (if you want to run individually):
+
+```bash
+# 1. Generate 19 per-adapter predictions (writes to predict_out/)
+bash scripts/launch_all_predicts.sh
+
+# 2. (JSONL → CSV conversion runs inline inside run_all.sh; no separate script)
+
+# 3. Ensemble → final CSV (writes submission.csv + md5 check)
+python scripts/build_ensemble.py
+```
+
+Full detail (env setup, LlamaFactory installation, retraining from scratch)
+is in `README.md`.
+
+## Known caveats & setup notes
+
+Before running `scripts/run_all.sh`, be aware of the following (from an
+end-to-end audit of a fresh clone from this repo):
+
+1. **Base models are NOT included** (license reasons). Reviewers must
+   download the three Qwen backbones separately from Hugging Face and
+   place them under `hub/`:
+
+   | Base model | HF link | Local path |
+   |---|---|---|
+   | Qwen3.5-27B | https://huggingface.co/Qwen/Qwen3.5-27B | `hub/Qwen3.5-27B/` |
+   | Qwen3.6-27B | https://huggingface.co/Qwen/Qwen3.6-27B | `hub/Qwen3.6-27B/` |
+   | Qwen3-32B   | https://huggingface.co/Qwen/Qwen3-32B   | `hub/Qwen3-32B/`   |
+
+   Example download:
+   ```bash
+   hf download Qwen/Qwen3.5-27B --local-dir hub/Qwen3.5-27B
+   hf download Qwen/Qwen3.6-27B --local-dir hub/Qwen3.6-27B
+   hf download Qwen/Qwen3-32B   --local-dir hub/Qwen3-32B
+   ```
+   If a repo ID 404s on your side, use a compatible mirror (e.g. an
+   `unsloth/` upload of the same weights).
+
+2. **`base_model_name_or_path` in every `adapter_config.json` points at
+   `/mnt/msrh/Magic_submission/hub/<base>`** — this is a submission-time
+   fake path. Two options:
+   - Extract this repo into `/mnt/msrh/Magic_submission/` (may need `sudo
+     mkdir /mnt/msrh` first) and populate `hub/` there — no code changes.
+   - Or edit `base_model_name_or_path` in each adapter config to point at
+     your local snapshot / HF repo ID.
+
+3. **`scripts/launch_all_predicts.sh` auto-locates its workspace root**
+   from the script path (default: parent dir of `scripts/`). If you want
+   to point at a different location, override the env var:
+   ```bash
+   ROOT=/my/extract/path bash scripts/launch_all_predicts.sh
+   ```
+
+4. **First-run vLLM warm-up is slow** — the FlashInfer GDN prefill kernel
+   is JIT-compiled on the first launch (~1 min extra per GPU). vLLM also
+   suggests `--gdn-prefill-backend triton` as an alternative if you want to
+   skip JIT; not required for correctness.
+
+5. **Regenerated `submission.csv` matches `go.csv` byte-for-byte only on
+   identical hardware / kernel / vLLM state.** vLLM inference is not
+   deterministic across hardware, driver versions, or torch.compile /
+   FlashInfer cache states. On a fresh environment, expect ~60-70% of rows
+   to match `go.csv` byte-for-byte; the remaining rows will be
+   paraphrases of the same underlying answer. **Functional LB equivalence
+   (ROUGE metrics) is what actually matters for evaluation.**
+
+## Citation
+
+If you use this work, please cite the Zindi competition:
+
+```
+Zindi Africa. "Multilingual Health Question Answering in Low-Resource
+African Languages Challenge", 2026. https://zindi.africa/competitions/
+multilingual-health-question-answering-in-low-resource-african-languages-challenge
+```
+
+## License
+
+Apache-2.0 for the adapter weights and code in this repository. The base
+Qwen models carry their own licenses (see the corresponding HF repos).

RECIPE.md

@@ -0,0 +1,457 @@
+# Top 1 Private Solution - Magic Team
+**Multilingual Health QA in Low-Resource African Languages Challenge**
+
+Submitter: **Magic**
+Best submission: **`go.csv`** (Zindi ID `vtbP7bCH`, submitted 21 Jun 17:12)
+Public LB: **0.738783** · Private LB: **Top 1 🏆**
+
+---
+
+## 1. Solution overview
+
+
+**19-candidate ensemble** built with **V2 medoid_ngram** consensus:
+- Base: 12 fewshot LoRA adapters across Qwen3.5/Qwen3.6/Qwen3-32B × K=3,4,5,7
+- v8 prompt swap: 1 Qwen3.5-27B K=5 LoRA trained with anchored-extraction prompt
+- 3 cross-arch "less-overfit" checkpoints
+- 3 no-fewshot mediators
+
+Per Test row, the medoid (highest sum of pairwise ROUGE-1.F + ROUGE-2.F) is chosen as the final answer.
+
+### Why it wins
+Six independent decorrelation axes simultaneously:
+1. **Architecture**: Qwen3.5-27B + Qwen3.6-27B + Qwen3-32B (3 base models)
+2. **K-count**: K=3, K=4, K=5, K=7 fewshot demos
+3. **Prompt recipe**: v1 baseline vs v8 anchored-extraction
+4. **Training schedule**: 3ep vs 5ep
+5. **Mediator**: 3 no-fewshot baselines
+
+---
+
+## 2. Workspace layout
+
+**All paths in this package are hard-coded to `/mnt/msrh/Magic_submission/`.** Extract this archive to that exact location (or mount/symlink it there) so every script — data builders, training YAMLs, inference launcher, ensemble — works without editing.
+
+```bash
+# One-time setup — choose ONE of these:
+
+# (a) Extract the archive to /mnt/msrh/
+sudo mkdir -p /mnt/msrh && sudo chown $USER /mnt/msrh
+tar -xf Magic_submission.tar -C /mnt/msrh/        # extracts to /mnt/msrh/Magic_submission/
+
+# (b) OR symlink an existing copy
+ln -s /path/to/existing/Magic_submission /mnt/msrh/Magic_submission
+
+# Verify
+ls /mnt/msrh/Magic_submission/{go.csv,checkpoints,configs,scripts,data_builders}
+```
+
+After extraction, you must also place under `/mnt/msrh/Magic_submission/`:
+- `data/Train.csv`, `data/Val.csv`, `data/Test.csv`, `data/SampleSubmission.csv` (Zindi competition data — download separately). Or you can manually download it from the Zindi data source.
+- `hub/Qwen3.5-27B/`, `hub/Qwen3.6-27B/`, `hub/Qwen3-32B/` (HF snapshot of each base model)
+- `hub/AfriE5-Large-instruct/` (HF snapshot of `McGill-NLP/AfriE5-Large-instruct`)
+
+```
+/mnt/msrh/Magic_submission/
+├── go.csv                            # Final submission CSV (== Zindi sub vtbP7bCH)
+├── README.md                         # This file — full reproduction guide
+├── environment.md                    # All env setup (conda, packages, models)
+├── checkpoints/                      # ⭐ 19 LoRA adapters (35 GB) — INPUT to inference
+│   # Each folder = 1 LoRA adapter with:
+│   #   adapter_model.safetensors  (1.8 GB for 27B, 2.1 GB for 32B)
+│   #   adapter_config.json
+│   #   chat_template.jinja
+│   #   tokenizer_config.json + tokenizer.json + processor_config.json
+│   ├── Qwen3.5-27B-3fewshots-bs64-3eps-ckpt-1200/
+│   ├── Qwen3.5-27B-3fewshots-bs64-3eps-ckpt-1100/
+│   ├── Qwen3.5-27B-3fewshots-bs64-5eps-ckpt-1200/
+│   ├── Qwen3.5-27B-4fewshots-bs64-3eps-ckpt-1600/
+│   ├── Qwen3.5-27B-5fewshots-bs64-3eps-v8prompt-ckpt-1500/    
+│   ├── Qwen3.5-27B-7fewshots-bs64-3eps-ckpt-1600/
+│   ├── Qwen3.5-27B-7fewshots-bs64-3eps-ckpt-1200/             
+│   ├── Qwen3.6-27B-3fewshots-bs64-3eps-ckpt-1600/
+│   ├── Qwen3.6-27B-4fewshots-bs64-3eps-ckpt-1400/
+│   ├── Qwen3.6-27B-5fewshots-bs64-3eps-ckpt-1200/
+│   ├── Qwen3.6-27B-5fewshots-bs64-3eps-ckpt-1000/             
+│   ├── Qwen3.6-27B-7fewshots-bs64-3eps-ckpt-1600/
+│   ├── Qwen3-32B-3fewshots-bs64-3eps-ckpt-1400/
+│   ├── Qwen3-32B-5fewshots-bs64-3eps-ckpt-1700/
+│   ├── Qwen3-32B-7fewshots-bs64-3eps-ckpt-1600/
+│   ├── Qwen3-32B-7fewshots-bs64-3eps-ckpt-1200/              
+│   ├── Qwen3.5-27B-NoFewshots-bs64-5eps-ckpt-2800/            
+│   ├── Qwen3.6-27B-NoFewshots-bs64-5eps-ckpt-2600/            
+│   └── Qwen3-32B-NoFewshots-bs64-4eps-ckpt-6500/              
+├── candidate_csvs/                   # OUTPUT folder for 19 per-model prediction CSVs
+│   # NOTE: any CSVs here are reference/audit-trail copies.
+│   # For verification, REGENERATE all 19 from checkpoints/ via launch_all_predicts.sh.
+│   # The ensemble script (build_ensemble.py) reads CSVs from this folder.
+├── LF/
+│   └── data/
+│       └── dataset_info.json         
+├── requirements/                     # per-phase pip pinned deps
+│   ├── train.txt                     #   Phase 1 (data builders) + Phase 2 (SFT)
+│   ├── infer.txt                     #   Phase 3 (vLLM inference)
+│   └── ensemble.txt                  #   Phase 4 (CPU medoid ensemble)
+├── data_builders/                    
+│   ├── build_afrie5_k5.py            
+│   ├── build_fewshot_train.py        
+│   ├── build_fewshot_train_k4.py
+│   ├── build_fewshot_train_k5.py
+│   ├── build_fewshot_train_k7.py
+│   ├── build_fewshot_test.py         
+│   ├── build_fewshot_test_k4.py
+│   ├── build_fewshot_test_k5.py
+│   └── build_fewshot_test_k7.py
+├── configs/                          
+│   # ── Fewshot SFT runs (13) ──
+│   ├── qwen35_27b_fewshot_3ep_8gpu.yaml             
+│   ├── qwen35_27b_fewshot_k4_3ep_8gpu.yaml          
+│   ├── qwen35_27b_fewshot_k5_v8_3ep_8gpu.yaml       
+│   ├── qwen35_27b_fewshot_k7_3ep_8gpu.yaml         
+│   ├── qwen35_27b_fewshot_2ep_8gpu.yaml             
+│   ├── qwen36_27b_fewshot_k4_3ep_8gpu.yaml          
+│   ├── qwen36_27b_fewshot_k5_3ep_8gpu.yaml          
+│   ├── qwen36_27b_fewshot_k7_v8_3ep_8gpu.yaml       # Q3.6 K=7 v8 prompt 3ep  → ck-1600
+│   ├── qwen3_32b_fewshot_3ep_bs64_r128_all_8gpu.yaml      
+│   ├── qwen3_32b_fewshot_3ep_r128_all_8gpu.yaml           
+│   ├── qwen3_32b_fewshot_k5_3ep_bs64_r128_all_8gpu.yaml   
+│   ├── qwen3_32b_fewshot_k7_3ep_bs64_r128_all_8gpu.yaml   
+│   # ── NoFewshots (mediator) SFT runs (3) ──
+│   ├── qwen35_27b_nofewshots_5ep_bs64_8gpu.yaml     
+│   ├── qwen36_27b_nofewshots_5ep_bs64_8gpu.yaml    
+│   ├── qwen3_32b_nofewshots_4ep_bs64_8gpu.yaml      
+│   ├── fewshot_e3_Q36.sh                                  
+│   ├── ds_z3_config.json                                  
+├── scripts/                          # Inference + ensemble scripts
+│   ├── build_v8_k5_fewshot.py        # v8 prompt swap (V1 → V8 prefix)
+│   ├── vllm_predict_extra.py         # vLLM batch inference w/ LoRA
+│   ├── jsonl_rowidx_to_csv.py        # jsonl → CSV converter
+│   ├── rag_to_zindi.py               # Alternate CSV builder (Plan D path)
+│   ├── launch_all_predicts.sh        # ⭐ Orchestrator: runs all 19 inferences → candidate_csvs/<descriptive>.csv
+│   ├── build_ensemble.py  # ⭐ FINAL ensemble medoid (uses candidate_csvs/)
+│   └── ds_z3_config.json
+└── docs/
+    └── (additional documentation if needed)
+```
+
+---
+
+## 3. Verification flow (RECOMMENDED for reviewers — uses provided checkpoints)
+
+The 19 LoRA adapters in `checkpoints/` are the WINNING trained weights — the official artifacts that produced `go.csv`. **Reviewers do NOT need to retrain.** Use them directly to regenerate the 19 prediction CSVs, then run the ensemble.
+
+All commands below run from `/mnt/msrh/Magic_submission/` (paths are hard-coded — no editing required).
+
+```bash
+cd /mnt/msrh/Magic_submission
+
+# Step 1: Setup envs (see environment.md for full conda setup)
+conda activate llama-qa35 && pip install -r requirements/train.txt    # Phase 1 + 2
+conda activate vllm       && pip install -r requirements/infer.txt    # Phase 3
+conda activate rouge      && pip install -r requirements/ensemble.txt # Phase 4
+
+# Step 2: Build TEST data (AfriE5 retrieval + fewshot demos) — train data not needed for verification
+conda activate llama-qa35
+python data_builders/build_afrie5_k5.py --k 3       # K=3 base retrieval (NoFewshots test JSON)
+python data_builders/build_fewshot_test.py          # K=3 fewshot test demos
+python data_builders/build_fewshot_test_k4.py       # K=4 fewshot test demos
+python data_builders/build_fewshot_test_k5.py       # K=5 fewshot test demos
+python data_builders/build_fewshot_test_k7.py       # K=7 fewshot test demos
+# v8 variants — REQUIRE K=5 / K=7 fewshot files to already exist:
+python scripts/build_v8_k5_fewshot.py               # rebuilds *_k5_v8.json from K=5 fewshot
+python scripts/build_v8_k7_fewshot.py               # rebuilds *_k7_v8.json from K=7 fewshot
+
+# Step 3: Generate 19 prediction CSVs from provided checkpoints (Reads checkpoints/, writes candidate_csvs/)
+conda activate vllm
+bash scripts/launch_all_predicts.sh
+
+# Step 4: Run ensemble (V2 medoid_ngram) → final CSV
+conda activate rouge
+python scripts/build_ensemble.py
+# Writes the regenerated submission CSV alongside the shipped go.csv.
+# Reference md5 of shipped go.csv: a2ecca4a8e1aa01acf9a8b9a1d56ebf2
+```
+
+### Note on byte-identity vs functional reproducibility
+
+We E2E-verified this pipeline on a fresh 8×H100 box (vLLM 0.19.1 + torch 2.10+cu128).
+**Expect the regenerated CSV to NOT be md5-equal to the shipped `go.csv`** — but the LB
+will reproduce within ~0.005 public / ~0.001 private of the winning result.
+
+| Run | Public LB | Private LB | Row match vs `go.csv` |
+|---|---:|---:|---|
+| Original `vtbP7bCH` (shipped `go.csv`) | 0.738783 | 0.730865 | byte-identical (md5 `a2ecca4a...`) |
+| Re-run #1 — `max_num_seqs=32` hard-coded | 0.734035 | 0.729336 | 65.0% |
+| Re-run #2 — per-cand `max_num_seqs` matched | 0.734104 | 0.729553 | 67.8% |
+
+**Root cause of the ~0.005 gap**: vLLM 0.19.1 paged-attention has **inherent non-determinism**
+between runs even at `--temperature 0.0` with identical flags (CUDA atomic ops, bfloat16
+rounding-order in attention, KV-cache block scheduling races). This affects ~30-40% of
+generated tokens at single-cand level; the V2 medoid_ngram ensemble suppresses most of the
+drift, leaving ~32% of per-row picks different but only a ~0.001 private-LB hit.
+
+**Acceptance criterion**: if the regenerated CSV yields **Private LB ≥ 0.728** (within −0.003
+of the shipped 0.730865), reproduction is considered successful and the pipeline is verified.
+md5 byte-equality is NOT achievable on a different machine/run.
+
+---
+
+## 3b. Full from-scratch reproduction (optional — trains 13 LoRAs from base models)
+
+Only needed if you want to retrain the LoRA adapters yourself instead of using the provided ones.
+
+```bash
+cd /mnt/msrh/Magic_submission
+conda activate llama-qa35   # training env
+
+# Build train data (same as Step 2 above but for train set)
+python data_builders/build_afrie5_k5.py --k 3
+python data_builders/build_afrie5_k5.py --k 5
+python data_builders/build_afrie5_k5.py --k 7
+python data_builders/build_fewshot_train.py
+python data_builders/build_fewshot_train_k4.py
+python data_builders/build_fewshot_train_k5.py
+python data_builders/build_fewshot_train_k7.py
+# v8 prompt swap — runs AFTER build_fewshot_train_k5.py and build_fewshot_test_k5.py
+python scripts/build_v8_k5_fewshot.py               # rebuilds *_k5_v8.json (train + test)
+
+# Train each of the 16 LoRA adapters (configs/*.yaml). Outputs go to
+# /mnt/msrh/Magic_submission/checkpoints_trained/<config-name>/ as set inside each YAML.
+# Example for the key v8 model:
+FORCE_TORCHRUN=1 NPROC_PER_NODE=8 \
+  llamafactory-cli train /mnt/msrh/Magic_submission/configs/qwen35_27b_fewshot_k5_v8_3ep_8gpu.yaml
+# ... repeat for all 16 configs (~14h each on 8× H100)
+
+# Replace adapters in checkpoints/<descriptive_name>/ with your fresh ckpts, then
+# follow Step 3 + Step 4 from Section 3 above.
+```
+
+---
+
+## 4. Data preparation (full pipeline)
+
+### Inputs (required)
+- Zindi competition data: `Train.csv`, `Val.csv`, `Test.csv`, `SampleSubmission.csv` (from competition page)
+- Path expected: `/mnt/msrh/Magic_submission/data/{Train,Val,Test,SampleSubmission}.csv` (no editing needed — every builder and script reads from this fixed location)
+
+### Pipeline
+
+1. **AfriE5 base retrieval** (`build_afrie5_k5.py`)
+   - For each train query, retrieve top-K from Train+Val pool using AfriE5 cosine similarity
+   - For each test query, retrieve top-K from Train+Val pool
+   - Output: `msrh_rag_train_afrie5_TV_k{K}.json` + `msrh_rag_test_k3_AfriE5_TV.json`
+   - Includes language-specific instruction tag (Akan/Amharic/Luganda/Swahili/English)
+
+2. **Fewshot demo prep** (`build_fewshot_train_k{K}.py` + `build_fewshot_test_k{K}.py`)
+   - Per row, retrieve K AfriE5-similar same-subset (Q,A) pairs from Train+Val
+   - Prepend as "Example N:" demos in user message
+   - Output: `msrh_rag_train_afrie5_TV_k{K}_fewshot.json`
+
+3. **v8 prompt swap** (`build_v8_k5_fewshot.py`)
+   - Replace v1 prompt:
+     ```
+     Use the retrieved contexts as your primary sources — copy exact phrasing where
+     the contexts already address the question. Be concise and factually accurate.
+     ```
+   - With v8 prompt (33 words, 220 chars):
+     ```
+     The retrieved contexts are your source of truth — copy or paraphrase their
+     exact phrasing to answer. Reply in the same language and script as the question.
+     Plain prose, no disclaimers or meta-commentary.
+     ```
+   - Output: `msrh_rag_train_afrie5_TV_k5_fewshot_v8.json`
+
+---
+
+## 5. Training (13 LoRA adapters)
+
+All training uses **LlamaFactory + DeepSpeed ZeRO-3** on 8× H100 (80GB).
+
+### Common hyperparameters (all 13 runs)
+```yaml
+finetuning_type: lora
+lora_rank: 128
+lora_alpha: 256
+lora_dropout: 0.05
+lora_target: all          # all linear layers
+learning_rate: 2.0e-4
+lr_scheduler_type: cosine
+warmup_ratio: 0.05
+bf16: true
+gradient_checkpointing: true
+deepspeed: scripts/ds_z3_config.json
+```
+
+### Per-model details
+
+### Per-model details (16 SFT runs total — all reproducible via configs/*.yaml)
+
+The 19 ensemble candidates come from **16 distinct SFT training runs** (3 of the runs export 2 checkpoints each as "early-tap" anti-overfit cands).
+
+#### Fewshot runs (13 SFT, each prepends top-K AfriE5-similar (Q,A) demos)
+
+| # | Model | K | Prompt | Epochs | cutoff_len | eff_bs | YAML | Best ckpt | Cand(s) |
+|---:|---|:---:|:---:|:---:|:---:|:---:|---|---|---|
+| 1 | Qwen3.5-27B | 3 | v1 | 3 | 4096 | 64 | qwen35_27b_fewshot_3ep_8gpu.yaml | ck-1200 | ck-1200 + ck-1100 (early-tap) |
+| 2 | Qwen3.5-27B | 3 | v1 | 5 | 4096 | 64 | qwen35_27b_fewshot_5ep_8gpu.yaml (modify epochs=5 in run 1's YAML) | ck-1200 | 1 cand |
+| 3 | Qwen3.5-27B | 4 | v1 | 3 | 5120 | 64 | qwen35_27b_fewshot_k4_3ep_8gpu.yaml | ck-1600 | 1 cand |
+| 4 | **Qwen3.5-27B** | **5** | **v8** | **3** | **6144** | **64** | **qwen35_27b_fewshot_k5_v8_3ep_8gpu.yaml** | **ck-1500** ⭐ | 1 cand ⭐ |
+| 5 | Qwen3.5-27B | 7 | v1 | 3 | 8192 | 64 | qwen35_27b_fewshot_k7_3ep_8gpu.yaml | ck-1600 | ck-1600 + ck-1200 (early-tap) |
+| 6 | Qwen3.6-27B | 3 | v1 (RecA) | 3 | 4096 | 64 | qwen36_27b_fewshot_3eps_bs64.yaml (Q3.5 K=3 YAML w/ model+template swap) | ck-1600 | 1 cand |
+| 7 | Qwen3.6-27B | 4 | v1 | 3 | 5120 | 64 | qwen36_27b_fewshot_k4_3ep_8gpu.yaml | ck-1400 | 1 cand |
+| 8 | Qwen3.6-27B | 5 | v1 | 3 | 6144 | 64 | qwen36_27b_fewshot_k5_3ep_8gpu.yaml | ck-1200 | ck-1200 + ck-1000 (early-tap) |
+| 9 | Qwen3.6-27B | 7 | **v8** | 3 | 8192 | 64 | qwen36_27b_fewshot_k7_v8_3ep_8gpu.yaml | ck-1600 | 1 cand |
+| 10 | Qwen3-32B | 3 | v1 | 3 | 4096 | 64 | qwen3_32b_fewshot_3ep_bs64_r128_all_8gpu.yaml | ck-1400 | 1 cand |
+| 11 | Qwen3-32B | 5 | v1 | 3 | 6144 | 64 | qwen3_32b_fewshot_k5_3ep_bs64_r128_all_8gpu.yaml | ck-1700 | 1 cand |
+| 12 | Qwen3-32B | 7 | v1 | 3 | 8192 | 64 | qwen3_32b_fewshot_k7_3ep_bs64_r128_all_8gpu.yaml | ck-1600 | ck-1600 + ck-1200 (early-tap) |
+
+#### NoFewshots runs (3 SFT, mediators — same K=3 AfriE5 retrieval but WITHOUT demos prepended)
+
+These 3 are **trained the same way** as the fewshot runs but use the **non-fewshot K=3 dataset** (`msrh_rag_train_afrie5_TV_k3.json` — output of `build_afrie5_k5.py --k 3` only, no `build_fewshot_train*.py` step). They serve as decorrelated "no-context-demos" mediators in the ensemble.
+
+| # | Model | K | Prompt | Epochs | cutoff_len | eff_bs | YAML | Best ckpt | Cand |
+|---:|---|:---:|:---:|:---:|:---:|:---:|---|---|---|
+| 13 | Qwen3.5-27B | 3 | v1 | 5 | 4096 | 64 | **qwen35_27b_nofewshots_5ep_bs64_8gpu.yaml** | ck-2800 | 1 cand |
+| 14 | Qwen3.6-27B | 3 | v1 | 5 | 4096 | 64 | **qwen36_27b_nofewshots_5ep_bs64_8gpu.yaml** | ck-2600 | 1 cand |
+| 15 | Qwen3-32B | 3 | v1 | 4 | 4096 | 64 | **qwen3_32b_nofewshots_4ep_bs64_8gpu.yaml** | ck-6500 | 1 cand |
+
+**Differences vs fewshot runs**: only `dataset: msrh_rag_train_afrie5_TV_k3` (vs `*_k3_fewshot`) and longer epoch budget (4-5 vs 3) since no-demo trains more slowly.
+
+#### Total: 15 SFT runs → 19 ensemble cands (3 runs contribute 2 ckpts each)
+
+### Launch example (any config)
+```bash
+HF_DATASETS_OFFLINE=1 TRANSFORMERS_OFFLINE=1 \
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+FORCE_TORCHRUN=1 NNODES=1 NPROC_PER_NODE=8 \
+llamafactory-cli train /mnt/msrh/Magic_submission/configs/qwen35_27b_fewshot_k5_v8_3ep_8gpu.yaml \
+  > /mnt/msrh/Magic_submission/training_logs/q35_k5_v8.log 2>&1
+```
+
+Save_steps=100-500 (depends on config), save_total_limit=25 (keep multiple ckpts for early-tap predictions).
+
+### Wall time per run (8× H100 80GB)
+- Q3.5-27B 3ep: ~7-8h
+- Q3.6-27B 3ep: ~8-10h
+- Q3-32B 3ep: ~13-14h
+- Q3-32B 4ep (NoFewshots): ~18h
+- Q3.5/3.6-27B 5ep (NoFewshots): ~12-15h
+- Total for 15 SFT runs (sequential): ~7-9 days
+- With 2-3 nodes parallel: ~3-4 days
+
+---
+
+## 6. Inference (each LoRA adapter)
+
+Single-GPU inference per adapter via vLLM (env: `vllm`). The wrapper `launch_all_predicts.sh` orchestrates all 19 runs and writes outputs into `candidate_csvs/<descriptive_name>.csv` ready for ensemble.
+
+### One-shot run all 19 predictions
+```bash
+bash /mnt/msrh/Magic_submission/scripts/launch_all_predicts.sh
+# Outputs: /mnt/msrh/Magic_submission/candidate_csvs/{Qwen3.5-27B-..., Qwen3.6-27B-..., Qwen3-32B-...}.csv (19 files)
+```
+
+### Manual single-cand example (the key v8 ckpt)
+```bash
+ROOT=/mnt/msrh/Magic_submission
+CUDA_VISIBLE_DEVICES=0 python $ROOT/scripts/vllm_predict_extra.py \
+  --base   $ROOT/hub/Qwen3.5-27B \
+  --adapter $ROOT/checkpoints/Qwen3.5-27B-5fewshots-bs64-3eps-v8prompt-ckpt-1500 \
+  --rag_test $ROOT/LF/data/msrh_rag_test_k3_AfriE5_TV_fewshot_k5_v8.json \
+  --out_jsonl $ROOT/_predict_workdir/Qwen3.5-27B-5fewshots-bs64-3eps-v8prompt-ckpt-1500/predictions.jsonl \
+  --max_lora_rank 128 --max_new 512 \
+  --max_model_len 8192 --mem_util 0.88 --max_num_seqs 32 \
+  --temperature 0.0 --top_p 1.0 --best_of 1 --no_think
+
+python $ROOT/scripts/jsonl_rowidx_to_csv.py \
+  --jsonl $ROOT/_predict_workdir/Qwen3.5-27B-5fewshots-bs64-3eps-v8prompt-ckpt-1500/predictions.jsonl \
+  --out_csv $ROOT/candidate_csvs/Qwen3.5-27B-5fewshots-bs64-3eps-v8prompt-ckpt-1500.csv
+```
+
+### Critical inference flags
+- `--no_think` — passes `enable_thinking=False` to chat template (Qwen3.5/6 trained with `qwen3_5_nothink` template). Default ON wastes generation budget on reasoning; LB drops 0.10-0.25.
+- `--temperature 0.0 --best_of 1` — greedy deterministic decoding
+- `--max_model_len`: 6144 for K=3/4; 8192 for K=5; 10240 for K=7 (long prompts)
+
+**Provide `--adapter checkpoints/<descriptive_name>`** pointing into the provided `checkpoints/` folder. Run inference for all 19 candidates (each on a different GPU in parallel via the launcher). The output 19 CSVs go to `candidate_csvs/`, ready for ensemble.
+
+> **Note**: This package does NOT ship pre-computed `candidate_csvs/*.csv`.
+> They must be regenerated from `checkpoints/` via `scripts/launch_all_predicts.sh`
+> so the reviewer can independently verify the inference step.
+
+---
+
+## 7. Final ensemble (V2 medoid_ngram)
+
+```bash
+# (no GPU needed — pure CPU + ROUGE scoring)
+pip install rouge-score==0.1.2
+
+python scripts/build_ensemble.py
+# Reads 19 CSVs from candidate_csvs/<descriptive_name>.csv
+# Computes per-row medoid, writes the regenerated submission CSV
+# Compares md5 against the shipped go.csv: a2ecca4a8e1aa01acf9a8b9a1d56ebf2
+```
+
+### V2 medoid_ngram algorithm
+For each Test row:
+1. Compute pairwise ROUGE-1.F + ROUGE-2.F between all 19 candidate answers
+2. For each candidate, sum its similarities to all OTHERS
+3. Pick the candidate with HIGHEST sum (the "medoid" of the cluster)
+4. Use its answer text as final prediction
+
+Implementation (excerpt from `build_ensemble.py`):
+```python
+from rouge_score import rouge_scorer
+scorer = rouge_scorer.RougeScorer(["rouge1", "rouge2"], use_stemmer=False)
+
+def medoid(texts):
+    best, best_s = 0, -1.0
+    for i in range(len(texts)):
+        s = sum(
+            scorer.score(texts[j], texts[i])["rouge1"].fmeasure
+          + scorer.score(texts[j], texts[i])["rouge2"].fmeasure
+            for j in range(len(texts)) if i != j
+        )
+        if s > best_s: best_s, best = s, i
+    return best
+```
+
+---
+
+## 8. Reproducibility
+
+### Hash of final submission
+```
+$ md5sum go.csv
+a2ecca4a8e1aa01acf9a8b9a1d56ebf2  go.csv
+```
+
+### Pinned versions (env)
+See `environment.md` — exact `torch` / `transformers` / `vllm` / `llamafactory` versions.
+
+### Random seed
+- LlamaFactory default seed (42) used for all trainings — reproducible LoRA weights given same env/data
+- vLLM inference is deterministic with `--temperature 0.0 --best_of 1`
+- Ensemble medoid is purely deterministic (no random selection)
+
+### Sanity checks
+1. Verify CSV row order matches `data/Test.csv` (2618 rows, 4 target columns: TargetRLF1, TargetR1F1, TargetLLMJudge, all = same answer)
+2. Confirm md5sum of `go.csv` matches above
+
+---
+
+## 9. Acknowledgments
+
+- **AfriE5** (`McGill-NLP/AfriE5-Large-instruct`) — multilingual retrieval encoder
+- **LlamaFactory** — LoRA training framework
+- **vLLM** — inference engine (LoRA + batched generation)
+- **DeepSpeed** — ZeRO-3 distributed training
+- **Qwen team** (Alibaba) — Qwen3.5/3.6/Qwen3 base models
+
+---
+
+## 10. Contact
+
+Submitter: Magic
+Submission deadline: 23 Jun 2026 17:00 GMT (handled in time ✓)

checkpoints/Qwen3-32B-3fewshots-bs64-3eps-ckpt-1400/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "/mnt/msrh/Magic_submission/hub/Qwen3-32B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 256,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 128,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "up_proj",
+    "q_proj",
+    "k_proj",
+    "o_proj",
+    "down_proj",
+    "gate_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoints/Qwen3-32B-3fewshots-bs64-3eps-ckpt-1400/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:433a7ea2b4ec4d63d621d18001ba1c207830bb80dda97f0b1c6e6b25da5f83dd
+size 2147607752

checkpoints/Qwen3-32B-3fewshots-bs64-3eps-ckpt-1400/chat_template.jinja

@@ -0,0 +1,89 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}

checkpoints/Qwen3-32B-3fewshots-bs64-3eps-ckpt-1400/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650

checkpoints/Qwen3-32B-3fewshots-bs64-3eps-ckpt-1400/tokenizer_config.json

@@ -0,0 +1,30 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": true,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

checkpoints/Qwen3-32B-3fewshots-bs64-3eps-ckpt-1400/trainer_state.json

@@ -0,0 +1,524 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.4522348816827346,
+  "eval_steps": 500,
+  "global_step": 1400,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.035056967572304996,
+      "grad_norm": 0.25972092924861306,
+      "learning_rate": 4.418604651162791e-05,
+      "loss": 1.129275131225586,
+      "step": 20
+    },
+    {
+      "epoch": 0.07011393514460999,
+      "grad_norm": 0.2422051633438551,
+      "learning_rate": 9.069767441860465e-05,
+      "loss": 0.8681215286254883,
+      "step": 40
+    },
+    {
+      "epoch": 0.10517090271691498,
+      "grad_norm": 0.2887073624701644,
+      "learning_rate": 0.0001372093023255814,
+      "loss": 0.8148941040039063,
+      "step": 60
+    },
+    {
+      "epoch": 0.14022787028921999,
+      "grad_norm": 0.366313821529039,
+      "learning_rate": 0.00018372093023255815,
+      "loss": 0.7794719219207764,
+      "step": 80
+    },
+    {
+      "epoch": 0.175284837861525,
+      "grad_norm": 0.30330125407575625,
+      "learning_rate": 0.00019996849651899697,
+      "loss": 0.7361757278442382,
+      "step": 100
+    },
+    {
+      "epoch": 0.21034180543382996,
+      "grad_norm": 0.3051692510470446,
+      "learning_rate": 0.00019979705630030178,
+      "loss": 0.7342663764953613,
+      "step": 120
+    },
+    {
+      "epoch": 0.24539877300613497,
+      "grad_norm": 0.2859980613591142,
+      "learning_rate": 0.00019947680050653714,
+      "loss": 0.706333065032959,
+      "step": 140
+    },
+    {
+      "epoch": 0.28045574057843997,
+      "grad_norm": 0.2761393123380369,
+      "learning_rate": 0.00019900820669738146,
+      "loss": 0.7175551891326905,
+      "step": 160
+    },
+    {
+      "epoch": 0.31551270815074495,
+      "grad_norm": 0.22278342882751814,
+      "learning_rate": 0.0001983919736314933,
+      "loss": 0.643967580795288,
+      "step": 180
+    },
+    {
+      "epoch": 0.35056967572305,
+      "grad_norm": 0.308908749150725,
+      "learning_rate": 0.00019762902022453483,
+      "loss": 0.6350432395935058,
+      "step": 200
+    },
+    {
+      "epoch": 0.38562664329535495,
+      "grad_norm": 0.24328698982598382,
+      "learning_rate": 0.00019672048417890148,
+      "loss": 0.6911206245422363,
+      "step": 220
+    },
+    {
+      "epoch": 0.42068361086765993,
+      "grad_norm": 0.26901727474976633,
+      "learning_rate": 0.00019566772028720047,
+      "loss": 0.6216803550720215,
+      "step": 240
+    },
+    {
+      "epoch": 0.45574057843996496,
+      "grad_norm": 0.25743933187713164,
+      "learning_rate": 0.0001944722984120086,
+      "loss": 0.6582651138305664,
+      "step": 260
+    },
+    {
+      "epoch": 0.49079754601226994,
+      "grad_norm": 0.247129684938608,
+      "learning_rate": 0.0001931360011449213,
+      "loss": 0.609255027770996,
+      "step": 280
+    },
+    {
+      "epoch": 0.5258545135845749,
+      "grad_norm": 0.26408100728659223,
+      "learning_rate": 0.00019166082114838404,
+      "loss": 0.5517263412475586,
+      "step": 300
+    },
+    {
+      "epoch": 0.5609114811568799,
+      "grad_norm": 0.2903976116104128,
+      "learning_rate": 0.00019004895818427016,
+      "loss": 0.6067933559417724,
+      "step": 320
+    },
+    {
+      "epoch": 0.595968448729185,
+      "grad_norm": 0.30035929820797935,
+      "learning_rate": 0.00018830281583363535,
+      "loss": 0.588967227935791,
+      "step": 340
+    },
+    {
+      "epoch": 0.6310254163014899,
+      "grad_norm": 0.3076789784365002,
+      "learning_rate": 0.00018642499791254085,
+      "loss": 0.5784377098083496,
+      "step": 360
+    },
+    {
+      "epoch": 0.6660823838737949,
+      "grad_norm": 0.18654871412958857,
+      "learning_rate": 0.0001844183045892899,
+      "loss": 0.5761405467987061,
+      "step": 380
+    },
+    {
+      "epoch": 0.7011393514461,
+      "grad_norm": 0.2609291155306374,
+      "learning_rate": 0.00018228572820886695,
+      "loss": 0.5722132682800293,
+      "step": 400
+    },
+    {
+      "epoch": 0.7361963190184049,
+      "grad_norm": 0.2545041382974561,
+      "learning_rate": 0.00018003044883080685,
+      "loss": 0.5407440185546875,
+      "step": 420
+    },
+    {
+      "epoch": 0.7712532865907099,
+      "grad_norm": 0.22992797521008082,
+      "learning_rate": 0.00017765582948714708,
+      "loss": 0.5364163398742676,
+      "step": 440
+    },
+    {
+      "epoch": 0.8063102541630149,
+      "grad_norm": 0.19774875758051882,
+      "learning_rate": 0.00017516541116753502,
+      "loss": 0.5518630504608154,
+      "step": 460
+    },
+    {
+      "epoch": 0.8413672217353199,
+      "grad_norm": 0.2574457625045496,
+      "learning_rate": 0.00017256290753896782,
+      "loss": 0.5212277412414551,
+      "step": 480
+    },
+    {
+      "epoch": 0.8764241893076249,
+      "grad_norm": 0.21733628753506873,
+      "learning_rate": 0.00016985219940803906,
+      "loss": 0.5246070384979248,
+      "step": 500
+    },
+    {
+      "epoch": 0.9114811568799299,
+      "grad_norm": 0.2788941457185204,
+      "learning_rate": 0.00016703732893394964,
+      "loss": 0.5095488548278808,
+      "step": 520
+    },
+    {
+      "epoch": 0.9465381244522348,
+      "grad_norm": 0.2786955638768144,
+      "learning_rate": 0.00016412249360091277,
+      "loss": 0.5111159324645996,
+      "step": 540
+    },
+    {
+      "epoch": 0.9815950920245399,
+      "grad_norm": 0.21493767193150215,
+      "learning_rate": 0.00016111203995894112,
+      "loss": 0.4957179069519043,
+      "step": 560
+    },
+    {
+      "epoch": 1.0157756354075373,
+      "grad_norm": 0.2999874031381283,
+      "learning_rate": 0.00015801045714234938,
+      "loss": 0.46585869789123535,
+      "step": 580
+    },
+    {
+      "epoch": 1.0508326029798423,
+      "grad_norm": 0.27182858126516,
+      "learning_rate": 0.00015482237017563812,
+      "loss": 0.40743699073791506,
+      "step": 600
+    },
+    {
+      "epoch": 1.0858895705521472,
+      "grad_norm": 0.2315917457694845,
+      "learning_rate": 0.00015155253307674055,
+      "loss": 0.40488224029541015,
+      "step": 620
+    },
+    {
+      "epoch": 1.1209465381244523,
+      "grad_norm": 0.2544813878414733,
+      "learning_rate": 0.00014820582176791621,
+      "loss": 0.39172637462615967,
+      "step": 640
+    },
+    {
+      "epoch": 1.1560035056967572,
+      "grad_norm": 0.26307335125726106,
+      "learning_rate": 0.00014478722680486358,
+      "loss": 0.3954883813858032,
+      "step": 660
+    },
+    {
+      "epoch": 1.1910604732690622,
+      "grad_norm": 0.22950942789781925,
+      "learning_rate": 0.00014130184593489327,
+      "loss": 0.39145448207855227,
+      "step": 680
+    },
+    {
+      "epoch": 1.2261174408413673,
+      "grad_norm": 0.2190049684104704,
+      "learning_rate": 0.00013775487649525872,
+      "loss": 0.3662476778030396,
+      "step": 700
+    },
+    {
+      "epoch": 1.2611744084136722,
+      "grad_norm": 0.25350828061700514,
+      "learning_rate": 0.0001341516076629807,
+      "loss": 0.41669607162475586,
+      "step": 720
+    },
+    {
+      "epoch": 1.2962313759859772,
+      "grad_norm": 0.27880610020425217,
+      "learning_rate": 0.00013049741256772129,
+      "loss": 0.4120013236999512,
+      "step": 740
+    },
+    {
+      "epoch": 1.331288343558282,
+      "grad_norm": 0.2547253635678651,
+      "learning_rate": 0.00012679774027946997,
+      "loss": 0.399219274520874,
+      "step": 760
+    },
+    {
+      "epoch": 1.3663453111305872,
+      "grad_norm": 0.24818247234458424,
+      "learning_rate": 0.00012305810768298812,
+      "loss": 0.383573579788208,
+      "step": 780
+    },
+    {
+      "epoch": 1.4014022787028921,
+      "grad_norm": 0.26467672232824363,
+      "learning_rate": 0.00011928409125113017,
+      "loss": 0.37129578590393064,
+      "step": 800
+    },
+    {
+      "epoch": 1.4364592462751973,
+      "grad_norm": 0.23843710989628483,
+      "learning_rate": 0.00011548131872930703,
+      "loss": 0.37547523975372316,
+      "step": 820
+    },
+    {
+      "epoch": 1.4715162138475022,
+      "grad_norm": 0.27932157919803147,
+      "learning_rate": 0.0001116554607434938,
+      "loss": 0.38849606513977053,
+      "step": 840
+    },
+    {
+      "epoch": 1.5065731814198071,
+      "grad_norm": 0.22744381445516224,
+      "learning_rate": 0.0001078122223442942,
+      "loss": 0.3888273239135742,
+      "step": 860
+    },
+    {
+      "epoch": 1.541630148992112,
+      "grad_norm": 0.22081815925303305,
+      "learning_rate": 0.00010395733449967172,
+      "loss": 0.3756257057189941,
+      "step": 880
+    },
+    {
+      "epoch": 1.5766871165644172,
+      "grad_norm": 0.24524791223665604,
+      "learning_rate": 0.00010009654554903323,
+      "loss": 0.36522583961486815,
+      "step": 900
+    },
+    {
+      "epoch": 1.6117440841367223,
+      "grad_norm": 0.27311550186403866,
+      "learning_rate": 9.62356126314088e-05,
+      "loss": 0.37842211723327634,
+      "step": 920
+    },
+    {
+      "epoch": 1.6468010517090272,
+      "grad_norm": 0.24957805768810531,
+      "learning_rate": 9.238029310050955e-05,
+      "loss": 0.36957459449768065,
+      "step": 940
+    },
+    {
+      "epoch": 1.6818580192813322,
+      "grad_norm": 0.2749659935193175,
+      "learning_rate": 8.853633593946508e-05,
+      "loss": 0.3913578510284424,
+      "step": 960
+    },
+    {
+      "epoch": 1.716914986853637,
+      "grad_norm": 0.2778166557209832,
+      "learning_rate": 8.47094731880437e-05,
+      "loss": 0.36133584976196287,
+      "step": 980
+    },
+    {
+      "epoch": 1.751971954425942,
+      "grad_norm": 0.28003762510185926,
+      "learning_rate": 8.090541139513744e-05,
+      "loss": 0.3650220394134521,
+      "step": 1000
+    },
+    {
+      "epoch": 1.7870289219982471,
+      "grad_norm": 0.2655418397043759,
+      "learning_rate": 7.712982310925923e-05,
+      "loss": 0.3749880790710449,
+      "step": 1020
+    },
+    {
+      "epoch": 1.8220858895705523,
+      "grad_norm": 0.22818762984408617,
+      "learning_rate": 7.33883384197406e-05,
+      "loss": 0.36241748332977297,
+      "step": 1040
+    },
+    {
+      "epoch": 1.8571428571428572,
+      "grad_norm": 0.22281359315424928,
+      "learning_rate": 6.96865365612436e-05,
+      "loss": 0.36051235198974607,
+      "step": 1060
+    },
+    {
+      "epoch": 1.8921998247151621,
+      "grad_norm": 0.24561953092610364,
+      "learning_rate": 6.602993759410652e-05,
+      "loss": 0.34312801361083983,
+      "step": 1080
+    },
+    {
+      "epoch": 1.927256792287467,
+      "grad_norm": 0.25437370430695394,
+      "learning_rate": 6.242399417292937e-05,
+      "loss": 0.3559926986694336,
+      "step": 1100
+    },
+    {
+      "epoch": 1.962313759859772,
+      "grad_norm": 0.22685063854469026,
+      "learning_rate": 5.887408341567306e-05,
+      "loss": 0.33498687744140626,
+      "step": 1120
+    },
+    {
+      "epoch": 1.997370727432077,
+      "grad_norm": 0.2123804155698742,
+      "learning_rate": 5.538549888539829e-05,
+      "loss": 0.34430203437805174,
+      "step": 1140
+    },
+    {
+      "epoch": 2.0315512708150747,
+      "grad_norm": 0.26834543697058755,
+      "learning_rate": 5.1963442696599096e-05,
+      "loss": 0.2343144178390503,
+      "step": 1160
+    },
+    {
+      "epoch": 2.0666082383873796,
+      "grad_norm": 0.2197628238907278,
+      "learning_rate": 4.861301775790361e-05,
+      "loss": 0.19864696264266968,
+      "step": 1180
+    },
+    {
+      "epoch": 2.1016652059596845,
+      "grad_norm": 0.29225792782050447,
+      "learning_rate": 4.5339220162708506e-05,
+      "loss": 0.18144433498382567,
+      "step": 1200
+    },
+    {
+      "epoch": 2.1367221735319895,
+      "grad_norm": 0.2805551896384275,
+      "learning_rate": 4.214693173909409e-05,
+      "loss": 0.1948443055152893,
+      "step": 1220
+    },
+    {
+      "epoch": 2.1717791411042944,
+      "grad_norm": 0.26588323236857764,
+      "learning_rate": 3.9040912770130454e-05,
+      "loss": 0.18083541393280028,
+      "step": 1240
+    },
+    {
+      "epoch": 2.2068361086765993,
+      "grad_norm": 0.24411128604094878,
+      "learning_rate": 3.602579489542883e-05,
+      "loss": 0.19837160110473634,
+      "step": 1260
+    },
+    {
+      "epoch": 2.2418930762489047,
+      "grad_norm": 0.28033662973674933,
+      "learning_rate": 3.3106074204523905e-05,
+      "loss": 0.17888429164886474,
+      "step": 1280
+    },
+    {
+      "epoch": 2.2769500438212096,
+      "grad_norm": 0.3015485429696721,
+      "learning_rate": 3.028610453238604e-05,
+      "loss": 0.1615851879119873,
+      "step": 1300
+    },
+    {
+      "epoch": 2.3120070113935145,
+      "grad_norm": 0.3401883272816503,
+      "learning_rate": 2.7570090967060868e-05,
+      "loss": 0.17859578132629395,
+      "step": 1320
+    },
+    {
+      "epoch": 2.3470639789658194,
+      "grad_norm": 0.3493775742732503,
+      "learning_rate": 2.4962083579117656e-05,
+      "loss": 0.19249472618103028,
+      "step": 1340
+    },
+    {
+      "epoch": 2.3821209465381243,
+      "grad_norm": 0.2586960289960212,
+      "learning_rate": 2.246597138225691e-05,
+      "loss": 0.17364020347595216,
+      "step": 1360
+    },
+    {
+      "epoch": 2.4171779141104293,
+      "grad_norm": 0.2353627458632912,
+      "learning_rate": 2.0085476534083103e-05,
+      "loss": 0.20382912158966066,
+      "step": 1380
+    },
+    {
+      "epoch": 2.4522348816827346,
+      "grad_norm": 0.30700117695397516,
+      "learning_rate": 1.7824148785690288e-05,
+      "loss": 0.18904035091400145,
+      "step": 1400
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 1713,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 636041574744064.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoints/Qwen3-32B-5fewshots-bs64-3eps-ckpt-1700/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "/mnt/msrh/Magic_submission/hub/Qwen3-32B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 256,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 128,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "down_proj",
+    "gate_proj",
+    "up_proj",
+    "o_proj",
+    "v_proj",
+    "k_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoints/Qwen3-32B-5fewshots-bs64-3eps-ckpt-1700/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b14d38d568ef04cfeb7606abae3a5f7def1a91c377d339590564e1ac4dd7f8c
+size 2147607752

checkpoints/Qwen3-32B-5fewshots-bs64-3eps-ckpt-1700/chat_template.jinja

@@ -0,0 +1,89 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}

checkpoints/Qwen3-32B-5fewshots-bs64-3eps-ckpt-1700/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650

checkpoints/Qwen3-32B-5fewshots-bs64-3eps-ckpt-1700/tokenizer_config.json

@@ -0,0 +1,30 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": true,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

checkpoints/Qwen3-32B-5fewshots-bs64-3eps-ckpt-1700/trainer_state.json

@@ -0,0 +1,629 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.9780893952673093,
+  "eval_steps": 500,
+  "global_step": 1700,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.035056967572304996,
+      "grad_norm": 0.2607329259109832,
+      "learning_rate": 4.418604651162791e-05,
+      "loss": 1.0697125434875487,
+      "step": 20
+    },
+    {
+      "epoch": 0.07011393514460999,
+      "grad_norm": 0.21943806459933496,
+      "learning_rate": 9.069767441860465e-05,
+      "loss": 0.8158013343811035,
+      "step": 40
+    },
+    {
+      "epoch": 0.10517090271691498,
+      "grad_norm": 0.23920311705242867,
+      "learning_rate": 0.0001372093023255814,
+      "loss": 0.7821506977081298,
+      "step": 60
+    },
+    {
+      "epoch": 0.14022787028921999,
+      "grad_norm": 0.35946578576738486,
+      "learning_rate": 0.00018372093023255815,
+      "loss": 0.7410243034362793,
+      "step": 80
+    },
+    {
+      "epoch": 0.175284837861525,
+      "grad_norm": 0.4306438291535812,
+      "learning_rate": 0.00019996849651899697,
+      "loss": 0.7126483917236328,
+      "step": 100
+    },
+    {
+      "epoch": 0.21034180543382996,
+      "grad_norm": 0.28500262211676214,
+      "learning_rate": 0.00019979705630030178,
+      "loss": 0.7145627975463867,
+      "step": 120
+    },
+    {
+      "epoch": 0.24539877300613497,
+      "grad_norm": 0.260601498041297,
+      "learning_rate": 0.00019947680050653714,
+      "loss": 0.6702024459838867,
+      "step": 140
+    },
+    {
+      "epoch": 0.28045574057843997,
+      "grad_norm": 0.24318856995850366,
+      "learning_rate": 0.00019900820669738146,
+      "loss": 0.6751674175262451,
+      "step": 160
+    },
+    {
+      "epoch": 0.31551270815074495,
+      "grad_norm": 0.25488466291309136,
+      "learning_rate": 0.0001983919736314933,
+      "loss": 0.6340686798095703,
+      "step": 180
+    },
+    {
+      "epoch": 0.35056967572305,
+      "grad_norm": 0.28321416380377346,
+      "learning_rate": 0.00019762902022453483,
+      "loss": 0.5914155960083007,
+      "step": 200
+    },
+    {
+      "epoch": 0.38562664329535495,
+      "grad_norm": 0.23973446586875694,
+      "learning_rate": 0.00019672048417890148,
+      "loss": 0.6677542686462402,
+      "step": 220
+    },
+    {
+      "epoch": 0.42068361086765993,
+      "grad_norm": 0.26172850316934704,
+      "learning_rate": 0.00019566772028720047,
+      "loss": 0.6005615711212158,
+      "step": 240
+    },
+    {
+      "epoch": 0.45574057843996496,
+      "grad_norm": 0.251076056535901,
+      "learning_rate": 0.0001944722984120086,
+      "loss": 0.6369094848632812,
+      "step": 260
+    },
+    {
+      "epoch": 0.49079754601226994,
+      "grad_norm": 0.21698674468743823,
+      "learning_rate": 0.0001931360011449213,
+      "loss": 0.5932572841644287,
+      "step": 280
+    },
+    {
+      "epoch": 0.5258545135845749,
+      "grad_norm": 0.2673393007859968,
+      "learning_rate": 0.00019166082114838404,
+      "loss": 0.5315813541412353,
+      "step": 300
+    },
+    {
+      "epoch": 0.5609114811568799,
+      "grad_norm": 0.2577888244518932,
+      "learning_rate": 0.00019004895818427016,
+      "loss": 0.6088322162628174,
+      "step": 320
+    },
+    {
+      "epoch": 0.595968448729185,
+      "grad_norm": 0.2525655347212635,
+      "learning_rate": 0.00018830281583363535,
+      "loss": 0.5806102275848388,
+      "step": 340
+    },
+    {
+      "epoch": 0.6310254163014899,
+      "grad_norm": 0.2723731142723013,
+      "learning_rate": 0.00018642499791254085,
+      "loss": 0.561124038696289,
+      "step": 360
+    },
+    {
+      "epoch": 0.6660823838737949,
+      "grad_norm": 0.17912741261394458,
+      "learning_rate": 0.0001844183045892899,
+      "loss": 0.5584434032440185,
+      "step": 380
+    },
+    {
+      "epoch": 0.7011393514461,
+      "grad_norm": 0.18549735701435288,
+      "learning_rate": 0.00018228572820886695,
+      "loss": 0.5559101104736328,
+      "step": 400
+    },
+    {
+      "epoch": 0.7361963190184049,
+      "grad_norm": 0.22414243532910372,
+      "learning_rate": 0.00018003044883080685,
+      "loss": 0.5252221107482911,
+      "step": 420
+    },
+    {
+      "epoch": 0.7712532865907099,
+      "grad_norm": 0.2167055079608228,
+      "learning_rate": 0.00017765582948714708,
+      "loss": 0.5327632904052735,
+      "step": 440
+    },
+    {
+      "epoch": 0.8063102541630149,
+      "grad_norm": 0.21328948837791448,
+      "learning_rate": 0.00017516541116753502,
+      "loss": 0.5318605422973632,
+      "step": 460
+    },
+    {
+      "epoch": 0.8413672217353199,
+      "grad_norm": 0.26134858618941426,
+      "learning_rate": 0.00017256290753896782,
+      "loss": 0.5201922416687011,
+      "step": 480
+    },
+    {
+      "epoch": 0.8764241893076249,
+      "grad_norm": 0.16844582128925603,
+      "learning_rate": 0.00016985219940803906,
+      "loss": 0.5006133556365967,
+      "step": 500
+    },
+    {
+      "epoch": 0.9114811568799299,
+      "grad_norm": 0.20268123902184498,
+      "learning_rate": 0.00016703732893394964,
+      "loss": 0.500429344177246,
+      "step": 520
+    },
+    {
+      "epoch": 0.9465381244522348,
+      "grad_norm": 0.2631331322702879,
+      "learning_rate": 0.00016412249360091277,
+      "loss": 0.49962558746337893,
+      "step": 540
+    },
+    {
+      "epoch": 0.9815950920245399,
+      "grad_norm": 0.1991947295240109,
+      "learning_rate": 0.00016111203995894112,
+      "loss": 0.48817882537841795,
+      "step": 560
+    },
+    {
+      "epoch": 1.0157756354075373,
+      "grad_norm": 0.33491726330950855,
+      "learning_rate": 0.00015801045714234938,
+      "loss": 0.4481941223144531,
+      "step": 580
+    },
+    {
+      "epoch": 1.0508326029798423,
+      "grad_norm": 0.2657827306745051,
+      "learning_rate": 0.00015482237017563812,
+      "loss": 0.396005916595459,
+      "step": 600
+    },
+    {
+      "epoch": 1.0858895705521472,
+      "grad_norm": 0.28475768185801104,
+      "learning_rate": 0.00015155253307674055,
+      "loss": 0.3950441837310791,
+      "step": 620
+    },
+    {
+      "epoch": 1.1209465381244523,
+      "grad_norm": 0.26358760611110005,
+      "learning_rate": 0.00014820582176791621,
+      "loss": 0.3884115695953369,
+      "step": 640
+    },
+    {
+      "epoch": 1.1560035056967572,
+      "grad_norm": 0.2512485387459438,
+      "learning_rate": 0.00014478722680486358,
+      "loss": 0.3917705535888672,
+      "step": 660
+    },
+    {
+      "epoch": 1.1910604732690622,
+      "grad_norm": 0.20657251754939812,
+      "learning_rate": 0.00014130184593489327,
+      "loss": 0.37499542236328126,
+      "step": 680
+    },
+    {
+      "epoch": 1.2261174408413673,
+      "grad_norm": 0.24373611516397556,
+      "learning_rate": 0.00013775487649525872,
+      "loss": 0.36916706562042234,
+      "step": 700
+    },
+    {
+      "epoch": 1.2611744084136722,
+      "grad_norm": 0.24121351808359115,
+      "learning_rate": 0.0001341516076629807,
+      "loss": 0.40265941619873047,
+      "step": 720
+    },
+    {
+      "epoch": 1.2962313759859772,
+      "grad_norm": 0.27014690636269456,
+      "learning_rate": 0.00013049741256772129,
+      "loss": 0.4007719039916992,
+      "step": 740
+    },
+    {
+      "epoch": 1.331288343558282,
+      "grad_norm": 0.2255514096887592,
+      "learning_rate": 0.00012679774027946997,
+      "loss": 0.40675530433654783,
+      "step": 760
+    },
+    {
+      "epoch": 1.3663453111305872,
+      "grad_norm": 0.20477370973186418,
+      "learning_rate": 0.00012305810768298812,
+      "loss": 0.3776648283004761,
+      "step": 780
+    },
+    {
+      "epoch": 1.4014022787028921,
+      "grad_norm": 0.2637082547476057,
+      "learning_rate": 0.00011928409125113017,
+      "loss": 0.36629462242126465,
+      "step": 800
+    },
+    {
+      "epoch": 1.4364592462751973,
+      "grad_norm": 0.24368053403889028,
+      "learning_rate": 0.00011548131872930703,
+      "loss": 0.37551727294921877,
+      "step": 820
+    },
+    {
+      "epoch": 1.4715162138475022,
+      "grad_norm": 0.24247939972502802,
+      "learning_rate": 0.0001116554607434938,
+      "loss": 0.37543768882751466,
+      "step": 840
+    },
+    {
+      "epoch": 1.5065731814198071,
+      "grad_norm": 0.23018505724245242,
+      "learning_rate": 0.0001078122223442942,
+      "loss": 0.38346543312072756,
+      "step": 860
+    },
+    {
+      "epoch": 1.541630148992112,
+      "grad_norm": 0.22079038789202285,
+      "learning_rate": 0.00010395733449967172,
+      "loss": 0.3711265802383423,
+      "step": 880
+    },
+    {
+      "epoch": 1.5766871165644172,
+      "grad_norm": 0.2388872581772674,
+      "learning_rate": 0.00010009654554903323,
+      "loss": 0.3541673421859741,
+      "step": 900
+    },
+    {
+      "epoch": 1.6117440841367223,
+      "grad_norm": 0.2784817886817032,
+      "learning_rate": 9.62356126314088e-05,
+      "loss": 0.3742485046386719,
+      "step": 920
+    },
+    {
+      "epoch": 1.6468010517090272,
+      "grad_norm": 0.25593059766609416,
+      "learning_rate": 9.238029310050955e-05,
+      "loss": 0.3586562633514404,
+      "step": 940
+    },
+    {
+      "epoch": 1.6818580192813322,
+      "grad_norm": 0.2806883141870909,
+      "learning_rate": 8.853633593946508e-05,
+      "loss": 0.3772284507751465,
+      "step": 960
+    },
+    {
+      "epoch": 1.716914986853637,
+      "grad_norm": 0.2450152979933712,
+      "learning_rate": 8.47094731880437e-05,
+      "loss": 0.35790233612060546,
+      "step": 980
+    },
+    {
+      "epoch": 1.751971954425942,
+      "grad_norm": 0.27629406054112926,
+      "learning_rate": 8.090541139513744e-05,
+      "loss": 0.3631918907165527,
+      "step": 1000
+    },
+    {
+      "epoch": 1.7870289219982471,
+      "grad_norm": 0.2898417678366289,
+      "learning_rate": 7.712982310925923e-05,
+      "loss": 0.37659993171691897,
+      "step": 1020
+    },
+    {
+      "epoch": 1.8220858895705523,
+      "grad_norm": 0.25203506526214525,
+      "learning_rate": 7.33883384197406e-05,
+      "loss": 0.3618522882461548,
+      "step": 1040
+    },
+    {
+      "epoch": 1.8571428571428572,
+      "grad_norm": 0.19830217542597472,
+      "learning_rate": 6.96865365612436e-05,
+      "loss": 0.3597991466522217,
+      "step": 1060
+    },
+    {
+      "epoch": 1.8921998247151621,
+      "grad_norm": 0.23656700271611542,
+      "learning_rate": 6.602993759410652e-05,
+      "loss": 0.3347702264785767,
+      "step": 1080
+    },
+    {
+      "epoch": 1.927256792287467,
+      "grad_norm": 0.21906548679161278,
+      "learning_rate": 6.242399417292937e-05,
+      "loss": 0.34678964614868163,
+      "step": 1100
+    },
+    {
+      "epoch": 1.962313759859772,
+      "grad_norm": 0.22231553177754299,
+      "learning_rate": 5.887408341567306e-05,
+      "loss": 0.33619203567504885,
+      "step": 1120
+    },
+    {
+      "epoch": 1.997370727432077,
+      "grad_norm": 0.21574250656389074,
+      "learning_rate": 5.538549888539829e-05,
+      "loss": 0.34700756072998046,
+      "step": 1140
+    },
+    {
+      "epoch": 2.0315512708150747,
+      "grad_norm": 0.2479814425125438,
+      "learning_rate": 5.1963442696599096e-05,
+      "loss": 0.23154356479644775,
+      "step": 1160
+    },
+    {
+      "epoch": 2.0666082383873796,
+      "grad_norm": 0.2149324450556186,
+      "learning_rate": 4.861301775790361e-05,
+      "loss": 0.19337258338928223,
+      "step": 1180
+    },
+    {
+      "epoch": 2.1016652059596845,
+      "grad_norm": 0.30076038455299203,
+      "learning_rate": 4.5339220162708506e-05,
+      "loss": 0.1809700608253479,
+      "step": 1200
+    },
+    {
+      "epoch": 2.1367221735319895,
+      "grad_norm": 0.2639614289581006,
+      "learning_rate": 4.214693173909409e-05,
+      "loss": 0.19562238454818726,
+      "step": 1220
+    },
+    {
+      "epoch": 2.1717791411042944,
+      "grad_norm": 0.26707425864651063,
+      "learning_rate": 3.9040912770130454e-05,
+      "loss": 0.176566743850708,
+      "step": 1240
+    },
+    {
+      "epoch": 2.2068361086765993,
+      "grad_norm": 0.23275234814494444,
+      "learning_rate": 3.602579489542883e-05,
+      "loss": 0.19572536945343016,
+      "step": 1260
+    },
+    {
+      "epoch": 2.2418930762489047,
+      "grad_norm": 0.25986541139016195,
+      "learning_rate": 3.3106074204523905e-05,
+      "loss": 0.17627604007720948,
+      "step": 1280
+    },
+    {
+      "epoch": 2.2769500438212096,
+      "grad_norm": 0.2927785159706672,
+      "learning_rate": 3.028610453238604e-05,
+      "loss": 0.1629226803779602,
+      "step": 1300
+    },
+    {
+      "epoch": 2.3120070113935145,
+      "grad_norm": 0.301171644263262,
+      "learning_rate": 2.7570090967060868e-05,
+      "loss": 0.17633190155029296,
+      "step": 1320
+    },
+    {
+      "epoch": 2.3470639789658194,
+      "grad_norm": 0.33307553082898383,
+      "learning_rate": 2.4962083579117656e-05,
+      "loss": 0.19391660690307616,
+      "step": 1340
+    },
+    {
+      "epoch": 2.3821209465381243,
+      "grad_norm": 0.22976355013021266,
+      "learning_rate": 2.246597138225691e-05,
+      "loss": 0.1659176230430603,
+      "step": 1360
+    },
+    {
+      "epoch": 2.4171779141104293,
+      "grad_norm": 0.2368618946629272,
+      "learning_rate": 2.0085476534083103e-05,
+      "loss": 0.20211286544799806,
+      "step": 1380
+    },
+    {
+      "epoch": 2.4522348816827346,
+      "grad_norm": 0.28220568763488246,
+      "learning_rate": 1.7824148785690288e-05,
+      "loss": 0.18701364994049072,
+      "step": 1400
+    },
+    {
+      "epoch": 2.4872918492550395,
+      "grad_norm": 0.27525417176012384,
+      "learning_rate": 1.568536018833694e-05,
+      "loss": 0.179413902759552,
+      "step": 1420
+    },
+    {
+      "epoch": 2.5223488168273445,
+      "grad_norm": 0.25506772504918335,
+      "learning_rate": 1.367230006510406e-05,
+      "loss": 0.1777553677558899,
+      "step": 1440
+    },
+    {
+      "epoch": 2.5574057843996494,
+      "grad_norm": 0.30619870252269366,
+      "learning_rate": 1.178797025503373e-05,
+      "loss": 0.18152525424957275,
+      "step": 1460
+    },
+    {
+      "epoch": 2.5924627519719543,
+      "grad_norm": 0.31819376457547116,
+      "learning_rate": 1.003518063684079e-05,
+      "loss": 0.17822539806365967,
+      "step": 1480
+    },
+    {
+      "epoch": 2.6275197195442592,
+      "grad_norm": 0.28512188355801915,
+      "learning_rate": 8.416544938872385e-06,
+      "loss": 0.16258721351623534,
+      "step": 1500
+    },
+    {
+      "epoch": 2.662576687116564,
+      "grad_norm": 0.2930999235526907,
+      "learning_rate": 6.934476841563054e-06,
+      "loss": 0.18719613552093506,
+      "step": 1520
+    },
+    {
+      "epoch": 2.6976336546888695,
+      "grad_norm": 0.24868905102330174,
+      "learning_rate": 5.591186378198032e-06,
+      "loss": 0.1639933705329895,
+      "step": 1540
+    },
+    {
+      "epoch": 2.7326906222611744,
+      "grad_norm": 0.2599144539418039,
+      "learning_rate": 4.388676639351396e-06,
+      "loss": 0.1779949903488159,
+      "step": 1560
+    },
+    {
+      "epoch": 2.7677475898334793,
+      "grad_norm": 0.3095617178212254,
+      "learning_rate": 3.328740785913298e-06,
+      "loss": 0.18669604063034057,
+      "step": 1580
+    },
+    {
+      "epoch": 2.8028045574057843,
+      "grad_norm": 0.2832079189302701,
+      "learning_rate": 2.412959375160806e-06,
+      "loss": 0.17197535037994385,
+      "step": 1600
+    },
+    {
+      "epoch": 2.8378615249780896,
+      "grad_norm": 0.30013896053364103,
+      "learning_rate": 1.6426980038593332e-06,
+      "loss": 0.17178089618682862,
+      "step": 1620
+    },
+    {
+      "epoch": 2.8729184925503946,
+      "grad_norm": 0.2507133402692292,
+      "learning_rate": 1.0191052719092487e-06,
+      "loss": 0.17527294158935547,
+      "step": 1640
+    },
+    {
+      "epoch": 2.9079754601226995,
+      "grad_norm": 0.2524125166727659,
+      "learning_rate": 5.431110695743869e-07,
+      "loss": 0.19358527660369873,
+      "step": 1660
+    },
+    {
+      "epoch": 2.9430324276950044,
+      "grad_norm": 0.288458138416179,
+      "learning_rate": 2.154251908464411e-07,
+      "loss": 0.17915793657302856,
+      "step": 1680
+    },
+    {
+      "epoch": 2.9780893952673093,
+      "grad_norm": 0.25025541063149187,
+      "learning_rate": 3.653627501291057e-08,
+      "loss": 0.1812071681022644,
+      "step": 1700
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 1713,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 997161613393920.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoints/Qwen3-32B-7fewshots-bs64-3eps-ckpt-1200/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "/mnt/msrh/Magic_submission/hub/Qwen3-32B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 256,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 128,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "v_proj",
+    "up_proj",
+    "gate_proj",
+    "down_proj",
+    "q_proj",
+    "k_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoints/Qwen3-32B-7fewshots-bs64-3eps-ckpt-1200/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8cf660db6b67fc37c39fdb447d036d316424bd100bbc22d36f143316ae4d2eb8
+size 2147607752

checkpoints/Qwen3-32B-7fewshots-bs64-3eps-ckpt-1200/chat_template.jinja

@@ -0,0 +1,89 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}

checkpoints/Qwen3-32B-7fewshots-bs64-3eps-ckpt-1200/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650

checkpoints/Qwen3-32B-7fewshots-bs64-3eps-ckpt-1200/tokenizer_config.json

@@ -0,0 +1,30 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": true,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

checkpoints/Qwen3-32B-7fewshots-bs64-3eps-ckpt-1200/trainer_state.json

@@ -0,0 +1,454 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.1016652059596845,
+  "eval_steps": 500,
+  "global_step": 1200,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.035056967572304996,
+      "grad_norm": 0.23483171452352822,
+      "learning_rate": 4.418604651162791e-05,
+      "loss": 0.9909378051757812,
+      "step": 20
+    },
+    {
+      "epoch": 0.07011393514460999,
+      "grad_norm": 0.19986055220147195,
+      "learning_rate": 9.069767441860465e-05,
+      "loss": 0.7854896068572998,
+      "step": 40
+    },
+    {
+      "epoch": 0.10517090271691498,
+      "grad_norm": 0.23130651455988926,
+      "learning_rate": 0.0001372093023255814,
+      "loss": 0.7465881824493408,
+      "step": 60
+    },
+    {
+      "epoch": 0.14022787028921999,
+      "grad_norm": 0.3197363225238927,
+      "learning_rate": 0.00018372093023255815,
+      "loss": 0.7164266586303711,
+      "step": 80
+    },
+    {
+      "epoch": 0.175284837861525,
+      "grad_norm": 0.2715619931198569,
+      "learning_rate": 0.00019996849651899697,
+      "loss": 0.6907209873199462,
+      "step": 100
+    },
+    {
+      "epoch": 0.21034180543382996,
+      "grad_norm": 0.2856851996713809,
+      "learning_rate": 0.00019979705630030178,
+      "loss": 0.6624889373779297,
+      "step": 120
+    },
+    {
+      "epoch": 0.24539877300613497,
+      "grad_norm": 0.2621092411083009,
+      "learning_rate": 0.00019947680050653714,
+      "loss": 0.642543363571167,
+      "step": 140
+    },
+    {
+      "epoch": 0.28045574057843997,
+      "grad_norm": 0.3060417273271011,
+      "learning_rate": 0.00019900820669738146,
+      "loss": 0.646620512008667,
+      "step": 160
+    },
+    {
+      "epoch": 0.31551270815074495,
+      "grad_norm": 0.21452369072192615,
+      "learning_rate": 0.0001983919736314933,
+      "loss": 0.6074985504150391,
+      "step": 180
+    },
+    {
+      "epoch": 0.35056967572305,
+      "grad_norm": 0.30612438725480096,
+      "learning_rate": 0.00019762902022453483,
+      "loss": 0.5916189193725586,
+      "step": 200
+    },
+    {
+      "epoch": 0.38562664329535495,
+      "grad_norm": 0.2672865616168627,
+      "learning_rate": 0.00019672048417890148,
+      "loss": 0.6512822151184082,
+      "step": 220
+    },
+    {
+      "epoch": 0.42068361086765993,
+      "grad_norm": 0.24914148404479688,
+      "learning_rate": 0.00019566772028720047,
+      "loss": 0.587531566619873,
+      "step": 240
+    },
+    {
+      "epoch": 0.45574057843996496,
+      "grad_norm": 0.24657546374303219,
+      "learning_rate": 0.0001944722984120086,
+      "loss": 0.6211313724517822,
+      "step": 260
+    },
+    {
+      "epoch": 0.49079754601226994,
+      "grad_norm": 0.2271230878176654,
+      "learning_rate": 0.0001931360011449213,
+      "loss": 0.5668744087219239,
+      "step": 280
+    },
+    {
+      "epoch": 0.5258545135845749,
+      "grad_norm": 0.25583543263520286,
+      "learning_rate": 0.00019166082114838404,
+      "loss": 0.5299773216247559,
+      "step": 300
+    },
+    {
+      "epoch": 0.5609114811568799,
+      "grad_norm": 0.26046386701317537,
+      "learning_rate": 0.00019004895818427016,
+      "loss": 0.5769920349121094,
+      "step": 320
+    },
+    {
+      "epoch": 0.595968448729185,
+      "grad_norm": 0.2698990350624982,
+      "learning_rate": 0.00018830281583363535,
+      "loss": 0.5816852569580078,
+      "step": 340
+    },
+    {
+      "epoch": 0.6310254163014899,
+      "grad_norm": 0.26049160579851316,
+      "learning_rate": 0.00018642499791254085,
+      "loss": 0.5477001190185546,
+      "step": 360
+    },
+    {
+      "epoch": 0.6660823838737949,
+      "grad_norm": 0.1708923438980022,
+      "learning_rate": 0.0001844183045892899,
+      "loss": 0.5549935340881348,
+      "step": 380
+    },
+    {
+      "epoch": 0.7011393514461,
+      "grad_norm": 0.18290090201506068,
+      "learning_rate": 0.00018228572820886695,
+      "loss": 0.5305068492889404,
+      "step": 400
+    },
+    {
+      "epoch": 0.7361963190184049,
+      "grad_norm": 0.2120975286999116,
+      "learning_rate": 0.00018003044883080685,
+      "loss": 0.5133780479431153,
+      "step": 420
+    },
+    {
+      "epoch": 0.7712532865907099,
+      "grad_norm": 0.22392071288652587,
+      "learning_rate": 0.00017765582948714708,
+      "loss": 0.5189687728881835,
+      "step": 440
+    },
+    {
+      "epoch": 0.8063102541630149,
+      "grad_norm": 0.17532736599040075,
+      "learning_rate": 0.00017516541116753502,
+      "loss": 0.5183590888977051,
+      "step": 460
+    },
+    {
+      "epoch": 0.8413672217353199,
+      "grad_norm": 0.24787663667303497,
+      "learning_rate": 0.00017256290753896782,
+      "loss": 0.5011738300323486,
+      "step": 480
+    },
+    {
+      "epoch": 0.8764241893076249,
+      "grad_norm": 0.1730628728128248,
+      "learning_rate": 0.00016985219940803906,
+      "loss": 0.48698010444641116,
+      "step": 500
+    },
+    {
+      "epoch": 0.9114811568799299,
+      "grad_norm": 0.19806277891741542,
+      "learning_rate": 0.00016703732893394964,
+      "loss": 0.4896900177001953,
+      "step": 520
+    },
+    {
+      "epoch": 0.9465381244522348,
+      "grad_norm": 0.24968865256753012,
+      "learning_rate": 0.00016412249360091277,
+      "loss": 0.48049116134643555,
+      "step": 540
+    },
+    {
+      "epoch": 0.9815950920245399,
+      "grad_norm": 0.21341584083366677,
+      "learning_rate": 0.00016111203995894112,
+      "loss": 0.47271203994750977,
+      "step": 560
+    },
+    {
+      "epoch": 1.0157756354075373,
+      "grad_norm": 0.2811762357022581,
+      "learning_rate": 0.00015801045714234938,
+      "loss": 0.45186548233032225,
+      "step": 580
+    },
+    {
+      "epoch": 1.0508326029798423,
+      "grad_norm": 0.2624449978453146,
+      "learning_rate": 0.00015482237017563812,
+      "loss": 0.38692054748535154,
+      "step": 600
+    },
+    {
+      "epoch": 1.0858895705521472,
+      "grad_norm": 0.21413306480701308,
+      "learning_rate": 0.00015155253307674055,
+      "loss": 0.3919252634048462,
+      "step": 620
+    },
+    {
+      "epoch": 1.1209465381244523,
+      "grad_norm": 0.2683586780002,
+      "learning_rate": 0.00014820582176791621,
+      "loss": 0.3797740936279297,
+      "step": 640
+    },
+    {
+      "epoch": 1.1560035056967572,
+      "grad_norm": 0.24565139075016496,
+      "learning_rate": 0.00014478722680486358,
+      "loss": 0.37922732830047606,
+      "step": 660
+    },
+    {
+      "epoch": 1.1910604732690622,
+      "grad_norm": 0.19742886528967668,
+      "learning_rate": 0.00014130184593489327,
+      "loss": 0.36946282386779783,
+      "step": 680
+    },
+    {
+      "epoch": 1.2261174408413673,
+      "grad_norm": 0.20685035595170095,
+      "learning_rate": 0.00013775487649525872,
+      "loss": 0.357561731338501,
+      "step": 700
+    },
+    {
+      "epoch": 1.2611744084136722,
+      "grad_norm": 0.24705150218547564,
+      "learning_rate": 0.0001341516076629807,
+      "loss": 0.39932737350463865,
+      "step": 720
+    },
+    {
+      "epoch": 1.2962313759859772,
+      "grad_norm": 0.2683288997975443,
+      "learning_rate": 0.00013049741256772129,
+      "loss": 0.39545512199401855,
+      "step": 740
+    },
+    {
+      "epoch": 1.331288343558282,
+      "grad_norm": 0.22560628282285475,
+      "learning_rate": 0.00012679774027946997,
+      "loss": 0.39340255260467527,
+      "step": 760
+    },
+    {
+      "epoch": 1.3663453111305872,
+      "grad_norm": 0.22070615293553159,
+      "learning_rate": 0.00012305810768298812,
+      "loss": 0.37275114059448244,
+      "step": 780
+    },
+    {
+      "epoch": 1.4014022787028921,
+      "grad_norm": 0.2649900027960825,
+      "learning_rate": 0.00011928409125113017,
+      "loss": 0.35739314556121826,
+      "step": 800
+    },
+    {
+      "epoch": 1.4364592462751973,
+      "grad_norm": 0.2407480584227103,
+      "learning_rate": 0.00011548131872930703,
+      "loss": 0.37084987163543703,
+      "step": 820
+    },
+    {
+      "epoch": 1.4715162138475022,
+      "grad_norm": 0.270524193933959,
+      "learning_rate": 0.0001116554607434938,
+      "loss": 0.37381908893585203,
+      "step": 840
+    },
+    {
+      "epoch": 1.5065731814198071,
+      "grad_norm": 0.22614419313290746,
+      "learning_rate": 0.0001078122223442942,
+      "loss": 0.3795423746109009,
+      "step": 860
+    },
+    {
+      "epoch": 1.541630148992112,
+      "grad_norm": 0.19336423988589488,
+      "learning_rate": 0.00010395733449967172,
+      "loss": 0.37464859485626223,
+      "step": 880
+    },
+    {
+      "epoch": 1.5766871165644172,
+      "grad_norm": 0.24670417800510697,
+      "learning_rate": 0.00010009654554903323,
+      "loss": 0.35675725936889646,
+      "step": 900
+    },
+    {
+      "epoch": 1.6117440841367223,
+      "grad_norm": 0.2734530829324958,
+      "learning_rate": 9.62356126314088e-05,
+      "loss": 0.37122507095336915,
+      "step": 920
+    },
+    {
+      "epoch": 1.6468010517090272,
+      "grad_norm": 0.2503106764897381,
+      "learning_rate": 9.238029310050955e-05,
+      "loss": 0.3705612659454346,
+      "step": 940
+    },
+    {
+      "epoch": 1.6818580192813322,
+      "grad_norm": 0.2989966813336377,
+      "learning_rate": 8.853633593946508e-05,
+      "loss": 0.38475995063781737,
+      "step": 960
+    },
+    {
+      "epoch": 1.716914986853637,
+      "grad_norm": 0.25534347655623346,
+      "learning_rate": 8.47094731880437e-05,
+      "loss": 0.3533210039138794,
+      "step": 980
+    },
+    {
+      "epoch": 1.751971954425942,
+      "grad_norm": 0.275257264717727,
+      "learning_rate": 8.090541139513744e-05,
+      "loss": 0.36080729961395264,
+      "step": 1000
+    },
+    {
+      "epoch": 1.7870289219982471,
+      "grad_norm": 0.27227637979008384,
+      "learning_rate": 7.712982310925923e-05,
+      "loss": 0.3737907886505127,
+      "step": 1020
+    },
+    {
+      "epoch": 1.8220858895705523,
+      "grad_norm": 0.21946775823047843,
+      "learning_rate": 7.33883384197406e-05,
+      "loss": 0.3572577953338623,
+      "step": 1040
+    },
+    {
+      "epoch": 1.8571428571428572,
+      "grad_norm": 0.1971440107080404,
+      "learning_rate": 6.96865365612436e-05,
+      "loss": 0.3543445587158203,
+      "step": 1060
+    },
+    {
+      "epoch": 1.8921998247151621,
+      "grad_norm": 0.2459756703168472,
+      "learning_rate": 6.602993759410652e-05,
+      "loss": 0.3383913040161133,
+      "step": 1080
+    },
+    {
+      "epoch": 1.927256792287467,
+      "grad_norm": 0.2208046754121449,
+      "learning_rate": 6.242399417292937e-05,
+      "loss": 0.3462701320648193,
+      "step": 1100
+    },
+    {
+      "epoch": 1.962313759859772,
+      "grad_norm": 0.21368660298153802,
+      "learning_rate": 5.887408341567306e-05,
+      "loss": 0.33631114959716796,
+      "step": 1120
+    },
+    {
+      "epoch": 1.997370727432077,
+      "grad_norm": 0.19930875245651597,
+      "learning_rate": 5.538549888539829e-05,
+      "loss": 0.3454852342605591,
+      "step": 1140
+    },
+    {
+      "epoch": 2.0315512708150747,
+      "grad_norm": 0.2748162634596323,
+      "learning_rate": 5.1963442696599096e-05,
+      "loss": 0.23102645874023436,
+      "step": 1160
+    },
+    {
+      "epoch": 2.0666082383873796,
+      "grad_norm": 0.21873172200282442,
+      "learning_rate": 4.861301775790361e-05,
+      "loss": 0.19582982063293458,
+      "step": 1180
+    },
+    {
+      "epoch": 2.1016652059596845,
+      "grad_norm": 0.2738253546879238,
+      "learning_rate": 4.5339220162708506e-05,
+      "loss": 0.18127157688140869,
+      "step": 1200
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 1713,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 855440715939840.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoints/Qwen3-32B-7fewshots-bs64-3eps-ckpt-1600/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "/mnt/msrh/Magic_submission/hub/Qwen3-32B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 256,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 128,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "v_proj",
+    "up_proj",
+    "gate_proj",
+    "down_proj",
+    "q_proj",
+    "k_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoints/Qwen3-32B-7fewshots-bs64-3eps-ckpt-1600/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e5230b5ca3c7dd4881a0cf4dc485df11b98d649f6e2e672c98141e252867e5eb
+size 2147607752

checkpoints/Qwen3-32B-7fewshots-bs64-3eps-ckpt-1600/chat_template.jinja

@@ -0,0 +1,89 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}

checkpoints/Qwen3-32B-7fewshots-bs64-3eps-ckpt-1600/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650

checkpoints/Qwen3-32B-7fewshots-bs64-3eps-ckpt-1600/tokenizer_config.json

@@ -0,0 +1,30 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": true,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

checkpoints/Qwen3-32B-7fewshots-bs64-3eps-ckpt-1600/trainer_state.json

@@ -0,0 +1,594 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.8028045574057843,
+  "eval_steps": 500,
+  "global_step": 1600,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.035056967572304996,
+      "grad_norm": 0.23483171452352822,
+      "learning_rate": 4.418604651162791e-05,
+      "loss": 0.9909378051757812,
+      "step": 20
+    },
+    {
+      "epoch": 0.07011393514460999,
+      "grad_norm": 0.19986055220147195,
+      "learning_rate": 9.069767441860465e-05,
+      "loss": 0.7854896068572998,
+      "step": 40
+    },
+    {
+      "epoch": 0.10517090271691498,
+      "grad_norm": 0.23130651455988926,
+      "learning_rate": 0.0001372093023255814,
+      "loss": 0.7465881824493408,
+      "step": 60
+    },
+    {
+      "epoch": 0.14022787028921999,
+      "grad_norm": 0.3197363225238927,
+      "learning_rate": 0.00018372093023255815,
+      "loss": 0.7164266586303711,
+      "step": 80
+    },
+    {
+      "epoch": 0.175284837861525,
+      "grad_norm": 0.2715619931198569,
+      "learning_rate": 0.00019996849651899697,
+      "loss": 0.6907209873199462,
+      "step": 100
+    },
+    {
+      "epoch": 0.21034180543382996,
+      "grad_norm": 0.2856851996713809,
+      "learning_rate": 0.00019979705630030178,
+      "loss": 0.6624889373779297,
+      "step": 120
+    },
+    {
+      "epoch": 0.24539877300613497,
+      "grad_norm": 0.2621092411083009,
+      "learning_rate": 0.00019947680050653714,
+      "loss": 0.642543363571167,
+      "step": 140
+    },
+    {
+      "epoch": 0.28045574057843997,
+      "grad_norm": 0.3060417273271011,
+      "learning_rate": 0.00019900820669738146,
+      "loss": 0.646620512008667,
+      "step": 160
+    },
+    {
+      "epoch": 0.31551270815074495,
+      "grad_norm": 0.21452369072192615,
+      "learning_rate": 0.0001983919736314933,
+      "loss": 0.6074985504150391,
+      "step": 180
+    },
+    {
+      "epoch": 0.35056967572305,
+      "grad_norm": 0.30612438725480096,
+      "learning_rate": 0.00019762902022453483,
+      "loss": 0.5916189193725586,
+      "step": 200
+    },
+    {
+      "epoch": 0.38562664329535495,
+      "grad_norm": 0.2672865616168627,
+      "learning_rate": 0.00019672048417890148,
+      "loss": 0.6512822151184082,
+      "step": 220
+    },
+    {
+      "epoch": 0.42068361086765993,
+      "grad_norm": 0.24914148404479688,
+      "learning_rate": 0.00019566772028720047,
+      "loss": 0.587531566619873,
+      "step": 240
+    },
+    {
+      "epoch": 0.45574057843996496,
+      "grad_norm": 0.24657546374303219,
+      "learning_rate": 0.0001944722984120086,
+      "loss": 0.6211313724517822,
+      "step": 260
+    },
+    {
+      "epoch": 0.49079754601226994,
+      "grad_norm": 0.2271230878176654,
+      "learning_rate": 0.0001931360011449213,
+      "loss": 0.5668744087219239,
+      "step": 280
+    },
+    {
+      "epoch": 0.5258545135845749,
+      "grad_norm": 0.25583543263520286,
+      "learning_rate": 0.00019166082114838404,
+      "loss": 0.5299773216247559,
+      "step": 300
+    },
+    {
+      "epoch": 0.5609114811568799,
+      "grad_norm": 0.26046386701317537,
+      "learning_rate": 0.00019004895818427016,
+      "loss": 0.5769920349121094,
+      "step": 320
+    },
+    {
+      "epoch": 0.595968448729185,
+      "grad_norm": 0.2698990350624982,
+      "learning_rate": 0.00018830281583363535,
+      "loss": 0.5816852569580078,
+      "step": 340
+    },
+    {
+      "epoch": 0.6310254163014899,
+      "grad_norm": 0.26049160579851316,
+      "learning_rate": 0.00018642499791254085,
+      "loss": 0.5477001190185546,
+      "step": 360
+    },
+    {
+      "epoch": 0.6660823838737949,
+      "grad_norm": 0.1708923438980022,
+      "learning_rate": 0.0001844183045892899,
+      "loss": 0.5549935340881348,
+      "step": 380
+    },
+    {
+      "epoch": 0.7011393514461,
+      "grad_norm": 0.18290090201506068,
+      "learning_rate": 0.00018228572820886695,
+      "loss": 0.5305068492889404,
+      "step": 400
+    },
+    {
+      "epoch": 0.7361963190184049,
+      "grad_norm": 0.2120975286999116,
+      "learning_rate": 0.00018003044883080685,
+      "loss": 0.5133780479431153,
+      "step": 420
+    },
+    {
+      "epoch": 0.7712532865907099,
+      "grad_norm": 0.22392071288652587,
+      "learning_rate": 0.00017765582948714708,
+      "loss": 0.5189687728881835,
+      "step": 440
+    },
+    {
+      "epoch": 0.8063102541630149,
+      "grad_norm": 0.17532736599040075,
+      "learning_rate": 0.00017516541116753502,
+      "loss": 0.5183590888977051,
+      "step": 460
+    },
+    {
+      "epoch": 0.8413672217353199,
+      "grad_norm": 0.24787663667303497,
+      "learning_rate": 0.00017256290753896782,
+      "loss": 0.5011738300323486,
+      "step": 480
+    },
+    {
+      "epoch": 0.8764241893076249,
+      "grad_norm": 0.1730628728128248,
+      "learning_rate": 0.00016985219940803906,
+      "loss": 0.48698010444641116,
+      "step": 500
+    },
+    {
+      "epoch": 0.9114811568799299,
+      "grad_norm": 0.19806277891741542,
+      "learning_rate": 0.00016703732893394964,
+      "loss": 0.4896900177001953,
+      "step": 520
+    },
+    {
+      "epoch": 0.9465381244522348,
+      "grad_norm": 0.24968865256753012,
+      "learning_rate": 0.00016412249360091277,
+      "loss": 0.48049116134643555,
+      "step": 540
+    },
+    {
+      "epoch": 0.9815950920245399,
+      "grad_norm": 0.21341584083366677,
+      "learning_rate": 0.00016111203995894112,
+      "loss": 0.47271203994750977,
+      "step": 560
+    },
+    {
+      "epoch": 1.0157756354075373,
+      "grad_norm": 0.2811762357022581,
+      "learning_rate": 0.00015801045714234938,
+      "loss": 0.45186548233032225,
+      "step": 580
+    },
+    {
+      "epoch": 1.0508326029798423,
+      "grad_norm": 0.2624449978453146,
+      "learning_rate": 0.00015482237017563812,
+      "loss": 0.38692054748535154,
+      "step": 600
+    },
+    {
+      "epoch": 1.0858895705521472,
+      "grad_norm": 0.21413306480701308,
+      "learning_rate": 0.00015155253307674055,
+      "loss": 0.3919252634048462,
+      "step": 620
+    },
+    {
+      "epoch": 1.1209465381244523,
+      "grad_norm": 0.2683586780002,
+      "learning_rate": 0.00014820582176791621,
+      "loss": 0.3797740936279297,
+      "step": 640
+    },
+    {
+      "epoch": 1.1560035056967572,
+      "grad_norm": 0.24565139075016496,
+      "learning_rate": 0.00014478722680486358,
+      "loss": 0.37922732830047606,
+      "step": 660
+    },
+    {
+      "epoch": 1.1910604732690622,
+      "grad_norm": 0.19742886528967668,
+      "learning_rate": 0.00014130184593489327,
+      "loss": 0.36946282386779783,
+      "step": 680
+    },
+    {
+      "epoch": 1.2261174408413673,
+      "grad_norm": 0.20685035595170095,
+      "learning_rate": 0.00013775487649525872,
+      "loss": 0.357561731338501,
+      "step": 700
+    },
+    {
+      "epoch": 1.2611744084136722,
+      "grad_norm": 0.24705150218547564,
+      "learning_rate": 0.0001341516076629807,
+      "loss": 0.39932737350463865,
+      "step": 720
+    },
+    {
+      "epoch": 1.2962313759859772,
+      "grad_norm": 0.2683288997975443,
+      "learning_rate": 0.00013049741256772129,
+      "loss": 0.39545512199401855,
+      "step": 740
+    },
+    {
+      "epoch": 1.331288343558282,
+      "grad_norm": 0.22560628282285475,
+      "learning_rate": 0.00012679774027946997,
+      "loss": 0.39340255260467527,
+      "step": 760
+    },
+    {
+      "epoch": 1.3663453111305872,
+      "grad_norm": 0.22070615293553159,
+      "learning_rate": 0.00012305810768298812,
+      "loss": 0.37275114059448244,
+      "step": 780
+    },
+    {
+      "epoch": 1.4014022787028921,
+      "grad_norm": 0.2649900027960825,
+      "learning_rate": 0.00011928409125113017,
+      "loss": 0.35739314556121826,
+      "step": 800
+    },
+    {
+      "epoch": 1.4364592462751973,
+      "grad_norm": 0.2407480584227103,
+      "learning_rate": 0.00011548131872930703,
+      "loss": 0.37084987163543703,
+      "step": 820
+    },
+    {
+      "epoch": 1.4715162138475022,
+      "grad_norm": 0.270524193933959,
+      "learning_rate": 0.0001116554607434938,
+      "loss": 0.37381908893585203,
+      "step": 840
+    },
+    {
+      "epoch": 1.5065731814198071,
+      "grad_norm": 0.22614419313290746,
+      "learning_rate": 0.0001078122223442942,
+      "loss": 0.3795423746109009,
+      "step": 860
+    },
+    {
+      "epoch": 1.541630148992112,
+      "grad_norm": 0.19336423988589488,
+      "learning_rate": 0.00010395733449967172,
+      "loss": 0.37464859485626223,
+      "step": 880
+    },
+    {
+      "epoch": 1.5766871165644172,
+      "grad_norm": 0.24670417800510697,
+      "learning_rate": 0.00010009654554903323,
+      "loss": 0.35675725936889646,
+      "step": 900
+    },
+    {
+      "epoch": 1.6117440841367223,
+      "grad_norm": 0.2734530829324958,
+      "learning_rate": 9.62356126314088e-05,
+      "loss": 0.37122507095336915,
+      "step": 920
+    },
+    {
+      "epoch": 1.6468010517090272,
+      "grad_norm": 0.2503106764897381,
+      "learning_rate": 9.238029310050955e-05,
+      "loss": 0.3705612659454346,
+      "step": 940
+    },
+    {
+      "epoch": 1.6818580192813322,
+      "grad_norm": 0.2989966813336377,
+      "learning_rate": 8.853633593946508e-05,
+      "loss": 0.38475995063781737,
+      "step": 960
+    },
+    {
+      "epoch": 1.716914986853637,
+      "grad_norm": 0.25534347655623346,
+      "learning_rate": 8.47094731880437e-05,
+      "loss": 0.3533210039138794,
+      "step": 980
+    },
+    {
+      "epoch": 1.751971954425942,
+      "grad_norm": 0.275257264717727,
+      "learning_rate": 8.090541139513744e-05,
+      "loss": 0.36080729961395264,
+      "step": 1000
+    },
+    {
+      "epoch": 1.7870289219982471,
+      "grad_norm": 0.27227637979008384,
+      "learning_rate": 7.712982310925923e-05,
+      "loss": 0.3737907886505127,
+      "step": 1020
+    },
+    {
+      "epoch": 1.8220858895705523,
+      "grad_norm": 0.21946775823047843,
+      "learning_rate": 7.33883384197406e-05,
+      "loss": 0.3572577953338623,
+      "step": 1040
+    },
+    {
+      "epoch": 1.8571428571428572,
+      "grad_norm": 0.1971440107080404,
+      "learning_rate": 6.96865365612436e-05,
+      "loss": 0.3543445587158203,
+      "step": 1060
+    },
+    {
+      "epoch": 1.8921998247151621,
+      "grad_norm": 0.2459756703168472,
+      "learning_rate": 6.602993759410652e-05,
+      "loss": 0.3383913040161133,
+      "step": 1080
+    },
+    {
+      "epoch": 1.927256792287467,
+      "grad_norm": 0.2208046754121449,
+      "learning_rate": 6.242399417292937e-05,
+      "loss": 0.3462701320648193,
+      "step": 1100
+    },
+    {
+      "epoch": 1.962313759859772,
+      "grad_norm": 0.21368660298153802,
+      "learning_rate": 5.887408341567306e-05,
+      "loss": 0.33631114959716796,
+      "step": 1120
+    },
+    {
+      "epoch": 1.997370727432077,
+      "grad_norm": 0.19930875245651597,
+      "learning_rate": 5.538549888539829e-05,
+      "loss": 0.3454852342605591,
+      "step": 1140
+    },
+    {
+      "epoch": 2.0315512708150747,
+      "grad_norm": 0.2748162634596323,
+      "learning_rate": 5.1963442696599096e-05,
+      "loss": 0.23102645874023436,
+      "step": 1160
+    },
+    {
+      "epoch": 2.0666082383873796,
+      "grad_norm": 0.21873172200282442,
+      "learning_rate": 4.861301775790361e-05,
+      "loss": 0.19582982063293458,
+      "step": 1180
+    },
+    {
+      "epoch": 2.1016652059596845,
+      "grad_norm": 0.2738253546879238,
+      "learning_rate": 4.5339220162708506e-05,
+      "loss": 0.18127157688140869,
+      "step": 1200
+    },
+    {
+      "epoch": 2.1367221735319895,
+      "grad_norm": 0.27023723639002367,
+      "learning_rate": 4.214693173909409e-05,
+      "loss": 0.19596079587936402,
+      "step": 1220
+    },
+    {
+      "epoch": 2.1717791411042944,
+      "grad_norm": 0.2549604833170729,
+      "learning_rate": 3.9040912770130454e-05,
+      "loss": 0.1777873754501343,
+      "step": 1240
+    },
+    {
+      "epoch": 2.2068361086765993,
+      "grad_norm": 0.21441475116085365,
+      "learning_rate": 3.602579489542883e-05,
+      "loss": 0.19610731601715087,
+      "step": 1260
+    },
+    {
+      "epoch": 2.2418930762489047,
+      "grad_norm": 0.2673235263264258,
+      "learning_rate": 3.3106074204523905e-05,
+      "loss": 0.17948269844055176,
+      "step": 1280
+    },
+    {
+      "epoch": 2.2769500438212096,
+      "grad_norm": 0.29521194033081255,
+      "learning_rate": 3.028610453238604e-05,
+      "loss": 0.16286152601242065,
+      "step": 1300
+    },
+    {
+      "epoch": 2.3120070113935145,
+      "grad_norm": 0.2779938716732372,
+      "learning_rate": 2.7570090967060868e-05,
+      "loss": 0.17882025241851807,
+      "step": 1320
+    },
+    {
+      "epoch": 2.3470639789658194,
+      "grad_norm": 0.338290787393958,
+      "learning_rate": 2.4962083579117656e-05,
+      "loss": 0.19410847425460814,
+      "step": 1340
+    },
+    {
+      "epoch": 2.3821209465381243,
+      "grad_norm": 0.23478644005908705,
+      "learning_rate": 2.246597138225691e-05,
+      "loss": 0.17010506391525268,
+      "step": 1360
+    },
+    {
+      "epoch": 2.4171779141104293,
+      "grad_norm": 0.23666992110532725,
+      "learning_rate": 2.0085476534083103e-05,
+      "loss": 0.19948906898498536,
+      "step": 1380
+    },
+    {
+      "epoch": 2.4522348816827346,
+      "grad_norm": 0.2717904238341419,
+      "learning_rate": 1.7824148785690288e-05,
+      "loss": 0.18881454467773437,
+      "step": 1400
+    },
+    {
+      "epoch": 2.4872918492550395,
+      "grad_norm": 0.25307966701552526,
+      "learning_rate": 1.568536018833694e-05,
+      "loss": 0.180185866355896,
+      "step": 1420
+    },
+    {
+      "epoch": 2.5223488168273445,
+      "grad_norm": 0.2573861037469995,
+      "learning_rate": 1.367230006510406e-05,
+      "loss": 0.1824875831604004,
+      "step": 1440
+    },
+    {
+      "epoch": 2.5574057843996494,
+      "grad_norm": 0.27731812480850276,
+      "learning_rate": 1.178797025503373e-05,
+      "loss": 0.17870738506317138,
+      "step": 1460
+    },
+    {
+      "epoch": 2.5924627519719543,
+      "grad_norm": 0.30734934066822084,
+      "learning_rate": 1.003518063684079e-05,
+      "loss": 0.18122551441192628,
+      "step": 1480
+    },
+    {
+      "epoch": 2.6275197195442592,
+      "grad_norm": 0.2646400918278933,
+      "learning_rate": 8.416544938872385e-06,
+      "loss": 0.16032501459121704,
+      "step": 1500
+    },
+    {
+      "epoch": 2.662576687116564,
+      "grad_norm": 0.29864309575164133,
+      "learning_rate": 6.934476841563054e-06,
+      "loss": 0.19240641593933105,
+      "step": 1520
+    },
+    {
+      "epoch": 2.6976336546888695,
+      "grad_norm": 0.21792778456861328,
+      "learning_rate": 5.591186378198032e-06,
+      "loss": 0.1665952205657959,
+      "step": 1540
+    },
+    {
+      "epoch": 2.7326906222611744,
+      "grad_norm": 0.25072518876392397,
+      "learning_rate": 4.388676639351396e-06,
+      "loss": 0.17795501947402953,
+      "step": 1560
+    },
+    {
+      "epoch": 2.7677475898334793,
+      "grad_norm": 0.3210307973248795,
+      "learning_rate": 3.328740785913298e-06,
+      "loss": 0.18478138446807862,
+      "step": 1580
+    },
+    {
+      "epoch": 2.8028045574057843,
+      "grad_norm": 0.2790969981011738,
+      "learning_rate": 2.412959375160806e-06,
+      "loss": 0.17509810924530028,
+      "step": 1600
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 1713,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1140521146777600.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoints/Qwen3-32B-NoFewshots-bs64-4eps-ckpt-6500/README.md

@@ -0,0 +1,208 @@
+---
+base_model: unsloth/Qwen3-32B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:unsloth/Qwen3-32B
+- llama-factory
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.19.1

checkpoints/Qwen3-32B-NoFewshots-bs64-4eps-ckpt-6500/adapter_config.json

@@ -0,0 +1,48 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "/mnt/msrh/Magic_submission/hub/Qwen3-32B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 256,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.19.1",
+  "qalora_group_size": 16,
+  "r": 128,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "k_proj",
+    "gate_proj",
+    "v_proj",
+    "down_proj",
+    "o_proj",
+    "up_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoints/Qwen3-32B-NoFewshots-bs64-4eps-ckpt-6500/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:016ad42fdea946023a75d2afdf7847fe6f9b09ae09191a16ee54020a0f246591
+size 2147607752

checkpoints/Qwen3-32B-NoFewshots-bs64-4eps-ckpt-6500/chat_template.jinja

@@ -0,0 +1,98 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for forward_message in messages %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- set message = messages[index] %}
+    {%- set current_content = message.content if message.content is not none else '' %}
+    {%- set tool_start = '<tool_response>' %}
+    {%- set tool_start_length = tool_start|length %}
+    {%- set start_of_message = current_content[:tool_start_length] %}
+    {%- set tool_end = '</tool_response>' %}
+    {%- set tool_end_length = tool_end|length %}
+    {%- set start_pos = (current_content|length) - tool_end_length %}
+    {%- if start_pos < 0 %}
+        {%- set start_pos = 0 %}
+    {%- endif %}
+    {%- set end_of_message = current_content[start_pos:] %}
+    {%- if ns.multi_step_tool and message.role == "user" and not(start_of_message == tool_start and end_of_message == tool_end) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set content = message.content %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is defined and message.reasoning_content is not none %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in message.content %}
+                {%- set content = (message.content.split('</think>')|last).lstrip('\n') %}
+                {%- set reasoning_content = (message.content.split('</think>')|first).rstrip('\n') %}
+                {%- set reasoning_content = (reasoning_content.split('<think>')|last).lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}

checkpoints/Qwen3-32B-NoFewshots-bs64-4eps-ckpt-6500/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650

checkpoints/Qwen3-32B-NoFewshots-bs64-4eps-ckpt-6500/tokenizer_config.json

@@ -0,0 +1,16 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 40960,
+  "pad_token": "<|vision_pad|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

checkpoints/Qwen3-32B-NoFewshots-bs64-4eps-ckpt-6500/trainer_state.json

@@ -0,0 +1,1854 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.8485645408722333,
+  "eval_steps": 500,
+  "global_step": 6500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.010957703265395573,
+      "grad_norm": 1.2978552097196294,
+      "learning_rate": 1.050328227571116e-05,
+      "loss": 1.3814047241210938,
+      "step": 25
+    },
+    {
+      "epoch": 0.021915406530791146,
+      "grad_norm": 0.7136933147809937,
+      "learning_rate": 2.1444201312910288e-05,
+      "loss": 1.1958548736572265,
+      "step": 50
+    },
+    {
+      "epoch": 0.03287310979618672,
+      "grad_norm": 0.6170623006014918,
+      "learning_rate": 3.238512035010941e-05,
+      "loss": 1.1010726928710937,
+      "step": 75
+    },
+    {
+      "epoch": 0.04383081306158229,
+      "grad_norm": 0.6609283433159858,
+      "learning_rate": 4.332603938730854e-05,
+      "loss": 1.0145082092285156,
+      "step": 100
+    },
+    {
+      "epoch": 0.054788516326977864,
+      "grad_norm": 0.783477976467315,
+      "learning_rate": 5.4266958424507664e-05,
+      "loss": 0.9656268310546875,
+      "step": 125
+    },
+    {
+      "epoch": 0.06574621959237344,
+      "grad_norm": 0.5380538102721405,
+      "learning_rate": 6.520787746170678e-05,
+      "loss": 0.9432225036621094,
+      "step": 150
+    },
+    {
+      "epoch": 0.07670392285776902,
+      "grad_norm": 0.5539337260003312,
+      "learning_rate": 7.614879649890591e-05,
+      "loss": 0.8418754577636719,
+      "step": 175
+    },
+    {
+      "epoch": 0.08766162612316458,
+      "grad_norm": 0.6054899600906472,
+      "learning_rate": 8.708971553610504e-05,
+      "loss": 0.9252291870117187,
+      "step": 200
+    },
+    {
+      "epoch": 0.09861932938856016,
+      "grad_norm": 0.6720454608055798,
+      "learning_rate": 9.803063457330416e-05,
+      "loss": 0.8630228424072266,
+      "step": 225
+    },
+    {
+      "epoch": 0.10957703265395573,
+      "grad_norm": 0.7750542553973204,
+      "learning_rate": 0.00010897155361050329,
+      "loss": 0.8796463012695312,
+      "step": 250
+    },
+    {
+      "epoch": 0.12053473591935131,
+      "grad_norm": 0.6871207112973714,
+      "learning_rate": 0.00011991247264770243,
+      "loss": 0.8309561157226563,
+      "step": 275
+    },
+    {
+      "epoch": 0.13149243918474687,
+      "grad_norm": 0.615645832188525,
+      "learning_rate": 0.00013085339168490153,
+      "loss": 0.8329324340820312,
+      "step": 300
+    },
+    {
+      "epoch": 0.14245014245014245,
+      "grad_norm": 0.4992434459601084,
+      "learning_rate": 0.00014179431072210066,
+      "loss": 0.8697052001953125,
+      "step": 325
+    },
+    {
+      "epoch": 0.15340784571553803,
+      "grad_norm": 0.5138330027601147,
+      "learning_rate": 0.00015273522975929978,
+      "loss": 0.7966581726074219,
+      "step": 350
+    },
+    {
+      "epoch": 0.16436554898093358,
+      "grad_norm": 0.5334733163812426,
+      "learning_rate": 0.0001636761487964989,
+      "loss": 0.791004638671875,
+      "step": 375
+    },
+    {
+      "epoch": 0.17532325224632916,
+      "grad_norm": 0.5241184516406538,
+      "learning_rate": 0.00017461706783369804,
+      "loss": 0.870574951171875,
+      "step": 400
+    },
+    {
+      "epoch": 0.18628095551172474,
+      "grad_norm": 0.5113413925855853,
+      "learning_rate": 0.00018555798687089716,
+      "loss": 0.7911457824707031,
+      "step": 425
+    },
+    {
+      "epoch": 0.19723865877712032,
+      "grad_norm": 0.7172256506710407,
+      "learning_rate": 0.0001964989059080963,
+      "loss": 0.8079358673095703,
+      "step": 450
+    },
+    {
+      "epoch": 0.20819636204251588,
+      "grad_norm": 0.6014604168590794,
+      "learning_rate": 0.0001999981031724577,
+      "loss": 0.8427057647705078,
+      "step": 475
+    },
+    {
+      "epoch": 0.21915406530791146,
+      "grad_norm": 0.676992513218163,
+      "learning_rate": 0.00019998842231904221,
+      "loss": 0.8817544555664063,
+      "step": 500
+    },
+    {
+      "epoch": 0.23011176857330704,
+      "grad_norm": 0.5180395697807777,
+      "learning_rate": 0.00019997053817426578,
+      "loss": 0.7796897888183594,
+      "step": 525
+    },
+    {
+      "epoch": 0.24106947183870261,
+      "grad_norm": 0.30769144895407374,
+      "learning_rate": 0.00019994445220538677,
+      "loss": 0.75110107421875,
+      "step": 550
+    },
+    {
+      "epoch": 0.25202717510409817,
+      "grad_norm": 0.6946306606035483,
+      "learning_rate": 0.000199910166552561,
+      "loss": 0.791603012084961,
+      "step": 575
+    },
+    {
+      "epoch": 0.26298487836949375,
+      "grad_norm": 0.6487575836141458,
+      "learning_rate": 0.0001998676840286661,
+      "loss": 0.8085326385498047,
+      "step": 600
+    },
+    {
+      "epoch": 0.2739425816348893,
+      "grad_norm": 0.5041253960929305,
+      "learning_rate": 0.00019981700811907082,
+      "loss": 0.7662266540527344,
+      "step": 625
+    },
+    {
+      "epoch": 0.2849002849002849,
+      "grad_norm": 0.5875852413952799,
+      "learning_rate": 0.00019975814298134906,
+      "loss": 0.7822332763671875,
+      "step": 650
+    },
+    {
+      "epoch": 0.2958579881656805,
+      "grad_norm": 0.3672633686477288,
+      "learning_rate": 0.00019969109344493866,
+      "loss": 0.7177228546142578,
+      "step": 675
+    },
+    {
+      "epoch": 0.30681569143107607,
+      "grad_norm": 0.3284350544286358,
+      "learning_rate": 0.00019961586501074537,
+      "loss": 0.6856938934326172,
+      "step": 700
+    },
+    {
+      "epoch": 0.31777339469647164,
+      "grad_norm": 0.5887357472274588,
+      "learning_rate": 0.00019953246385069138,
+      "loss": 0.729765625,
+      "step": 725
+    },
+    {
+      "epoch": 0.32873109796186717,
+      "grad_norm": 0.5339181934387838,
+      "learning_rate": 0.000199440896807209,
+      "loss": 0.6660807800292968,
+      "step": 750
+    },
+    {
+      "epoch": 0.33968880122726275,
+      "grad_norm": 0.4472635523322388,
+      "learning_rate": 0.00019934117139267944,
+      "loss": 0.7095761108398437,
+      "step": 775
+    },
+    {
+      "epoch": 0.35064650449265833,
+      "grad_norm": 0.647261631419608,
+      "learning_rate": 0.00019923329578881623,
+      "loss": 0.7111241912841797,
+      "step": 800
+    },
+    {
+      "epoch": 0.3616042077580539,
+      "grad_norm": 0.5322344853153645,
+      "learning_rate": 0.0001991172788459941,
+      "loss": 0.751135482788086,
+      "step": 825
+    },
+    {
+      "epoch": 0.3725619110234495,
+      "grad_norm": 0.4006392658135849,
+      "learning_rate": 0.00019899313008252297,
+      "loss": 0.8565126037597657,
+      "step": 850
+    },
+    {
+      "epoch": 0.38351961428884507,
+      "grad_norm": 0.4807677057285955,
+      "learning_rate": 0.0001988608596838668,
+      "loss": 0.6795193481445313,
+      "step": 875
+    },
+    {
+      "epoch": 0.39447731755424065,
+      "grad_norm": 0.7020250893855614,
+      "learning_rate": 0.00019872047850180819,
+      "loss": 0.7597425079345703,
+      "step": 900
+    },
+    {
+      "epoch": 0.4054350208196362,
+      "grad_norm": 0.45570623163372787,
+      "learning_rate": 0.00019857199805355794,
+      "loss": 0.6242760467529297,
+      "step": 925
+    },
+    {
+      "epoch": 0.41639272408503175,
+      "grad_norm": 0.5196355349949331,
+      "learning_rate": 0.00019841543052081018,
+      "loss": 0.7238587188720703,
+      "step": 950
+    },
+    {
+      "epoch": 0.42735042735042733,
+      "grad_norm": 0.5908561395788653,
+      "learning_rate": 0.000198250788748743,
+      "loss": 0.75541748046875,
+      "step": 975
+    },
+    {
+      "epoch": 0.4383081306158229,
+      "grad_norm": 0.537095758875929,
+      "learning_rate": 0.00019807808624496448,
+      "loss": 0.7320578002929687,
+      "step": 1000
+    },
+    {
+      "epoch": 0.4492658338812185,
+      "grad_norm": 0.5297475648151708,
+      "learning_rate": 0.0001978973371784047,
+      "loss": 0.6988375854492187,
+      "step": 1025
+    },
+    {
+      "epoch": 0.46022353714661407,
+      "grad_norm": 0.5141575664980547,
+      "learning_rate": 0.00019770855637815307,
+      "loss": 0.6565898132324218,
+      "step": 1050
+    },
+    {
+      "epoch": 0.47118124041200965,
+      "grad_norm": 0.4631625004938986,
+      "learning_rate": 0.00019751175933224176,
+      "loss": 0.7007794952392579,
+      "step": 1075
+    },
+    {
+      "epoch": 0.48213894367740523,
+      "grad_norm": 0.6216674266285784,
+      "learning_rate": 0.00019730696218637518,
+      "loss": 0.6264320755004883,
+      "step": 1100
+    },
+    {
+      "epoch": 0.4930966469428008,
+      "grad_norm": 0.6332346570330109,
+      "learning_rate": 0.0001970941817426052,
+      "loss": 0.744112777709961,
+      "step": 1125
+    },
+    {
+      "epoch": 0.5040543502081963,
+      "grad_norm": 0.5981225065113883,
+      "learning_rate": 0.0001968734354579527,
+      "loss": 0.6386201858520508,
+      "step": 1150
+    },
+    {
+      "epoch": 0.5150120534735919,
+      "grad_norm": 0.558032978032838,
+      "learning_rate": 0.00019664474144297534,
+      "loss": 0.5493645477294922,
+      "step": 1175
+    },
+    {
+      "epoch": 0.5259697567389875,
+      "grad_norm": 0.39313732450997557,
+      "learning_rate": 0.00019640811846028178,
+      "loss": 0.6679231262207032,
+      "step": 1200
+    },
+    {
+      "epoch": 0.5369274600043831,
+      "grad_norm": 0.4119452624600624,
+      "learning_rate": 0.00019616358592299232,
+      "loss": 0.6557364654541016,
+      "step": 1225
+    },
+    {
+      "epoch": 0.5478851632697787,
+      "grad_norm": 0.6414670142686668,
+      "learning_rate": 0.00019591116389314615,
+      "loss": 0.6710714721679687,
+      "step": 1250
+    },
+    {
+      "epoch": 0.5588428665351742,
+      "grad_norm": 0.5975703356000196,
+      "learning_rate": 0.00019565087308005553,
+      "loss": 0.6270205688476562,
+      "step": 1275
+    },
+    {
+      "epoch": 0.5698005698005698,
+      "grad_norm": 0.41235442325426047,
+      "learning_rate": 0.0001953827348386066,
+      "loss": 0.7304449462890625,
+      "step": 1300
+    },
+    {
+      "epoch": 0.5807582730659654,
+      "grad_norm": 0.5275683310744128,
+      "learning_rate": 0.00019510677116750755,
+      "loss": 0.6244300079345703,
+      "step": 1325
+    },
+    {
+      "epoch": 0.591715976331361,
+      "grad_norm": 0.39351578691496925,
+      "learning_rate": 0.00019482300470748362,
+      "loss": 0.6354802322387695,
+      "step": 1350
+    },
+    {
+      "epoch": 0.6026736795967566,
+      "grad_norm": 0.5434189297758576,
+      "learning_rate": 0.00019453145873941973,
+      "loss": 0.6688744354248047,
+      "step": 1375
+    },
+    {
+      "epoch": 0.6136313828621521,
+      "grad_norm": 0.3141344348607065,
+      "learning_rate": 0.00019423215718245026,
+      "loss": 0.6204352188110351,
+      "step": 1400
+    },
+    {
+      "epoch": 0.6245890861275477,
+      "grad_norm": 0.5030832686997331,
+      "learning_rate": 0.00019392512459199705,
+      "loss": 0.6411018371582031,
+      "step": 1425
+    },
+    {
+      "epoch": 0.6355467893929433,
+      "grad_norm": 0.4634768793832465,
+      "learning_rate": 0.0001936103861577544,
+      "loss": 0.6204791641235352,
+      "step": 1450
+    },
+    {
+      "epoch": 0.6465044926583388,
+      "grad_norm": 0.9757986404500928,
+      "learning_rate": 0.00019328796770162269,
+      "loss": 0.6713768768310547,
+      "step": 1475
+    },
+    {
+      "epoch": 0.6574621959237343,
+      "grad_norm": 0.33258831559287305,
+      "learning_rate": 0.00019295789567558974,
+      "loss": 0.6151275634765625,
+      "step": 1500
+    },
+    {
+      "epoch": 0.6684198991891299,
+      "grad_norm": 0.4044670699191439,
+      "learning_rate": 0.00019262019715956088,
+      "loss": 0.6552862548828124,
+      "step": 1525
+    },
+    {
+      "epoch": 0.6793776024545255,
+      "grad_norm": 0.5681492231996502,
+      "learning_rate": 0.00019227489985913687,
+      "loss": 0.5921820831298829,
+      "step": 1550
+    },
+    {
+      "epoch": 0.6903353057199211,
+      "grad_norm": 0.4343026202194038,
+      "learning_rate": 0.00019192203210334122,
+      "loss": 0.6558747863769532,
+      "step": 1575
+    },
+    {
+      "epoch": 0.7012930089853167,
+      "grad_norm": 0.45481447167039024,
+      "learning_rate": 0.0001915616228422958,
+      "loss": 0.6270633697509765,
+      "step": 1600
+    },
+    {
+      "epoch": 0.7122507122507122,
+      "grad_norm": 0.47237387095063543,
+      "learning_rate": 0.00019119370164484578,
+      "loss": 0.6521648406982422,
+      "step": 1625
+    },
+    {
+      "epoch": 0.7232084155161078,
+      "grad_norm": 0.4651570244883477,
+      "learning_rate": 0.0001908182986961337,
+      "loss": 0.5692562866210937,
+      "step": 1650
+    },
+    {
+      "epoch": 0.7341661187815034,
+      "grad_norm": 0.5779163868208466,
+      "learning_rate": 0.00019043544479512302,
+      "loss": 0.5608213424682618,
+      "step": 1675
+    },
+    {
+      "epoch": 0.745123822046899,
+      "grad_norm": 0.48312154624427917,
+      "learning_rate": 0.00019004517135207127,
+      "loss": 0.6319043731689453,
+      "step": 1700
+    },
+    {
+      "epoch": 0.7560815253122946,
+      "grad_norm": 0.6169038093687549,
+      "learning_rate": 0.00018964751038595324,
+      "loss": 0.5678458786010743,
+      "step": 1725
+    },
+    {
+      "epoch": 0.7670392285776901,
+      "grad_norm": 0.46474648979398947,
+      "learning_rate": 0.00018924249452183372,
+      "loss": 0.605274772644043,
+      "step": 1750
+    },
+    {
+      "epoch": 0.7779969318430857,
+      "grad_norm": 0.6934437347685344,
+      "learning_rate": 0.0001888301569881913,
+      "loss": 0.5229089736938477,
+      "step": 1775
+    },
+    {
+      "epoch": 0.7889546351084813,
+      "grad_norm": 0.48717433781950337,
+      "learning_rate": 0.0001884105316141918,
+      "loss": 0.6208004760742187,
+      "step": 1800
+    },
+    {
+      "epoch": 0.7999123383738769,
+      "grad_norm": 0.4602133778756322,
+      "learning_rate": 0.00018798365282691318,
+      "loss": 0.6161478424072265,
+      "step": 1825
+    },
+    {
+      "epoch": 0.8108700416392725,
+      "grad_norm": 0.6487304684461032,
+      "learning_rate": 0.0001875495556485208,
+      "loss": 0.5896058654785157,
+      "step": 1850
+    },
+    {
+      "epoch": 0.821827744904668,
+      "grad_norm": 0.5309729149980255,
+      "learning_rate": 0.00018710827569339445,
+      "loss": 0.5441777420043945,
+      "step": 1875
+    },
+    {
+      "epoch": 0.8327854481700635,
+      "grad_norm": 0.3756485152004339,
+      "learning_rate": 0.00018665984916520597,
+      "loss": 0.5416929626464844,
+      "step": 1900
+    },
+    {
+      "epoch": 0.8437431514354591,
+      "grad_norm": 0.5496224680846356,
+      "learning_rate": 0.00018620431285394954,
+      "loss": 0.6156987762451172,
+      "step": 1925
+    },
+    {
+      "epoch": 0.8547008547008547,
+      "grad_norm": 0.4941794346195356,
+      "learning_rate": 0.00018574170413292293,
+      "loss": 0.591865348815918,
+      "step": 1950
+    },
+    {
+      "epoch": 0.8656585579662502,
+      "grad_norm": 0.6244829886543544,
+      "learning_rate": 0.00018527206095566158,
+      "loss": 0.5739054107666015,
+      "step": 1975
+    },
+    {
+      "epoch": 0.8766162612316458,
+      "grad_norm": 0.5337331049570524,
+      "learning_rate": 0.00018479542185282456,
+      "loss": 0.5181359100341797,
+      "step": 2000
+    },
+    {
+      "epoch": 0.8875739644970414,
+      "grad_norm": 0.5480705050808666,
+      "learning_rate": 0.00018431182592903365,
+      "loss": 0.5471242904663086,
+      "step": 2025
+    },
+    {
+      "epoch": 0.898531667762437,
+      "grad_norm": 0.5421739618077583,
+      "learning_rate": 0.000183821312859665,
+      "loss": 0.5618992614746093,
+      "step": 2050
+    },
+    {
+      "epoch": 0.9094893710278326,
+      "grad_norm": 0.3311633275942834,
+      "learning_rate": 0.00018332392288759403,
+      "loss": 0.6257458877563477,
+      "step": 2075
+    },
+    {
+      "epoch": 0.9204470742932281,
+      "grad_norm": 0.29974889200756905,
+      "learning_rate": 0.00018281969681989386,
+      "loss": 0.5611452484130859,
+      "step": 2100
+    },
+    {
+      "epoch": 0.9314047775586237,
+      "grad_norm": 0.9653908820248759,
+      "learning_rate": 0.00018230867602448742,
+      "loss": 0.5552873992919922,
+      "step": 2125
+    },
+    {
+      "epoch": 0.9423624808240193,
+      "grad_norm": 0.3238436719865102,
+      "learning_rate": 0.0001817909024267534,
+      "loss": 0.5431422424316407,
+      "step": 2150
+    },
+    {
+      "epoch": 0.9533201840894149,
+      "grad_norm": 0.601723849110238,
+      "learning_rate": 0.0001812664185060868,
+      "loss": 0.6057305526733399,
+      "step": 2175
+    },
+    {
+      "epoch": 0.9642778873548105,
+      "grad_norm": 0.47464675875116885,
+      "learning_rate": 0.00018073526729241358,
+      "loss": 0.5485016632080079,
+      "step": 2200
+    },
+    {
+      "epoch": 0.975235590620206,
+      "grad_norm": 0.6859751205226805,
+      "learning_rate": 0.00018019749236266065,
+      "loss": 0.5430702972412109,
+      "step": 2225
+    },
+    {
+      "epoch": 0.9861932938856016,
+      "grad_norm": 0.5407864887541908,
+      "learning_rate": 0.00017965313783718036,
+      "loss": 0.5579008483886718,
+      "step": 2250
+    },
+    {
+      "epoch": 0.9971509971509972,
+      "grad_norm": 0.5465960072870302,
+      "learning_rate": 0.00017910224837613118,
+      "loss": 0.5626018524169922,
+      "step": 2275
+    },
+    {
+      "epoch": 1.0078895463510849,
+      "grad_norm": 0.5251661713406559,
+      "learning_rate": 0.00017854486917581332,
+      "loss": 0.4892612075805664,
+      "step": 2300
+    },
+    {
+      "epoch": 1.0188472496164804,
+      "grad_norm": 0.6152523909810319,
+      "learning_rate": 0.0001779810459649609,
+      "loss": 0.42003105163574217,
+      "step": 2325
+    },
+    {
+      "epoch": 1.029804952881876,
+      "grad_norm": 0.6686959168385372,
+      "learning_rate": 0.0001774108250009902,
+      "loss": 0.41712055206298826,
+      "step": 2350
+    },
+    {
+      "epoch": 1.0407626561472716,
+      "grad_norm": 0.6084073217343031,
+      "learning_rate": 0.00017683425306620464,
+      "loss": 0.4078028106689453,
+      "step": 2375
+    },
+    {
+      "epoch": 1.0517203594126672,
+      "grad_norm": 0.4780975609098201,
+      "learning_rate": 0.0001762513774639565,
+      "loss": 0.4441735076904297,
+      "step": 2400
+    },
+    {
+      "epoch": 1.0626780626780628,
+      "grad_norm": 0.6010378983840258,
+      "learning_rate": 0.00017566224601476627,
+      "loss": 0.45469318389892577,
+      "step": 2425
+    },
+    {
+      "epoch": 1.0736357659434583,
+      "grad_norm": 0.6967390042692502,
+      "learning_rate": 0.00017506690705239913,
+      "loss": 0.4237989044189453,
+      "step": 2450
+    },
+    {
+      "epoch": 1.084593469208854,
+      "grad_norm": 0.39791866992468133,
+      "learning_rate": 0.0001744654094198996,
+      "loss": 0.41963016510009765,
+      "step": 2475
+    },
+    {
+      "epoch": 1.0955511724742495,
+      "grad_norm": 0.7232124845466396,
+      "learning_rate": 0.00017385780246558446,
+      "loss": 0.4468261337280273,
+      "step": 2500
+    },
+    {
+      "epoch": 1.106508875739645,
+      "grad_norm": 0.4443727169257166,
+      "learning_rate": 0.0001732441360389939,
+      "loss": 0.43564907073974607,
+      "step": 2525
+    },
+    {
+      "epoch": 1.1174665790050406,
+      "grad_norm": 0.4325837055491758,
+      "learning_rate": 0.00017262446048680184,
+      "loss": 0.42488311767578124,
+      "step": 2550
+    },
+    {
+      "epoch": 1.1284242822704362,
+      "grad_norm": 0.3980195564635358,
+      "learning_rate": 0.00017199882664868538,
+      "loss": 0.42963691711425783,
+      "step": 2575
+    },
+    {
+      "epoch": 1.1393819855358318,
+      "grad_norm": 0.6009465004570181,
+      "learning_rate": 0.0001713672858531538,
+      "loss": 0.4399003601074219,
+      "step": 2600
+    },
+    {
+      "epoch": 1.1503396888012272,
+      "grad_norm": 0.7477269407779865,
+      "learning_rate": 0.00017072988991333733,
+      "loss": 0.41332855224609377,
+      "step": 2625
+    },
+    {
+      "epoch": 1.1612973920666227,
+      "grad_norm": 0.47967935298933445,
+      "learning_rate": 0.00017008669112273642,
+      "loss": 0.40812835693359373,
+      "step": 2650
+    },
+    {
+      "epoch": 1.1722550953320183,
+      "grad_norm": 0.5365226964392461,
+      "learning_rate": 0.00016943774225093144,
+      "loss": 0.40122032165527344,
+      "step": 2675
+    },
+    {
+      "epoch": 1.183212798597414,
+      "grad_norm": 0.508159226031207,
+      "learning_rate": 0.00016878309653925314,
+      "loss": 0.4251136016845703,
+      "step": 2700
+    },
+    {
+      "epoch": 1.1941705018628095,
+      "grad_norm": 0.44432375746596137,
+      "learning_rate": 0.00016812280769641492,
+      "loss": 0.4017474365234375,
+      "step": 2725
+    },
+    {
+      "epoch": 1.205128205128205,
+      "grad_norm": 0.3713975277440623,
+      "learning_rate": 0.00016745692989410623,
+      "loss": 0.4365106201171875,
+      "step": 2750
+    },
+    {
+      "epoch": 1.2160859083936006,
+      "grad_norm": 0.5399569210290663,
+      "learning_rate": 0.0001667855177625482,
+      "loss": 0.38633155822753906,
+      "step": 2775
+    },
+    {
+      "epoch": 1.2270436116589962,
+      "grad_norm": 0.5287964277729839,
+      "learning_rate": 0.00016610862638601163,
+      "loss": 0.4066593170166016,
+      "step": 2800
+    },
+    {
+      "epoch": 1.2380013149243918,
+      "grad_norm": 0.5583879530662426,
+      "learning_rate": 0.00016542631129829794,
+      "loss": 0.45244613647460935,
+      "step": 2825
+    },
+    {
+      "epoch": 1.2489590181897874,
+      "grad_norm": 0.40620191555003865,
+      "learning_rate": 0.00016473862847818277,
+      "loss": 0.4667433929443359,
+      "step": 2850
+    },
+    {
+      "epoch": 1.259916721455183,
+      "grad_norm": 0.4535954318973333,
+      "learning_rate": 0.00016404563434482356,
+      "loss": 0.4026165008544922,
+      "step": 2875
+    },
+    {
+      "epoch": 1.2708744247205785,
+      "grad_norm": 0.513322677830209,
+      "learning_rate": 0.00016334738575313063,
+      "loss": 0.4219857788085937,
+      "step": 2900
+    },
+    {
+      "epoch": 1.2818321279859741,
+      "grad_norm": 0.42389586092570064,
+      "learning_rate": 0.0001626439399891028,
+      "loss": 0.44494407653808593,
+      "step": 2925
+    },
+    {
+      "epoch": 1.2927898312513697,
+      "grad_norm": 0.6766414860070618,
+      "learning_rate": 0.0001619353547651274,
+      "loss": 0.42329719543457034,
+      "step": 2950
+    },
+    {
+      "epoch": 1.3037475345167653,
+      "grad_norm": 0.5985020346801904,
+      "learning_rate": 0.00016122168821524545,
+      "loss": 0.45065216064453123,
+      "step": 2975
+    },
+    {
+      "epoch": 1.3147052377821609,
+      "grad_norm": 0.6405826298129538,
+      "learning_rate": 0.00016050299889038225,
+      "loss": 0.45827613830566405,
+      "step": 3000
+    },
+    {
+      "epoch": 1.3256629410475564,
+      "grad_norm": 0.6549621659634891,
+      "learning_rate": 0.00015977934575354366,
+      "loss": 0.4362139129638672,
+      "step": 3025
+    },
+    {
+      "epoch": 1.336620644312952,
+      "grad_norm": 0.6106103227315635,
+      "learning_rate": 0.0001590507881749786,
+      "loss": 0.4384369659423828,
+      "step": 3050
+    },
+    {
+      "epoch": 1.3475783475783476,
+      "grad_norm": 0.4371166163225976,
+      "learning_rate": 0.00015831738592730825,
+      "loss": 0.4177792358398438,
+      "step": 3075
+    },
+    {
+      "epoch": 1.3585360508437432,
+      "grad_norm": 0.5206936757463656,
+      "learning_rate": 0.0001575791991806222,
+      "loss": 0.4393328857421875,
+      "step": 3100
+    },
+    {
+      "epoch": 1.3694937541091388,
+      "grad_norm": 0.601154140872349,
+      "learning_rate": 0.0001568362884975417,
+      "loss": 0.4083403015136719,
+      "step": 3125
+    },
+    {
+      "epoch": 1.3804514573745343,
+      "grad_norm": 0.6660346927890635,
+      "learning_rate": 0.0001560887148282514,
+      "loss": 0.4107976531982422,
+      "step": 3150
+    },
+    {
+      "epoch": 1.39140916063993,
+      "grad_norm": 0.39369331922521944,
+      "learning_rate": 0.00015533653950549828,
+      "loss": 0.43758338928222656,
+      "step": 3175
+    },
+    {
+      "epoch": 1.4023668639053255,
+      "grad_norm": 0.4295971116287204,
+      "learning_rate": 0.00015457982423956038,
+      "loss": 0.4095724105834961,
+      "step": 3200
+    },
+    {
+      "epoch": 1.413324567170721,
+      "grad_norm": 0.3797449112023034,
+      "learning_rate": 0.0001538186311131834,
+      "loss": 0.4068620681762695,
+      "step": 3225
+    },
+    {
+      "epoch": 1.4242822704361167,
+      "grad_norm": 0.4276983810707517,
+      "learning_rate": 0.00015305302257648758,
+      "loss": 0.42065086364746096,
+      "step": 3250
+    },
+    {
+      "epoch": 1.4352399737015122,
+      "grad_norm": 0.4666197159909488,
+      "learning_rate": 0.00015228306144184414,
+      "loss": 0.3989574432373047,
+      "step": 3275
+    },
+    {
+      "epoch": 1.4461976769669078,
+      "grad_norm": 0.40133694963201777,
+      "learning_rate": 0.00015150881087872185,
+      "loss": 0.40041751861572267,
+      "step": 3300
+    },
+    {
+      "epoch": 1.4571553802323032,
+      "grad_norm": 0.583938330669856,
+      "learning_rate": 0.00015073033440850454,
+      "loss": 0.4483616256713867,
+      "step": 3325
+    },
+    {
+      "epoch": 1.4681130834976988,
+      "grad_norm": 0.6309031584352569,
+      "learning_rate": 0.00014994769589927974,
+      "loss": 0.4432932662963867,
+      "step": 3350
+    },
+    {
+      "epoch": 1.4790707867630943,
+      "grad_norm": 0.5412128473094439,
+      "learning_rate": 0.0001491609595605986,
+      "loss": 0.4114042663574219,
+      "step": 3375
+    },
+    {
+      "epoch": 1.49002849002849,
+      "grad_norm": 0.6205106303455942,
+      "learning_rate": 0.00014837018993820814,
+      "loss": 0.41695125579833986,
+      "step": 3400
+    },
+    {
+      "epoch": 1.5009861932938855,
+      "grad_norm": 0.9752950103207944,
+      "learning_rate": 0.0001475754519087557,
+      "loss": 0.45008018493652346,
+      "step": 3425
+    },
+    {
+      "epoch": 1.511943896559281,
+      "grad_norm": 0.5641458237912382,
+      "learning_rate": 0.00014677681067446627,
+      "loss": 0.44077972412109373,
+      "step": 3450
+    },
+    {
+      "epoch": 1.5229015998246767,
+      "grad_norm": 0.5737224320842583,
+      "learning_rate": 0.00014597433175779322,
+      "loss": 0.42770389556884764,
+      "step": 3475
+    },
+    {
+      "epoch": 1.5338593030900722,
+      "grad_norm": 0.41428253922267605,
+      "learning_rate": 0.00014516808099604268,
+      "loss": 0.4266217803955078,
+      "step": 3500
+    },
+    {
+      "epoch": 1.5448170063554678,
+      "grad_norm": 0.6569121745141964,
+      "learning_rate": 0.00014435812453597192,
+      "loss": 0.41614547729492185,
+      "step": 3525
+    },
+    {
+      "epoch": 1.5557747096208634,
+      "grad_norm": 0.5741297578975548,
+      "learning_rate": 0.00014354452882836273,
+      "loss": 0.40014656066894533,
+      "step": 3550
+    },
+    {
+      "epoch": 1.566732412886259,
+      "grad_norm": 0.3617286031559024,
+      "learning_rate": 0.00014272736062256958,
+      "loss": 0.353155632019043,
+      "step": 3575
+    },
+    {
+      "epoch": 1.5776901161516546,
+      "grad_norm": 0.6194726963689449,
+      "learning_rate": 0.00014190668696104322,
+      "loss": 0.4024491882324219,
+      "step": 3600
+    },
+    {
+      "epoch": 1.5886478194170501,
+      "grad_norm": 0.3812826850618246,
+      "learning_rate": 0.00014108257517383047,
+      "loss": 0.4127301025390625,
+      "step": 3625
+    },
+    {
+      "epoch": 1.5996055226824457,
+      "grad_norm": 0.6027967532353757,
+      "learning_rate": 0.00014025509287305036,
+      "loss": 0.4099040222167969,
+      "step": 3650
+    },
+    {
+      "epoch": 1.6105632259478413,
+      "grad_norm": 0.6577687378159764,
+      "learning_rate": 0.00013942430794734687,
+      "loss": 0.36910625457763674,
+      "step": 3675
+    },
+    {
+      "epoch": 1.6215209292132369,
+      "grad_norm": 0.5118439158326048,
+      "learning_rate": 0.0001385902885563193,
+      "loss": 0.4529541015625,
+      "step": 3700
+    },
+    {
+      "epoch": 1.6324786324786325,
+      "grad_norm": 0.45743704025700843,
+      "learning_rate": 0.00013775310312493037,
+      "loss": 0.3970626449584961,
+      "step": 3725
+    },
+    {
+      "epoch": 1.643436335744028,
+      "grad_norm": 0.5306975225842998,
+      "learning_rate": 0.00013691282033789236,
+      "loss": 0.35301612854003905,
+      "step": 3750
+    },
+    {
+      "epoch": 1.6543940390094236,
+      "grad_norm": 0.481209904136711,
+      "learning_rate": 0.00013606950913403208,
+      "loss": 0.41025718688964846,
+      "step": 3775
+    },
+    {
+      "epoch": 1.6653517422748192,
+      "grad_norm": 0.4468197234526083,
+      "learning_rate": 0.00013522323870063488,
+      "loss": 0.4039540100097656,
+      "step": 3800
+    },
+    {
+      "epoch": 1.6763094455402148,
+      "grad_norm": 0.3798246354927787,
+      "learning_rate": 0.00013437407846776854,
+      "loss": 0.38895225524902344,
+      "step": 3825
+    },
+    {
+      "epoch": 1.6872671488056104,
+      "grad_norm": 0.5813760717850384,
+      "learning_rate": 0.0001335220981025869,
+      "loss": 0.44722671508789064,
+      "step": 3850
+    },
+    {
+      "epoch": 1.698224852071006,
+      "grad_norm": 0.5020958735964437,
+      "learning_rate": 0.00013266736750361431,
+      "loss": 0.37591259002685545,
+      "step": 3875
+    },
+    {
+      "epoch": 1.7091825553364015,
+      "grad_norm": 0.5910378889000357,
+      "learning_rate": 0.0001318099567950109,
+      "loss": 0.38099960327148436,
+      "step": 3900
+    },
+    {
+      "epoch": 1.720140258601797,
+      "grad_norm": 0.4355687433094115,
+      "learning_rate": 0.00013094993632081948,
+      "loss": 0.41964092254638674,
+      "step": 3925
+    },
+    {
+      "epoch": 1.7310979618671927,
+      "grad_norm": 0.7044250035872699,
+      "learning_rate": 0.0001300873766391943,
+      "loss": 0.3695980072021484,
+      "step": 3950
+    },
+    {
+      "epoch": 1.7420556651325882,
+      "grad_norm": 0.4701082647207719,
+      "learning_rate": 0.00012922234851661232,
+      "loss": 0.3810658645629883,
+      "step": 3975
+    },
+    {
+      "epoch": 1.7530133683979838,
+      "grad_norm": 0.47321051780622075,
+      "learning_rate": 0.00012835492292206734,
+      "loss": 0.42526351928710937,
+      "step": 4000
+    },
+    {
+      "epoch": 1.7639710716633794,
+      "grad_norm": 0.5300139983011811,
+      "learning_rate": 0.00012748517102124755,
+      "loss": 0.4171455383300781,
+      "step": 4025
+    },
+    {
+      "epoch": 1.774928774928775,
+      "grad_norm": 0.5480913021210451,
+      "learning_rate": 0.000126613164170697,
+      "loss": 0.38637245178222657,
+      "step": 4050
+    },
+    {
+      "epoch": 1.7858864781941706,
+      "grad_norm": 0.5028066369365886,
+      "learning_rate": 0.00012573897391196108,
+      "loss": 0.40807647705078126,
+      "step": 4075
+    },
+    {
+      "epoch": 1.7968441814595661,
+      "grad_norm": 0.7192463460046236,
+      "learning_rate": 0.0001248626719657174,
+      "loss": 0.39599617004394533,
+      "step": 4100
+    },
+    {
+      "epoch": 1.8078018847249617,
+      "grad_norm": 0.2827808969257674,
+      "learning_rate": 0.00012398433022589147,
+      "loss": 0.3685328674316406,
+      "step": 4125
+    },
+    {
+      "epoch": 1.8187595879903573,
+      "grad_norm": 0.4274683889238332,
+      "learning_rate": 0.00012310402075375832,
+      "loss": 0.43625213623046877,
+      "step": 4150
+    },
+    {
+      "epoch": 1.8297172912557529,
+      "grad_norm": 0.44692450404712275,
+      "learning_rate": 0.00012222181577203065,
+      "loss": 0.4161438751220703,
+      "step": 4175
+    },
+    {
+      "epoch": 1.8406749945211485,
+      "grad_norm": 0.41658336460133644,
+      "learning_rate": 0.00012133778765893313,
+      "loss": 0.3899082183837891,
+      "step": 4200
+    },
+    {
+      "epoch": 1.851632697786544,
+      "grad_norm": 0.5997761212772421,
+      "learning_rate": 0.0001204520089422647,
+      "loss": 0.38892837524414064,
+      "step": 4225
+    },
+    {
+      "epoch": 1.8625904010519396,
+      "grad_norm": 0.6126818822909509,
+      "learning_rate": 0.00011956455229344792,
+      "loss": 0.3780496978759766,
+      "step": 4250
+    },
+    {
+      "epoch": 1.8735481043173352,
+      "grad_norm": 0.40127703894536587,
+      "learning_rate": 0.00011867549052156701,
+      "loss": 0.30914302825927736,
+      "step": 4275
+    },
+    {
+      "epoch": 1.8845058075827308,
+      "grad_norm": 0.43205111787119116,
+      "learning_rate": 0.00011778489656739435,
+      "loss": 0.35536827087402345,
+      "step": 4300
+    },
+    {
+      "epoch": 1.8954635108481264,
+      "grad_norm": 0.46351963773506527,
+      "learning_rate": 0.00011689284349740629,
+      "loss": 0.3959296035766602,
+      "step": 4325
+    },
+    {
+      "epoch": 1.906421214113522,
+      "grad_norm": 0.4229112462335532,
+      "learning_rate": 0.00011599940449778841,
+      "loss": 0.41460060119628905,
+      "step": 4350
+    },
+    {
+      "epoch": 1.9173789173789175,
+      "grad_norm": 0.4449274400172185,
+      "learning_rate": 0.0001151046528684315,
+      "loss": 0.3677532958984375,
+      "step": 4375
+    },
+    {
+      "epoch": 1.928336620644313,
+      "grad_norm": 0.37239875530774874,
+      "learning_rate": 0.00011420866201691754,
+      "loss": 0.3918146514892578,
+      "step": 4400
+    },
+    {
+      "epoch": 1.9392943239097087,
+      "grad_norm": 0.5479251595725931,
+      "learning_rate": 0.0001133115054524973,
+      "loss": 0.3971866989135742,
+      "step": 4425
+    },
+    {
+      "epoch": 1.9502520271751043,
+      "grad_norm": 0.6738385136938495,
+      "learning_rate": 0.00011241325678005952,
+      "loss": 0.3569622039794922,
+      "step": 4450
+    },
+    {
+      "epoch": 1.9612097304404996,
+      "grad_norm": 0.3304334793339243,
+      "learning_rate": 0.000111513989694092,
+      "loss": 0.35907974243164065,
+      "step": 4475
+    },
+    {
+      "epoch": 1.9721674337058952,
+      "grad_norm": 0.5072052544273601,
+      "learning_rate": 0.0001106137779726357,
+      "loss": 0.3592760467529297,
+      "step": 4500
+    },
+    {
+      "epoch": 1.9831251369712908,
+      "grad_norm": 0.3944491895367133,
+      "learning_rate": 0.0001097126954712318,
+      "loss": 0.3897372817993164,
+      "step": 4525
+    },
+    {
+      "epoch": 1.9940828402366864,
+      "grad_norm": 0.6295150987269155,
+      "learning_rate": 0.00010881081611686228,
+      "loss": 0.3533794403076172,
+      "step": 4550
+    },
+    {
+      "epoch": 2.004821389436774,
+      "grad_norm": 0.42475925428031835,
+      "learning_rate": 0.00010790821390188493,
+      "loss": 0.30100307464599607,
+      "step": 4575
+    },
+    {
+      "epoch": 2.0157790927021697,
+      "grad_norm": 0.6599014787251235,
+      "learning_rate": 0.00010700496287796275,
+      "loss": 0.20593643188476562,
+      "step": 4600
+    },
+    {
+      "epoch": 2.0267367959675653,
+      "grad_norm": 0.7528878273483994,
+      "learning_rate": 0.00010610113714998856,
+      "loss": 0.2313397979736328,
+      "step": 4625
+    },
+    {
+      "epoch": 2.037694499232961,
+      "grad_norm": 0.5793530412033049,
+      "learning_rate": 0.00010519681087000526,
+      "loss": 0.22497249603271485,
+      "step": 4650
+    },
+    {
+      "epoch": 2.0486522024983564,
+      "grad_norm": 0.536535545512749,
+      "learning_rate": 0.00010429205823112235,
+      "loss": 0.18831499099731444,
+      "step": 4675
+    },
+    {
+      "epoch": 2.059609905763752,
+      "grad_norm": 0.4654007970079509,
+      "learning_rate": 0.00010338695346142882,
+      "loss": 0.18600177764892578,
+      "step": 4700
+    },
+    {
+      "epoch": 2.0705676090291476,
+      "grad_norm": 0.26546424867098994,
+      "learning_rate": 0.00010248157081790324,
+      "loss": 0.18008216857910156,
+      "step": 4725
+    },
+    {
+      "epoch": 2.081525312294543,
+      "grad_norm": 0.5303633209359471,
+      "learning_rate": 0.00010157598458032165,
+      "loss": 0.18256614685058595,
+      "step": 4750
+    },
+    {
+      "epoch": 2.0924830155599388,
+      "grad_norm": 0.47371462788258106,
+      "learning_rate": 0.00010067026904516348,
+      "loss": 0.20771533966064454,
+      "step": 4775
+    },
+    {
+      "epoch": 2.1034407188253343,
+      "grad_norm": 0.4992461440902292,
+      "learning_rate": 9.9764498519516e-05,
+      "loss": 0.17464292526245118,
+      "step": 4800
+    },
+    {
+      "epoch": 2.11439842209073,
+      "grad_norm": 0.6353285066286388,
+      "learning_rate": 9.885874731497806e-05,
+      "loss": 0.20087291717529296,
+      "step": 4825
+    },
+    {
+      "epoch": 2.1253561253561255,
+      "grad_norm": 0.5718972692052943,
+      "learning_rate": 9.795308974156334e-05,
+      "loss": 0.21221614837646485,
+      "step": 4850
+    },
+    {
+      "epoch": 2.136313828621521,
+      "grad_norm": 0.5830582148895164,
+      "learning_rate": 9.704760010160385e-05,
+      "loss": 0.21111564636230468,
+      "step": 4875
+    },
+    {
+      "epoch": 2.1472715318869167,
+      "grad_norm": 0.4959043548616016,
+      "learning_rate": 9.614235268365385e-05,
+      "loss": 0.17894338607788085,
+      "step": 4900
+    },
+    {
+      "epoch": 2.1582292351523122,
+      "grad_norm": 0.5614418636383869,
+      "learning_rate": 9.523742175639514e-05,
+      "loss": 0.17951549530029298,
+      "step": 4925
+    },
+    {
+      "epoch": 2.169186938417708,
+      "grad_norm": 0.5186563360064427,
+      "learning_rate": 9.433288156254394e-05,
+      "loss": 0.1830301284790039,
+      "step": 4950
+    },
+    {
+      "epoch": 2.1801446416831034,
+      "grad_norm": 0.5002629074885125,
+      "learning_rate": 9.342880631275967e-05,
+      "loss": 0.22070056915283204,
+      "step": 4975
+    },
+    {
+      "epoch": 2.191102344948499,
+      "grad_norm": 0.4255229272859531,
+      "learning_rate": 9.252527017955665e-05,
+      "loss": 0.21093353271484375,
+      "step": 5000
+    },
+    {
+      "epoch": 2.2020600482138946,
+      "grad_norm": 0.5043061355611567,
+      "learning_rate": 9.162234729121876e-05,
+      "loss": 0.19972602844238282,
+      "step": 5025
+    },
+    {
+      "epoch": 2.21301775147929,
+      "grad_norm": 0.6041952884940843,
+      "learning_rate": 9.072011172571786e-05,
+      "loss": 0.18799491882324218,
+      "step": 5050
+    },
+    {
+      "epoch": 2.2239754547446857,
+      "grad_norm": 0.5727698534096206,
+      "learning_rate": 8.981863750463608e-05,
+      "loss": 0.1831727409362793,
+      "step": 5075
+    },
+    {
+      "epoch": 2.2349331580100813,
+      "grad_norm": 0.4045749347222439,
+      "learning_rate": 8.891799858709309e-05,
+      "loss": 0.1804789733886719,
+      "step": 5100
+    },
+    {
+      "epoch": 2.245890861275477,
+      "grad_norm": 0.46390632444913404,
+      "learning_rate": 8.801826886367825e-05,
+      "loss": 0.1879850769042969,
+      "step": 5125
+    },
+    {
+      "epoch": 2.2568485645408725,
+      "grad_norm": 0.6669996913479316,
+      "learning_rate": 8.711952215038836e-05,
+      "loss": 0.16245586395263673,
+      "step": 5150
+    },
+    {
+      "epoch": 2.267806267806268,
+      "grad_norm": 0.5222544677950751,
+      "learning_rate": 8.622183218257178e-05,
+      "loss": 0.18196407318115235,
+      "step": 5175
+    },
+    {
+      "epoch": 2.2787639710716636,
+      "grad_norm": 0.3025874110311181,
+      "learning_rate": 8.53252726088789e-05,
+      "loss": 0.1737771987915039,
+      "step": 5200
+    },
+    {
+      "epoch": 2.289721674337059,
+      "grad_norm": 0.41280050630518345,
+      "learning_rate": 8.442991698521983e-05,
+      "loss": 0.18274156570434572,
+      "step": 5225
+    },
+    {
+      "epoch": 2.3006793776024543,
+      "grad_norm": 0.5448823180733036,
+      "learning_rate": 8.353583876872973e-05,
+      "loss": 0.2126680374145508,
+      "step": 5250
+    },
+    {
+      "epoch": 2.31163708086785,
+      "grad_norm": 0.39029761062234003,
+      "learning_rate": 8.26431113117422e-05,
+      "loss": 0.17433416366577148,
+      "step": 5275
+    },
+    {
+      "epoch": 2.3225947841332455,
+      "grad_norm": 0.5190303850276005,
+      "learning_rate": 8.175180785577126e-05,
+      "loss": 0.18076852798461915,
+      "step": 5300
+    },
+    {
+      "epoch": 2.333552487398641,
+      "grad_norm": 0.7482891566791909,
+      "learning_rate": 8.086200152550245e-05,
+      "loss": 0.18866779327392577,
+      "step": 5325
+    },
+    {
+      "epoch": 2.3445101906640367,
+      "grad_norm": 0.7997774036170018,
+      "learning_rate": 7.997376532279353e-05,
+      "loss": 0.21518733978271484,
+      "step": 5350
+    },
+    {
+      "epoch": 2.3554678939294322,
+      "grad_norm": 0.40611285671427333,
+      "learning_rate": 7.908717212068516e-05,
+      "loss": 0.18596004486083983,
+      "step": 5375
+    },
+    {
+      "epoch": 2.366425597194828,
+      "grad_norm": 0.5001178592867455,
+      "learning_rate": 7.820229465742233e-05,
+      "loss": 0.17369129180908202,
+      "step": 5400
+    },
+    {
+      "epoch": 2.3773833004602234,
+      "grad_norm": 0.44206195396632475,
+      "learning_rate": 7.731920553048664e-05,
+      "loss": 0.17895523071289063,
+      "step": 5425
+    },
+    {
+      "epoch": 2.388341003725619,
+      "grad_norm": 0.438105736567722,
+      "learning_rate": 7.643797719064025e-05,
+      "loss": 0.1888243293762207,
+      "step": 5450
+    },
+    {
+      "epoch": 2.3992987069910146,
+      "grad_norm": 0.751123609103379,
+      "learning_rate": 7.555868193598188e-05,
+      "loss": 0.22261222839355468,
+      "step": 5475
+    },
+    {
+      "epoch": 2.41025641025641,
+      "grad_norm": 0.38209582613354354,
+      "learning_rate": 7.468139190601522e-05,
+      "loss": 0.22064062118530273,
+      "step": 5500
+    },
+    {
+      "epoch": 2.4212141135218057,
+      "grad_norm": 0.367712009346715,
+      "learning_rate": 7.380617907573069e-05,
+      "loss": 0.1783095932006836,
+      "step": 5525
+    },
+    {
+      "epoch": 2.4321718167872013,
+      "grad_norm": 0.4986161008330237,
+      "learning_rate": 7.293311524969996e-05,
+      "loss": 0.1941888427734375,
+      "step": 5550
+    },
+    {
+      "epoch": 2.443129520052597,
+      "grad_norm": 0.4896068761776909,
+      "learning_rate": 7.206227205618539e-05,
+      "loss": 0.21169797897338868,
+      "step": 5575
+    },
+    {
+      "epoch": 2.4540872233179924,
+      "grad_norm": 0.38617190954847214,
+      "learning_rate": 7.119372094126324e-05,
+      "loss": 0.21524496078491212,
+      "step": 5600
+    },
+    {
+      "epoch": 2.465044926583388,
+      "grad_norm": 0.4265355504283047,
+      "learning_rate": 7.03275331629621e-05,
+      "loss": 0.19470691680908203,
+      "step": 5625
+    },
+    {
+      "epoch": 2.4760026298487836,
+      "grad_norm": 0.3946204507602001,
+      "learning_rate": 6.946377978541672e-05,
+      "loss": 0.17049610137939453,
+      "step": 5650
+    },
+    {
+      "epoch": 2.486960333114179,
+      "grad_norm": 0.37752529397719053,
+      "learning_rate": 6.860253167303782e-05,
+      "loss": 0.18190874099731447,
+      "step": 5675
+    },
+    {
+      "epoch": 2.4979180363795748,
+      "grad_norm": 0.08608989309293166,
+      "learning_rate": 6.774385948469798e-05,
+      "loss": 0.17841957092285157,
+      "step": 5700
+    },
+    {
+      "epoch": 2.5088757396449703,
+      "grad_norm": 0.22563363737773465,
+      "learning_rate": 6.688783366793487e-05,
+      "loss": 0.18140106201171874,
+      "step": 5725
+    },
+    {
+      "epoch": 2.519833442910366,
+      "grad_norm": 0.36334007682627234,
+      "learning_rate": 6.603452445317141e-05,
+      "loss": 0.20584403991699218,
+      "step": 5750
+    },
+    {
+      "epoch": 2.5307911461757615,
+      "grad_norm": 0.5326637941553573,
+      "learning_rate": 6.518400184795399e-05,
+      "loss": 0.18029726028442383,
+      "step": 5775
+    },
+    {
+      "epoch": 2.541748849441157,
+      "grad_norm": 0.6623342855081067,
+      "learning_rate": 6.433633563120873e-05,
+      "loss": 0.2009827423095703,
+      "step": 5800
+    },
+    {
+      "epoch": 2.5527065527065527,
+      "grad_norm": 0.47515403185012606,
+      "learning_rate": 6.349159534751686e-05,
+      "loss": 0.19584007263183595,
+      "step": 5825
+    },
+    {
+      "epoch": 2.5636642559719482,
+      "grad_norm": 0.7770078049833565,
+      "learning_rate": 6.264985030140889e-05,
+      "loss": 0.17622970581054687,
+      "step": 5850
+    },
+    {
+      "epoch": 2.574621959237344,
+      "grad_norm": 0.3173062014548844,
+      "learning_rate": 6.18111695516789e-05,
+      "loss": 0.16865333557128906,
+      "step": 5875
+    },
+    {
+      "epoch": 2.5855796625027394,
+      "grad_norm": 0.5045863359740529,
+      "learning_rate": 6.097562190571871e-05,
+      "loss": 0.18796072006225586,
+      "step": 5900
+    },
+    {
+      "epoch": 2.596537365768135,
+      "grad_norm": 0.3801300693170537,
+      "learning_rate": 6.0143275913872685e-05,
+      "loss": 0.1898968505859375,
+      "step": 5925
+    },
+    {
+      "epoch": 2.6074950690335306,
+      "grad_norm": 0.6268892328040334,
+      "learning_rate": 5.9314199863813814e-05,
+      "loss": 0.17133827209472657,
+      "step": 5950
+    },
+    {
+      "epoch": 2.618452772298926,
+      "grad_norm": 0.4344103574651637,
+      "learning_rate": 5.848846177494113e-05,
+      "loss": 0.16486854553222657,
+      "step": 5975
+    },
+    {
+      "epoch": 2.6294104755643217,
+      "grad_norm": 0.43174106801547035,
+      "learning_rate": 5.766612939279936e-05,
+      "loss": 0.18146709442138673,
+      "step": 6000
+    },
+    {
+      "epoch": 2.6403681788297173,
+      "grad_norm": 0.8320871030708143,
+      "learning_rate": 5.6847270183520704e-05,
+      "loss": 0.1793173599243164,
+      "step": 6025
+    },
+    {
+      "epoch": 2.651325882095113,
+      "grad_norm": 0.6969900218332219,
+      "learning_rate": 5.603195132829004e-05,
+      "loss": 0.1941357421875,
+      "step": 6050
+    },
+    {
+      "epoch": 2.6622835853605085,
+      "grad_norm": 0.5786431019167166,
+      "learning_rate": 5.52202397178329e-05,
+      "loss": 0.18954509735107422,
+      "step": 6075
+    },
+    {
+      "epoch": 2.673241288625904,
+      "grad_norm": 0.5843830848252451,
+      "learning_rate": 5.441220194692791e-05,
+      "loss": 0.17464656829833985,
+      "step": 6100
+    },
+    {
+      "epoch": 2.6841989918912996,
+      "grad_norm": 0.3232102699066598,
+      "learning_rate": 5.360790430894287e-05,
+      "loss": 0.1684918975830078,
+      "step": 6125
+    },
+    {
+      "epoch": 2.695156695156695,
+      "grad_norm": 0.6443713085572776,
+      "learning_rate": 5.2807412790396236e-05,
+      "loss": 0.1757213592529297,
+      "step": 6150
+    },
+    {
+      "epoch": 2.706114398422091,
+      "grad_norm": 0.18177662418436313,
+      "learning_rate": 5.2010793065543106e-05,
+      "loss": 0.16545526504516603,
+      "step": 6175
+    },
+    {
+      "epoch": 2.7170721016874864,
+      "grad_norm": 0.5210754933601423,
+      "learning_rate": 5.121811049098728e-05,
+      "loss": 0.19662631988525392,
+      "step": 6200
+    },
+    {
+      "epoch": 2.728029804952882,
+      "grad_norm": 0.585218121092206,
+      "learning_rate": 5.0429430100319374e-05,
+      "loss": 0.172868709564209,
+      "step": 6225
+    },
+    {
+      "epoch": 2.7389875082182775,
+      "grad_norm": 0.5818397863464598,
+      "learning_rate": 4.964481659878104e-05,
+      "loss": 0.21251907348632812,
+      "step": 6250
+    },
+    {
+      "epoch": 2.749945211483673,
+      "grad_norm": 0.5736086489916201,
+      "learning_rate": 4.8864334357956685e-05,
+      "loss": 0.1800657653808594,
+      "step": 6275
+    },
+    {
+      "epoch": 2.7609029147490687,
+      "grad_norm": 0.5398473637763418,
+      "learning_rate": 4.8088047410492e-05,
+      "loss": 0.17316413879394532,
+      "step": 6300
+    },
+    {
+      "epoch": 2.7718606180144643,
+      "grad_norm": 0.4882500455280308,
+      "learning_rate": 4.7316019444840675e-05,
+      "loss": 0.17947908401489257,
+      "step": 6325
+    },
+    {
+      "epoch": 2.78281832127986,
+      "grad_norm": 0.5173728299684417,
+      "learning_rate": 4.65483138000394e-05,
+      "loss": 0.18379810333251953,
+      "step": 6350
+    },
+    {
+      "epoch": 2.7937760245452554,
+      "grad_norm": 0.2833415056055466,
+      "learning_rate": 4.5784993460511075e-05,
+      "loss": 0.1703580856323242,
+      "step": 6375
+    },
+    {
+      "epoch": 2.804733727810651,
+      "grad_norm": 0.41885716839100023,
+      "learning_rate": 4.502612105089772e-05,
+      "loss": 0.16782974243164062,
+      "step": 6400
+    },
+    {
+      "epoch": 2.8156914310760466,
+      "grad_norm": 0.5808133864295705,
+      "learning_rate": 4.42717588309223e-05,
+      "loss": 0.1972628974914551,
+      "step": 6425
+    },
+    {
+      "epoch": 2.826649134341442,
+      "grad_norm": 0.6332897360024736,
+      "learning_rate": 4.352196869028113e-05,
+      "loss": 0.16635608673095703,
+      "step": 6450
+    },
+    {
+      "epoch": 2.8376068376068377,
+      "grad_norm": 0.5334112612301344,
+      "learning_rate": 4.277681214356591e-05,
+      "loss": 0.1516483783721924,
+      "step": 6475
+    },
+    {
+      "epoch": 2.8485645408722333,
+      "grad_norm": 0.4253163458685073,
+      "learning_rate": 4.203635032521727e-05,
+      "loss": 0.15661946296691895,
+      "step": 6500
+    }
+  ],
+  "logging_steps": 25,
+  "max_steps": 9128,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 4,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 308809376858112.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}