Download train.py from debudash/slm-training: direct link, hf CLI and curl.
- Browser
- Download file 6.56 kB
-
https://huggingface.co/debudash/slm-training/resolve/main/train.py
- Command line
-
hf download hf://debudash/slm-training/train.py
-
curl -L -o train.py https://huggingface.co/debudash/slm-training/resolve/main/train.py
6.56 kB
| """QLoRA fine-tune of Qwen3-4B-Instruct-2507 on Modal, one run per dataset size, pushed to the Hugging Face Hub. | |
| Loss is on assistant turns only. The first N lines of the dataset are used, so sizes are nested subsets. | |
| Each run pushes its LoRA adapter to a public repo and writes its log to results/training/<run>.json. | |
| Usage: modal run train.py --data data/train_v1.jsonl --ns 250,500,1000,2000 --tag v1 | |
| """ | |
| import json | |
| import time | |
| from pathlib import Path | |
| import modal | |
| BASE_MODEL = "Qwen/Qwen3-4B-Instruct-2507" | |
| BASE_REVISION = "cdbee75f17c01a7cc42f958dc650907174af0554" | |
| HF_USER = "debudash" | |
| LORA = {"r": 16, "lora_alpha": 32, "lora_dropout": 0.05, | |
| "target_modules": ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"]} | |
| HPARAMS = {"epochs": 3, "lr": 2e-4, "batch": 8, "grad_accum": 2, "max_len": 1536, "warmup_ratio": 0.03, "seed": 7} | |
| image = (modal.Image.debian_slim(python_version="3.12") | |
| .pip_install("torch==2.14.1", "transformers==5.21.0", "peft==0.21.2", "bitsandbytes==0.50.2", | |
| "accelerate==1.15.0", "huggingface_hub==2.3.0")) | |
| hf_cache = modal.Volume.from_name("hf-cache", create_if_missing=True) | |
| app = modal.App("slm-training-train", image=image) | |
| def train(examples: list, run_name: str, hparams: dict, lora: dict): | |
| import torch | |
| from huggingface_hub import HfApi | |
| from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training | |
| from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, Trainer, | |
| TrainerCallback, TrainingArguments) | |
| tok = AutoTokenizer.from_pretrained(BASE_MODEL, revision=BASE_REVISION) | |
| def encode(messages): | |
| """Token ids and labels, with labels only on assistant spans (including their end-of-turn token).""" | |
| render = lambda msgs, gen=False: tok.apply_chat_template(msgs, tokenize=False, add_generation_prompt=gen) | |
| full = render(messages) | |
| spans = [] | |
| for k, m in enumerate(messages): | |
| if m["role"] == "assistant": | |
| start, end = len(render(messages[:k], gen=True)), len(render(messages[:k + 1])) | |
| assert full.startswith(render(messages[:k + 1])), "chat template is not prefix-stable" | |
| spans.append((start, end)) | |
| enc = tok(full, return_offsets_mapping=True, add_special_tokens=False) | |
| labels = [i if any(s <= a < e for s, e in spans) else -100 | |
| for i, (a, _) in zip(enc["input_ids"], enc["offset_mapping"])] | |
| return {"input_ids": enc["input_ids"][: hparams["max_len"]], "labels": labels[: hparams["max_len"]]} | |
| data = [encode(e["messages"]) for e in examples] | |
| def collate(batch): | |
| n = max(len(b["input_ids"]) for b in batch) | |
| pad = lambda xs, v: xs + [v] * (n - len(xs)) | |
| return {"input_ids": torch.tensor([pad(b["input_ids"], tok.pad_token_id) for b in batch]), | |
| "labels": torch.tensor([pad(b["labels"], -100) for b in batch]), | |
| "attention_mask": torch.tensor([pad([1] * len(b["input_ids"]), 0) for b in batch])} | |
| model = AutoModelForCausalLM.from_pretrained( | |
| BASE_MODEL, revision=BASE_REVISION, dtype=torch.bfloat16, | |
| quantization_config=BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_quant_type="nf4", | |
| bnb_4bit_compute_dtype=torch.bfloat16, bnb_4bit_use_double_quant=True)) | |
| model = prepare_model_for_kbit_training(model, use_gradient_checkpointing=True) | |
| model = get_peft_model(model, LoraConfig(task_type="CAUSAL_LM", **lora)) | |
| log = [] | |
| class Log(TrainerCallback): | |
| def on_log(self, args, state, control, logs=None, **kw): | |
| if logs: | |
| log.append({"step": state.global_step, "epoch": round(state.epoch or 0, 3), **logs}) | |
| out_dir = f"/tmp/{run_name}" | |
| steps = -(-len(data) // (hparams["batch"] * hparams["grad_accum"])) * hparams["epochs"] | |
| t0 = time.time() | |
| Trainer(model=model, train_dataset=data, data_collator=collate, callbacks=[Log()], | |
| args=TrainingArguments(output_dir=out_dir, num_train_epochs=hparams["epochs"], | |
| per_device_train_batch_size=hparams["batch"], | |
| gradient_accumulation_steps=hparams["grad_accum"], learning_rate=hparams["lr"], | |
| lr_scheduler_type="cosine", bf16=True, | |
| warmup_steps=max(1, round(steps * hparams["warmup_ratio"])), | |
| logging_steps=5, save_strategy="no", report_to=[], seed=hparams["seed"], | |
| gradient_checkpointing=True, remove_unused_columns=False)).train() | |
| minutes = (time.time() - t0) / 60 | |
| model.save_pretrained(out_dir) | |
| tok.save_pretrained(out_dir) | |
| repo = f"{HF_USER}/{run_name}" | |
| api = HfApi() | |
| api.create_repo(repo, private=False, exist_ok=True) | |
| info = api.upload_folder(folder_path=out_dir, repo_id=repo, commit_message=f"{run_name}: {len(examples)} examples", | |
| ignore_patterns=["checkpoint-*", "runs/*"]) | |
| return {"run": run_name, "repo": repo, "commit": info.oid, "n_examples": len(examples), "train_minutes": round(minutes, 1), | |
| "base_model": BASE_MODEL, "base_revision": BASE_REVISION, "hparams": hparams, "lora": lora, "log": log} | |
| def main(data: str = "data/train_v1.jsonl", ns: str = "250,500,1000,2000", tag: str = "v1", gpu: str = "L40S"): | |
| examples = [json.loads(l) for l in open(data, encoding="utf-8")] | |
| calls = [] | |
| fn = train.with_options(gpu=gpu) | |
| for n in [int(x) for x in ns.split(",")]: | |
| run = f"qwen3-4b-tz-convert-{tag}-n{n}" | |
| calls.append(fn.spawn(examples[:n], run, HPARAMS, LORA)) | |
| print(f"started {run}") | |
| out = Path("results/training") | |
| out.mkdir(parents=True, exist_ok=True) | |
| for c in calls: | |
| r = c.get() | |
| r["dataset"], r["gpu"] = data, gpu | |
| (out / f"{r['run']}.json").write_text(json.dumps(r, indent=1), encoding="utf-8") | |
| losses = [x["loss"] for x in r["log"] if "loss" in x] | |
| print(f"{r['run']}: {r['train_minutes']} min, loss {losses[0]:.3f} -> {losses[-1]:.3f}, " | |
| f"pushed {r['repo']}@{r['commit'][:8]}") | |