File size: 6,563 Bytes
4643f62
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
"""QLoRA fine-tune of Qwen3-4B-Instruct-2507 on Modal, one run per dataset size, pushed to the Hugging Face Hub.



Loss is on assistant turns only. The first N lines of the dataset are used, so sizes are nested subsets.

Each run pushes its LoRA adapter to a public repo and writes its log to results/training/<run>.json.



Usage: modal run train.py --data data/train_v1.jsonl --ns 250,500,1000,2000 --tag v1

"""
import json
import time
from pathlib import Path

import modal

BASE_MODEL = "Qwen/Qwen3-4B-Instruct-2507"
BASE_REVISION = "cdbee75f17c01a7cc42f958dc650907174af0554"
HF_USER = "debudash"
LORA = {"r": 16, "lora_alpha": 32, "lora_dropout": 0.05,
        "target_modules": ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"]}
HPARAMS = {"epochs": 3, "lr": 2e-4, "batch": 8, "grad_accum": 2, "max_len": 1536, "warmup_ratio": 0.03, "seed": 7}

image = (modal.Image.debian_slim(python_version="3.12")
         .pip_install("torch==2.14.1", "transformers==5.21.0", "peft==0.21.2", "bitsandbytes==0.50.2",
                      "accelerate==1.15.0", "huggingface_hub==2.3.0"))
hf_cache = modal.Volume.from_name("hf-cache", create_if_missing=True)
app = modal.App("slm-training-train", image=image)


@app.function(gpu="L40S", timeout=3 * 3600, volumes={"/root/.cache/huggingface": hf_cache},

              secrets=[modal.Secret.from_name("huggingface")])
def train(examples: list, run_name: str, hparams: dict, lora: dict):
    import torch
    from huggingface_hub import HfApi
    from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training
    from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, Trainer,
                              TrainerCallback, TrainingArguments)

    tok = AutoTokenizer.from_pretrained(BASE_MODEL, revision=BASE_REVISION)

    def encode(messages):
        """Token ids and labels, with labels only on assistant spans (including their end-of-turn token)."""
        render = lambda msgs, gen=False: tok.apply_chat_template(msgs, tokenize=False, add_generation_prompt=gen)
        full = render(messages)
        spans = []
        for k, m in enumerate(messages):
            if m["role"] == "assistant":
                start, end = len(render(messages[:k], gen=True)), len(render(messages[:k + 1]))
                assert full.startswith(render(messages[:k + 1])), "chat template is not prefix-stable"
                spans.append((start, end))
        enc = tok(full, return_offsets_mapping=True, add_special_tokens=False)
        labels = [i if any(s <= a < e for s, e in spans) else -100
                  for i, (a, _) in zip(enc["input_ids"], enc["offset_mapping"])]
        return {"input_ids": enc["input_ids"][: hparams["max_len"]], "labels": labels[: hparams["max_len"]]}

    data = [encode(e["messages"]) for e in examples]

    def collate(batch):
        n = max(len(b["input_ids"]) for b in batch)
        pad = lambda xs, v: xs + [v] * (n - len(xs))
        return {"input_ids": torch.tensor([pad(b["input_ids"], tok.pad_token_id) for b in batch]),
                "labels": torch.tensor([pad(b["labels"], -100) for b in batch]),
                "attention_mask": torch.tensor([pad([1] * len(b["input_ids"]), 0) for b in batch])}

    model = AutoModelForCausalLM.from_pretrained(
        BASE_MODEL, revision=BASE_REVISION, dtype=torch.bfloat16,
        quantization_config=BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_quant_type="nf4",
                                               bnb_4bit_compute_dtype=torch.bfloat16, bnb_4bit_use_double_quant=True))
    model = prepare_model_for_kbit_training(model, use_gradient_checkpointing=True)
    model = get_peft_model(model, LoraConfig(task_type="CAUSAL_LM", **lora))

    log = []

    class Log(TrainerCallback):
        def on_log(self, args, state, control, logs=None, **kw):
            if logs:
                log.append({"step": state.global_step, "epoch": round(state.epoch or 0, 3), **logs})

    out_dir = f"/tmp/{run_name}"
    steps = -(-len(data) // (hparams["batch"] * hparams["grad_accum"])) * hparams["epochs"]
    t0 = time.time()
    Trainer(model=model, train_dataset=data, data_collator=collate, callbacks=[Log()],
            args=TrainingArguments(output_dir=out_dir, num_train_epochs=hparams["epochs"],
                                   per_device_train_batch_size=hparams["batch"],
                                   gradient_accumulation_steps=hparams["grad_accum"], learning_rate=hparams["lr"],
                                   lr_scheduler_type="cosine", bf16=True,
                                   warmup_steps=max(1, round(steps * hparams["warmup_ratio"])),
                                   logging_steps=5, save_strategy="no", report_to=[], seed=hparams["seed"],
                                   gradient_checkpointing=True, remove_unused_columns=False)).train()
    minutes = (time.time() - t0) / 60

    model.save_pretrained(out_dir)
    tok.save_pretrained(out_dir)
    repo = f"{HF_USER}/{run_name}"
    api = HfApi()
    api.create_repo(repo, private=False, exist_ok=True)
    info = api.upload_folder(folder_path=out_dir, repo_id=repo, commit_message=f"{run_name}: {len(examples)} examples",
                             ignore_patterns=["checkpoint-*", "runs/*"])
    return {"run": run_name, "repo": repo, "commit": info.oid, "n_examples": len(examples), "train_minutes": round(minutes, 1),
            "base_model": BASE_MODEL, "base_revision": BASE_REVISION, "hparams": hparams, "lora": lora, "log": log}


@app.local_entrypoint()
def main(data: str = "data/train_v1.jsonl", ns: str = "250,500,1000,2000", tag: str = "v1", gpu: str = "L40S"):
    examples = [json.loads(l) for l in open(data, encoding="utf-8")]
    calls = []
    fn = train.with_options(gpu=gpu)
    for n in [int(x) for x in ns.split(",")]:
        run = f"qwen3-4b-tz-convert-{tag}-n{n}"
        calls.append(fn.spawn(examples[:n], run, HPARAMS, LORA))
        print(f"started {run}")
    out = Path("results/training")
    out.mkdir(parents=True, exist_ok=True)
    for c in calls:
        r = c.get()
        r["dataset"], r["gpu"] = data, gpu
        (out / f"{r['run']}.json").write_text(json.dumps(r, indent=1), encoding="utf-8")
        losses = [x["loss"] for x in r["log"] if "loss" in x]
        print(f"{r['run']}: {r['train_minutes']} min, loss {losses[0]:.3f} -> {losses[-1]:.3f}, "
              f"pushed {r['repo']}@{r['commit'][:8]}")