xevyo-01 / code /train /sft_lora.py
meet4150's picture
xevyo-01 round 4: merged weights, adapter, model card, code
a46f204 verified
Raw History Blame Contribute Delete
25.7 kB
"""Supervised LoRA fine-tuning of Qwen/Qwen3.5-9B into xevyo-01 (the turn planner).
Input JSONL rows: {"input": <example_input>, "target": "<canonical plan JSON>"}
("target" may also be the plan object). The chat messages are built here with
spec/render.py (render_messages; render_target for an object target), so training,
evaluation and serving all use the one renderer. Rows that already carry
"messages": [system, user] are accepted too. The prompt is the Qwen chat template
with thinking OFF; the loss covers only the target tokens plus the closing <|im_end|>.
Run (inside the image built from train/Dockerfile):
one GPU : python train/sft_lora.py --config train/configs/r4.yaml
two GPUs: torchrun --nproc_per_node 2 train/sft_lora.py --config train/configs/r4.yaml
Design notes:
* Model class: AutoModelForCausalLM -> Qwen3_5ForCausalLM, the text-only half of the
checkpoint. The vision tower is never loaded, so it can never be trained.
The checkpoint's mtp.* weights are ignored by the loader.
* LoRA targets: every linear projection of the language model. Qwen3.5 mixes 24
Gated DeltaNet layers (linear_attn.in_proj_qkv/in_proj_z/out_proj, plus the tiny
4096->32 gates in_proj_a/in_proj_b) with 8 full-attention layers
(self_attn.q/k/v/o_proj) and an MLP in every layer (gate/up/down_proj).
* Loss: logits are computed only at positions that predict a target token
(logits_to_keep with a boolean mask), so the 248k-vocab head never runs over the
long system prompt. This is the main memory saving.
* No sequence packing: the DeltaNet layers carry a recurrent state, and keeping
packed examples apart needs extra kernels and care. Batches are right-padded and
grouped by length instead (train_sampling_strategy=group_by_length).
* On a GPU shared with other services (XV01_SHARED_GPU_UUID) this process
caps itself with torch.cuda.set_per_process_memory_fraction. The cap is the
smaller of 0.78 and what keeps 16 GB of the card free next to the memory the
other services hold.
* Every run writes run_meta.json (base revision, dataset sha256, config, git commit,
library versions) and metrics.jsonl (one JSON object per log/eval event).
* The best checkpoint by eval loss is restored at the end and saved to
<run_dir>/adapter (PEFT format for Qwen3_5ForCausalLM) and <run_dir>/adapter_vllm
(same weights, module names of the full checkpoint, which is what vLLM expects).
"""
import argparse
import hashlib
import json
import os
import random
import shutil
import sys
import time
from pathlib import Path
import torch
import torch.nn.functional as F
import yaml
sys.path.insert(0, str(Path(__file__).resolve().parents[1])) # repo root, for spec/
from spec.render import render_messages, render_target
# A GPU shared with other services: the per-process cap below is applied automatically
# on it. Set XV01_SHARED_GPU_UUID to its uuid (nvidia-smi -L); empty = no shared GPU.
GPU6_UUID = os.environ.get("XV01_SHARED_GPU_UUID", "")
SHARED_GPU_FRACTION = 0.78 # never more than this on the shared GPU
SHARED_GPU_KEEP_FREE_GB = 16 # leave this much of the shared GPU unused
SHARED_GPU_NON_TORCH_GB = 8 # CUDA context + Triton/cuBLAS memory outside the torch cap
# (measured 7.1 GB in the smoke run of 2026-11-13)
SHARED_GPU_MIN_FRACTION = 0.30 # below ~24 GB the 9B model cannot train: stop instead
THINK_OFF_SUFFIX = "<|im_start|>assistant\n<think>\n\n</think>\n\n"
# ── small helpers ─────────────────────────────────────────────────────────────
def rank_info():
return int(os.environ.get("RANK", 0)), int(os.environ.get("LOCAL_RANK", 0)), int(os.environ.get("WORLD_SIZE", 1))
def log(msg):
r, _, _ = rank_info()
print(f"[rank{r} {time.strftime('%H:%M:%S')}] {msg}", flush=True)
def sha256_file(path):
h = hashlib.sha256()
with open(path, "rb") as f:
for chunk in iter(lambda: f.read(1 << 20), b""):
h.update(chunk)
return h.hexdigest()
def norm_uuid(u):
u = str(u).strip().lower()
return u[4:] if u.startswith("gpu-") else u
def apply_memory_cap(cfg_fraction):
"""Cap this process on shared GPUs. Must run before the model touches the GPU."""
dev = torch.cuda.current_device()
uuid = norm_uuid(torch.cuda.get_device_properties(dev).uuid)
shared = {norm_uuid(u) for u in os.environ.get("XV01_SHARED_GPU_UUIDS", GPU6_UUID).split(",") if u.strip()}
fraction = cfg_fraction
if fraction is None and os.environ.get("XV01_MEM_FRACTION"):
fraction = float(os.environ["XV01_MEM_FRACTION"])
free_b, total_b = torch.cuda.mem_get_info(dev) # whole card, all processes
others_gb = (total_b - free_b) / 2**30
if uuid in shared:
# set_per_process_memory_fraction counts only OUR process, so 0.78 alone would
# let us fill the card to 0.78 plus the share of the live services. The second
# term keeps SHARED_GPU_KEEP_FREE_GB free on top of what the others hold now,
# after the memory this process uses outside the torch allocator (which the
# torch cap does not see).
keep_free = (free_b / 2**30 - SHARED_GPU_KEEP_FREE_GB - SHARED_GPU_NON_TORCH_GB) / (total_b / 2**30)
fraction = round(min(fraction or SHARED_GPU_FRACTION, SHARED_GPU_FRACTION, keep_free), 3)
if fraction < SHARED_GPU_MIN_FRACTION:
raise SystemExit(f"shared GPU {uuid}: other processes hold {others_gb:.1f} GB, which leaves a cap of "
f"{fraction:.2f}; too little to train. Check nvidia-smi and report.")
if fraction:
torch.cuda.set_per_process_memory_fraction(fraction, dev)
cap_gb = round(fraction * total_b / 2**30, 1) if fraction else None
log(f"cuda:{dev} uuid={uuid} shared={uuid in shared} others_gb={others_gb:.1f} "
f"memory_fraction={fraction} cap_gb={cap_gb}")
return {"device": dev, "uuid": uuid, "shared": uuid in shared, "memory_fraction": fraction,
"cap_gb": cap_gb, "others_gb_at_start": round(others_gb, 2),
"total_gb": round(total_b / 2**30, 2), "name": torch.cuda.get_device_name(dev)}
def device_used_gb():
"""Memory in use on the GPU of this process by ALL processes (what nvidia-smi shows)."""
free_b, total_b = torch.cuda.mem_get_info()
return round((total_b - free_b) / 2**30, 2)
def git_commit():
if os.environ.get("XV01_GIT_COMMIT"):
return os.environ["XV01_GIT_COMMIT"]
p = Path(__file__).resolve().parents[1] / "GIT_COMMIT"
return p.read_text().strip() if p.exists() else "unknown"
def base_revision(model_path, cfg):
if cfg.get("base_revision"):
return cfg["base_revision"]
parts = Path(model_path).parts
return parts[parts.index("snapshots") + 1] if "snapshots" in parts else "unknown"
# ── data ──────────────────────────────────────────────────────────────────────
def load_jsonl(path):
with open(path, encoding="utf-8") as f:
return [json.loads(l) for l in f if l.strip()]
def row_messages(row):
"""[system, user] for a row: rendered from "input" (the normal case) or given. Round 4: a row may say
"window": "agent" (spec/render.py WINDOWS: the backend's XEVYO01_CONTEXT=agent, 24 earlier messages within
12,000 characters); no key = "trained", the window rounds 1-3 used, byte for byte as before."""
if "input" in row:
return render_messages(row["input"], row.get("window", "trained"))
msgs = row.get("messages")
if not msgs or msgs[-1]["role"] == "assistant":
raise ValueError(f"row {row.get('id')}: needs 'input' or 'messages' ending with the user turn; "
"the answer goes in 'target'")
return msgs
def row_target(row):
"""The target string: canonical plan JSON. A plan object is rendered here."""
target = row["target"]
if isinstance(target, dict):
return render_target(target)
json.loads(target) # a target that is not JSON is a data bug: fail before training
return target
def encode(row, tok, im_end_id):
"""prompt ids (template, thinking off) + target ids (+ <|im_end|>); labels mask the prompt.
The prompt and the target are tokenized separately, exactly as at serving time:
vLLM tokenizes the rendered prompt, the model then emits the target tokens."""
msgs = row_messages(row)
prompt = tok.apply_chat_template(msgs, tokenize=False, add_generation_prompt=True, enable_thinking=False)
if not prompt.endswith(THINK_OFF_SUFFIX):
raise ValueError("chat template did not render the non-thinking assistant prefix; check the tokenizer")
p_ids = tok(prompt, add_special_tokens=False)["input_ids"]
t_ids = tok(row_target(row), add_special_tokens=False)["input_ids"] + [im_end_id]
return {"input_ids": p_ids + t_ids, "labels": [-100] * len(p_ids) + t_ids,
"length": len(p_ids) + len(t_ids), "n_prompt": len(p_ids), "n_target": len(t_ids)}
def build_dataset(rows, tok, im_end_id, max_len, name):
from datasets import Dataset
enc, dropped = [], 0
for r in rows:
e = encode(r, tok, im_end_id)
if e["length"] > max_len:
dropped += 1 # never truncate a target: drop the row and report it
continue
enc.append(e)
stats = {
"rows": len(enc), "dropped_too_long": dropped,
"prompt_tokens_mean": round(sum(e["n_prompt"] for e in enc) / max(len(enc), 1), 1),
"prompt_tokens_max": max((e["n_prompt"] for e in enc), default=0),
"target_tokens_mean": round(sum(e["n_target"] for e in enc) / max(len(enc), 1), 1),
"target_tokens_max": max((e["n_target"] for e in enc), default=0),
}
log(f"{name}: {stats}")
ds = Dataset.from_list([{k: e[k] for k in ("input_ids", "labels", "length")} for e in enc])
return ds, stats
class PadCollator:
"""Right-pad to the longest row (rounded up to a multiple of 16)."""
def __init__(self, pad_id):
self.pad_id = pad_id
def __call__(self, feats):
n = max(len(f["input_ids"]) for f in feats)
n = (n + 15) // 16 * 16
ids = torch.full((len(feats), n), self.pad_id, dtype=torch.long)
lab = torch.full((len(feats), n), -100, dtype=torch.long)
att = torch.zeros((len(feats), n), dtype=torch.long)
for i, f in enumerate(feats):
k = len(f["input_ids"])
ids[i, :k] = torch.tensor(f["input_ids"])
lab[i, :k] = torch.tensor(f["labels"])
att[i, :k] = 1
return {"input_ids": ids, "attention_mask": att, "labels": lab}
# ── trainer ───────────────────────────────────────────────────────────────────
def target_logits_and_labels(model, batch):
"""Logits only at the positions that predict a target token, and those tokens.
Returns (logits (n_target, vocab), target (n_target,), targets per row (list))."""
labels = batch["labels"]
shift = labels[:, 1:] # token t+1 is predicted at position t
keep = torch.zeros_like(labels, dtype=torch.bool)
keep[:, :-1] = shift.ne(-100)
out = model(input_ids=batch["input_ids"], attention_mask=batch["attention_mask"],
logits_to_keep=keep, use_cache=False)
logits = out.logits.reshape(-1, out.logits.shape[-1])
return logits, shift[shift.ne(-100)], shift.ne(-100).sum(1).tolist()
def eval_loss_of_batch(logits, target, per_row):
"""Mean over the rows of each row's mean token loss.
The Trainer averages batch losses weighted by rows, so with this definition
eval_loss is the mean per-example loss whatever the batch size or grouping.
(A token mean per batch would change with how rows fall into batches: the
Trainer groups eval rows by length too, and that moved the number by 3%.)"""
token_loss = F.cross_entropy(logits.float(), target, reduction="none")
rows = [chunk.mean() for chunk in torch.split(token_loss, per_row) if chunk.numel()]
return torch.stack(rows).mean()
def make_trainer_class():
from transformers import Trainer
class TargetOnlyTrainer(Trainer):
"""Computes logits only where a target token is predicted.
Training: token-mean CE over the whole (accumulated, all-rank) batch.
Evaluation: mean of per-example mean CE (see eval_loss_of_batch)."""
def __init__(self, *a, **k):
super().__init__(*a, **k)
# Our compute_loss normalises by num_items_in_batch itself, so the
# Trainer must not divide by the gradient-accumulation steps again.
self.model_accepts_loss_kwargs = True
self.token_counts = {"tokens": 0, "target_tokens": 0}
def compute_loss(self, model, inputs, return_outputs=False, num_items_in_batch=None):
logits, target, per_row = target_logits_and_labels(model, inputs)
if not model.training:
loss = eval_loss_of_batch(logits, target, per_row)
return (loss, {"loss": loss}) if return_outputs else loss
loss_sum = F.cross_entropy(logits.float(), target, reduction="sum")
if num_items_in_batch is not None:
loss = loss_sum / num_items_in_batch
# num_items_in_batch is summed over all ranks; DDP averages gradients,
# so scale back up (same rule as Trainer.compute_loss).
if self.args.average_tokens_across_devices and self.args.world_size > 1:
loss = loss * self.args.world_size
else:
loss = loss_sum / max(target.numel(), 1)
self.token_counts["tokens"] += int(inputs["attention_mask"].sum())
self.token_counts["target_tokens"] += int(target.numel())
return (loss, {"loss": loss}) if return_outputs else loss
return TargetOnlyTrainer
def make_metrics_callback(path, trainer_ref):
from transformers import TrainerCallback
class JsonlMetrics(TrainerCallback):
"""Appends one JSON object per log/eval event; adds throughput and memory."""
def __init__(self):
self.t_last = time.time()
self.step_last = 0
self.step_t0 = None
self.step_times = []
self.device_peak = 0.0
def on_step_begin(self, args, state, control, **kw):
self.step_t0 = time.time()
def on_step_end(self, args, state, control, **kw):
if self.step_t0 is not None:
self.step_times.append(time.time() - self.step_t0)
# torch keeps freed blocks reserved, so sampling after each step still sees the peak
self.device_peak = max(self.device_peak, device_used_gb())
def on_log(self, args, state, control, logs=None, **kw):
if not state.is_world_process_zero:
return
now = time.time()
tr = trainer_ref()
counts = tr.token_counts
dt = max(now - self.t_last, 1e-6)
rec = {"time": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(now)), "step": state.global_step,
"epoch": round(state.epoch or 0, 4), **(logs or {})}
if "loss" in (logs or {}) and self.step_times:
ws = args.world_size
rec.update({
"step_time_s": round(sum(self.step_times) / len(self.step_times), 3),
# rank 0 counts its own batches; x world_size estimates the total
"tokens_per_s_rank0": round(counts["tokens"] / dt, 1),
"tokens_per_s_total_est": round(counts["tokens"] * ws / dt, 1),
"target_tokens_per_s_total_est": round(counts["target_tokens"] * ws / dt, 1),
})
self.step_times = []
counts["tokens"] = counts["target_tokens"] = 0
self.t_last = now
rec["mem_alloc_peak_gb_rank0"] = round(torch.cuda.max_memory_allocated() / 2**30, 2)
rec["mem_reserved_peak_gb_rank0"] = round(torch.cuda.max_memory_reserved() / 2**30, 2)
rec["device_used_peak_gb_rank0"] = max(self.device_peak, device_used_gb()) # all processes on the card
with open(path, "a") as f:
f.write(json.dumps(rec) + "\n")
return JsonlMetrics()
# ── adapter export ────────────────────────────────────────────────────────────
def export_vllm_adapter(src, dst):
"""Copy a PEFT adapter trained on Qwen3_5ForCausalLM (`model.layers.N...`) to the
module names of the full checkpoint (`model.language_model.layers.N...`).
vLLM loads Qwen3.5 as Qwen3_5ForConditionalGeneration and maps
`model.language_model.` -> `language_model.model.`. It only checks the last name
part of each LoRA module, so an adapter with the wrong prefix loads without error
and changes nothing. Always serve the exported copy."""
from safetensors.torch import load_file, save_file
src, dst = Path(src), Path(dst)
dst.mkdir(parents=True, exist_ok=True)
tensors = load_file(str(src / "adapter_model.safetensors"))
old, new = "base_model.model.model.layers.", "base_model.model.model.language_model.layers."
out = {}
for k, v in tensors.items():
if not k.startswith(old):
raise ValueError(f"unexpected adapter key {k}")
out[new + k[len(old):]] = v.contiguous()
save_file(out, str(dst / "adapter_model.safetensors"), metadata={"format": "pt"})
cfg = json.loads((src / "adapter_config.json").read_text())
(dst / "adapter_config.json").write_text(json.dumps(cfg, indent=2))
return len(out)
# ── main ──────────────────────────────────────────────────────────────────────
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--config", required=True)
ap.add_argument("--run-dir", help="default: <output_root>/<run_name>")
ap.add_argument("--overwrite", action="store_true")
a = ap.parse_args()
cfg = yaml.safe_load(open(a.config))
rank, local_rank, world = rank_info()
torch.cuda.set_device(local_rank)
gpu = apply_memory_cap(cfg.get("memory_fraction")) # before any CUDA allocation
from peft import LoraConfig, get_peft_model
from transformers import AutoModelForCausalLM, AutoTokenizer, TrainingArguments, set_seed
run_dir = Path(a.run_dir or Path(cfg["output_root"]) / cfg["run_name"])
if rank == 0:
if (run_dir / "adapter").exists() and not a.overwrite:
raise SystemExit(f"{run_dir} already has a finished adapter; use --overwrite or another --run-dir")
run_dir.mkdir(parents=True, exist_ok=True)
shutil.copy(a.config, run_dir / "config.yaml")
tcfg = cfg["train"]
set_seed(tcfg.get("seed", 42))
model_path = cfg["model_path"]
tok = AutoTokenizer.from_pretrained(model_path)
im_end_id = tok.convert_tokens_to_ids("<|im_end|>")
# Data: explicit eval file, or a seeded split of the train file.
train_rows = load_jsonl(cfg["train_file"])
if cfg.get("eval_file"):
eval_rows = load_jsonl(cfg["eval_file"])
else:
rng = random.Random(tcfg.get("seed", 42))
rng.shuffle(train_rows)
n_eval = max(1, int(len(train_rows) * cfg.get("eval_fraction", 0.05)))
eval_rows, train_rows = train_rows[:n_eval], train_rows[n_eval:]
max_len = cfg.get("max_seq_len", 4096)
train_ds, train_stats = build_dataset(train_rows, tok, im_end_id, max_len, "train")
eval_ds, eval_stats = build_dataset(eval_rows, tok, im_end_id, max_len, "eval")
# Model: text-only causal LM in bf16; SDPA for the 8 full-attention layers.
model = AutoModelForCausalLM.from_pretrained(model_path, dtype=torch.bfloat16, attn_implementation="sdpa")
model.config.use_cache = False
assert type(model).__name__ == "Qwen3_5ForCausalLM", type(model).__name__
assert not any("visual" in n for n, _ in model.named_parameters()), "vision weights loaded"
lcfg = cfg["lora"]
lora = LoraConfig(r=lcfg["r"], lora_alpha=lcfg["alpha"], lora_dropout=lcfg.get("dropout", 0.0),
target_modules=lcfg["target_modules"], bias="none", task_type="CAUSAL_LM")
model = get_peft_model(model, lora)
trainable = sum(p.numel() for p in model.parameters() if p.requires_grad)
total = sum(p.numel() for p in model.parameters())
wrapped = sorted({n.split(".lora_A")[0].rsplit(".", 1)[-1] for n, _ in model.named_parameters() if ".lora_A" in n})
log(f"LoRA r={lora.r} alpha={lora.lora_alpha} trainable={trainable/1e6:.1f}M of {total/1e9:.3f}B; modules={wrapped}")
args = TrainingArguments(
output_dir=str(run_dir / "checkpoints"),
learning_rate=float(tcfg["learning_rate"]),
num_train_epochs=tcfg.get("num_train_epochs", 2),
max_steps=tcfg.get("max_steps", -1),
per_device_train_batch_size=tcfg["per_device_train_batch_size"],
per_device_eval_batch_size=tcfg.get("per_device_eval_batch_size", tcfg["per_device_train_batch_size"]),
gradient_accumulation_steps=tcfg.get("gradient_accumulation_steps", 1),
lr_scheduler_type=tcfg.get("lr_scheduler_type", "cosine"),
warmup_steps=tcfg.get("warmup_steps", 0),
weight_decay=tcfg.get("weight_decay", 0.0),
max_grad_norm=tcfg.get("max_grad_norm", 1.0),
bf16=True,
gradient_checkpointing=True,
gradient_checkpointing_kwargs={"use_reentrant": False},
train_sampling_strategy="group_by_length" if tcfg.get("group_by_length", True) else "random",
eval_strategy="steps", eval_steps=tcfg["eval_steps"],
save_strategy="steps", save_steps=tcfg["eval_steps"], save_total_limit=tcfg.get("save_total_limit", 2),
load_best_model_at_end=True, metric_for_best_model="eval_loss", greater_is_better=False,
logging_steps=tcfg.get("logging_steps", 10), logging_first_step=True,
label_names=["labels"], prediction_loss_only=True, remove_unused_columns=False,
dataloader_num_workers=2, ddp_find_unused_parameters=False,
optim=tcfg.get("optim", "adamw_torch_fused"), seed=tcfg.get("seed", 42),
report_to=[], disable_tqdm=True,
)
meta = {
"run_dir": str(run_dir), "started_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
"base_model": model_path, "base_revision": base_revision(model_path, cfg),
"train_file": cfg["train_file"], "train_sha256": sha256_file(cfg["train_file"]),
"eval_file": cfg.get("eval_file"), "eval_sha256": sha256_file(cfg["eval_file"]) if cfg.get("eval_file") else None,
"train_stats": train_stats, "eval_stats": eval_stats, "config": cfg, "git_commit": git_commit(),
"world_size": world, "gpu_rank0": gpu, "trainable_params": trainable, "lora_modules": wrapped,
"versions": {m: __import__(m).__version__ for m in ("torch", "transformers", "peft", "accelerate", "datasets")},
}
for opt in ("fla", "causal_conv1d", "trl"):
try:
meta["versions"][opt] = __import__(opt).__version__
except Exception:
meta["versions"][opt] = None
if rank == 0:
(run_dir / "run_meta.json").write_text(json.dumps(meta, indent=2))
Trainer = make_trainer_class()
trainer = Trainer(model=model, args=args, train_dataset=train_ds, eval_dataset=eval_ds,
data_collator=PadCollator(tok.pad_token_id), processing_class=tok)
import weakref
trainer.add_callback(make_metrics_callback(run_dir / "metrics.jsonl", weakref.ref(trainer)))
t0 = time.time()
result = trainer.train()
final_eval = trainer.evaluate() # on the best checkpoint (load_best_model_at_end)
peak = {"alloc_gb": round(torch.cuda.max_memory_allocated() / 2**30, 2),
"reserved_gb": round(torch.cuda.max_memory_reserved() / 2**30, 2),
"device_used_gb_all_processes": device_used_gb(), "device_total_gb": gpu["total_gb"],
"device_uuid": gpu["uuid"], "shared": gpu["shared"],
"memory_fraction": gpu["memory_fraction"], "cap_gb": gpu["cap_gb"]}
run_dir.mkdir(parents=True, exist_ok=True)
(run_dir / f"peak_memory_rank{rank}.json").write_text(json.dumps(peak))
if trainer.is_world_process_zero():
trainer.model.save_pretrained(str(run_dir / "adapter"))
tok.save_pretrained(str(run_dir / "adapter"))
n = export_vllm_adapter(run_dir / "adapter", run_dir / "adapter_vllm")
summary = {
"finished_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
"wall_s": round(time.time() - t0, 1), "train_metrics": result.metrics,
"best_checkpoint": trainer.state.best_model_checkpoint, "best_eval_loss": trainer.state.best_metric,
"final_eval": final_eval, "peak_memory_rank0": peak,
"adapter_sha256": sha256_file(run_dir / "adapter" / "adapter_model.safetensors"),
"adapter_vllm_tensors": n,
}
(run_dir / "summary.json").write_text(json.dumps(summary, indent=2))
log(f"done: {json.dumps(summary)}")
if __name__ == "__main__":
main()