"""Supervised LoRA fine-tuning of Qwen/Qwen3.5-9B into xevyo-01 (the turn planner). Input JSONL rows: {"input": , "target": ""} ("target" may also be the plan object). The chat messages are built here with spec/render.py (render_messages; render_target for an object target), so training, evaluation and serving all use the one renderer. Rows that already carry "messages": [system, user] are accepted too. The prompt is the Qwen chat template with thinking OFF; the loss covers only the target tokens plus the closing <|im_end|>. Run (inside the image built from train/Dockerfile): one GPU : python train/sft_lora.py --config train/configs/r4.yaml two GPUs: torchrun --nproc_per_node 2 train/sft_lora.py --config train/configs/r4.yaml Design notes: * Model class: AutoModelForCausalLM -> Qwen3_5ForCausalLM, the text-only half of the checkpoint. The vision tower is never loaded, so it can never be trained. The checkpoint's mtp.* weights are ignored by the loader. * LoRA targets: every linear projection of the language model. Qwen3.5 mixes 24 Gated DeltaNet layers (linear_attn.in_proj_qkv/in_proj_z/out_proj, plus the tiny 4096->32 gates in_proj_a/in_proj_b) with 8 full-attention layers (self_attn.q/k/v/o_proj) and an MLP in every layer (gate/up/down_proj). * Loss: logits are computed only at positions that predict a target token (logits_to_keep with a boolean mask), so the 248k-vocab head never runs over the long system prompt. This is the main memory saving. * No sequence packing: the DeltaNet layers carry a recurrent state, and keeping packed examples apart needs extra kernels and care. Batches are right-padded and grouped by length instead (train_sampling_strategy=group_by_length). * On a GPU shared with other services (XV01_SHARED_GPU_UUID) this process caps itself with torch.cuda.set_per_process_memory_fraction. The cap is the smaller of 0.78 and what keeps 16 GB of the card free next to the memory the other services hold. * Every run writes run_meta.json (base revision, dataset sha256, config, git commit, library versions) and metrics.jsonl (one JSON object per log/eval event). * The best checkpoint by eval loss is restored at the end and saved to /adapter (PEFT format for Qwen3_5ForCausalLM) and /adapter_vllm (same weights, module names of the full checkpoint, which is what vLLM expects). """ import argparse import hashlib import json import os import random import shutil import sys import time from pathlib import Path import torch import torch.nn.functional as F import yaml sys.path.insert(0, str(Path(__file__).resolve().parents[1])) # repo root, for spec/ from spec.render import render_messages, render_target # A GPU shared with other services: the per-process cap below is applied automatically # on it. Set XV01_SHARED_GPU_UUID to its uuid (nvidia-smi -L); empty = no shared GPU. GPU6_UUID = os.environ.get("XV01_SHARED_GPU_UUID", "") SHARED_GPU_FRACTION = 0.78 # never more than this on the shared GPU SHARED_GPU_KEEP_FREE_GB = 16 # leave this much of the shared GPU unused SHARED_GPU_NON_TORCH_GB = 8 # CUDA context + Triton/cuBLAS memory outside the torch cap # (measured 7.1 GB in the smoke run of 2026-11-13) SHARED_GPU_MIN_FRACTION = 0.30 # below ~24 GB the 9B model cannot train: stop instead THINK_OFF_SUFFIX = "<|im_start|>assistant\n\n\n\n\n" # ── small helpers ───────────────────────────────────────────────────────────── def rank_info(): return int(os.environ.get("RANK", 0)), int(os.environ.get("LOCAL_RANK", 0)), int(os.environ.get("WORLD_SIZE", 1)) def log(msg): r, _, _ = rank_info() print(f"[rank{r} {time.strftime('%H:%M:%S')}] {msg}", flush=True) def sha256_file(path): h = hashlib.sha256() with open(path, "rb") as f: for chunk in iter(lambda: f.read(1 << 20), b""): h.update(chunk) return h.hexdigest() def norm_uuid(u): u = str(u).strip().lower() return u[4:] if u.startswith("gpu-") else u def apply_memory_cap(cfg_fraction): """Cap this process on shared GPUs. Must run before the model touches the GPU.""" dev = torch.cuda.current_device() uuid = norm_uuid(torch.cuda.get_device_properties(dev).uuid) shared = {norm_uuid(u) for u in os.environ.get("XV01_SHARED_GPU_UUIDS", GPU6_UUID).split(",") if u.strip()} fraction = cfg_fraction if fraction is None and os.environ.get("XV01_MEM_FRACTION"): fraction = float(os.environ["XV01_MEM_FRACTION"]) free_b, total_b = torch.cuda.mem_get_info(dev) # whole card, all processes others_gb = (total_b - free_b) / 2**30 if uuid in shared: # set_per_process_memory_fraction counts only OUR process, so 0.78 alone would # let us fill the card to 0.78 plus the share of the live services. The second # term keeps SHARED_GPU_KEEP_FREE_GB free on top of what the others hold now, # after the memory this process uses outside the torch allocator (which the # torch cap does not see). keep_free = (free_b / 2**30 - SHARED_GPU_KEEP_FREE_GB - SHARED_GPU_NON_TORCH_GB) / (total_b / 2**30) fraction = round(min(fraction or SHARED_GPU_FRACTION, SHARED_GPU_FRACTION, keep_free), 3) if fraction < SHARED_GPU_MIN_FRACTION: raise SystemExit(f"shared GPU {uuid}: other processes hold {others_gb:.1f} GB, which leaves a cap of " f"{fraction:.2f}; too little to train. Check nvidia-smi and report.") if fraction: torch.cuda.set_per_process_memory_fraction(fraction, dev) cap_gb = round(fraction * total_b / 2**30, 1) if fraction else None log(f"cuda:{dev} uuid={uuid} shared={uuid in shared} others_gb={others_gb:.1f} " f"memory_fraction={fraction} cap_gb={cap_gb}") return {"device": dev, "uuid": uuid, "shared": uuid in shared, "memory_fraction": fraction, "cap_gb": cap_gb, "others_gb_at_start": round(others_gb, 2), "total_gb": round(total_b / 2**30, 2), "name": torch.cuda.get_device_name(dev)} def device_used_gb(): """Memory in use on the GPU of this process by ALL processes (what nvidia-smi shows).""" free_b, total_b = torch.cuda.mem_get_info() return round((total_b - free_b) / 2**30, 2) def git_commit(): if os.environ.get("XV01_GIT_COMMIT"): return os.environ["XV01_GIT_COMMIT"] p = Path(__file__).resolve().parents[1] / "GIT_COMMIT" return p.read_text().strip() if p.exists() else "unknown" def base_revision(model_path, cfg): if cfg.get("base_revision"): return cfg["base_revision"] parts = Path(model_path).parts return parts[parts.index("snapshots") + 1] if "snapshots" in parts else "unknown" # ── data ────────────────────────────────────────────────────────────────────── def load_jsonl(path): with open(path, encoding="utf-8") as f: return [json.loads(l) for l in f if l.strip()] def row_messages(row): """[system, user] for a row: rendered from "input" (the normal case) or given. Round 4: a row may say "window": "agent" (spec/render.py WINDOWS: the backend's XEVYO01_CONTEXT=agent, 24 earlier messages within 12,000 characters); no key = "trained", the window rounds 1-3 used, byte for byte as before.""" if "input" in row: return render_messages(row["input"], row.get("window", "trained")) msgs = row.get("messages") if not msgs or msgs[-1]["role"] == "assistant": raise ValueError(f"row {row.get('id')}: needs 'input' or 'messages' ending with the user turn; " "the answer goes in 'target'") return msgs def row_target(row): """The target string: canonical plan JSON. A plan object is rendered here.""" target = row["target"] if isinstance(target, dict): return render_target(target) json.loads(target) # a target that is not JSON is a data bug: fail before training return target def encode(row, tok, im_end_id): """prompt ids (template, thinking off) + target ids (+ <|im_end|>); labels mask the prompt. The prompt and the target are tokenized separately, exactly as at serving time: vLLM tokenizes the rendered prompt, the model then emits the target tokens.""" msgs = row_messages(row) prompt = tok.apply_chat_template(msgs, tokenize=False, add_generation_prompt=True, enable_thinking=False) if not prompt.endswith(THINK_OFF_SUFFIX): raise ValueError("chat template did not render the non-thinking assistant prefix; check the tokenizer") p_ids = tok(prompt, add_special_tokens=False)["input_ids"] t_ids = tok(row_target(row), add_special_tokens=False)["input_ids"] + [im_end_id] return {"input_ids": p_ids + t_ids, "labels": [-100] * len(p_ids) + t_ids, "length": len(p_ids) + len(t_ids), "n_prompt": len(p_ids), "n_target": len(t_ids)} def build_dataset(rows, tok, im_end_id, max_len, name): from datasets import Dataset enc, dropped = [], 0 for r in rows: e = encode(r, tok, im_end_id) if e["length"] > max_len: dropped += 1 # never truncate a target: drop the row and report it continue enc.append(e) stats = { "rows": len(enc), "dropped_too_long": dropped, "prompt_tokens_mean": round(sum(e["n_prompt"] for e in enc) / max(len(enc), 1), 1), "prompt_tokens_max": max((e["n_prompt"] for e in enc), default=0), "target_tokens_mean": round(sum(e["n_target"] for e in enc) / max(len(enc), 1), 1), "target_tokens_max": max((e["n_target"] for e in enc), default=0), } log(f"{name}: {stats}") ds = Dataset.from_list([{k: e[k] for k in ("input_ids", "labels", "length")} for e in enc]) return ds, stats class PadCollator: """Right-pad to the longest row (rounded up to a multiple of 16).""" def __init__(self, pad_id): self.pad_id = pad_id def __call__(self, feats): n = max(len(f["input_ids"]) for f in feats) n = (n + 15) // 16 * 16 ids = torch.full((len(feats), n), self.pad_id, dtype=torch.long) lab = torch.full((len(feats), n), -100, dtype=torch.long) att = torch.zeros((len(feats), n), dtype=torch.long) for i, f in enumerate(feats): k = len(f["input_ids"]) ids[i, :k] = torch.tensor(f["input_ids"]) lab[i, :k] = torch.tensor(f["labels"]) att[i, :k] = 1 return {"input_ids": ids, "attention_mask": att, "labels": lab} # ── trainer ─────────────────────────────────────────────────────────────────── def target_logits_and_labels(model, batch): """Logits only at the positions that predict a target token, and those tokens. Returns (logits (n_target, vocab), target (n_target,), targets per row (list)).""" labels = batch["labels"] shift = labels[:, 1:] # token t+1 is predicted at position t keep = torch.zeros_like(labels, dtype=torch.bool) keep[:, :-1] = shift.ne(-100) out = model(input_ids=batch["input_ids"], attention_mask=batch["attention_mask"], logits_to_keep=keep, use_cache=False) logits = out.logits.reshape(-1, out.logits.shape[-1]) return logits, shift[shift.ne(-100)], shift.ne(-100).sum(1).tolist() def eval_loss_of_batch(logits, target, per_row): """Mean over the rows of each row's mean token loss. The Trainer averages batch losses weighted by rows, so with this definition eval_loss is the mean per-example loss whatever the batch size or grouping. (A token mean per batch would change with how rows fall into batches: the Trainer groups eval rows by length too, and that moved the number by 3%.)""" token_loss = F.cross_entropy(logits.float(), target, reduction="none") rows = [chunk.mean() for chunk in torch.split(token_loss, per_row) if chunk.numel()] return torch.stack(rows).mean() def make_trainer_class(): from transformers import Trainer class TargetOnlyTrainer(Trainer): """Computes logits only where a target token is predicted. Training: token-mean CE over the whole (accumulated, all-rank) batch. Evaluation: mean of per-example mean CE (see eval_loss_of_batch).""" def __init__(self, *a, **k): super().__init__(*a, **k) # Our compute_loss normalises by num_items_in_batch itself, so the # Trainer must not divide by the gradient-accumulation steps again. self.model_accepts_loss_kwargs = True self.token_counts = {"tokens": 0, "target_tokens": 0} def compute_loss(self, model, inputs, return_outputs=False, num_items_in_batch=None): logits, target, per_row = target_logits_and_labels(model, inputs) if not model.training: loss = eval_loss_of_batch(logits, target, per_row) return (loss, {"loss": loss}) if return_outputs else loss loss_sum = F.cross_entropy(logits.float(), target, reduction="sum") if num_items_in_batch is not None: loss = loss_sum / num_items_in_batch # num_items_in_batch is summed over all ranks; DDP averages gradients, # so scale back up (same rule as Trainer.compute_loss). if self.args.average_tokens_across_devices and self.args.world_size > 1: loss = loss * self.args.world_size else: loss = loss_sum / max(target.numel(), 1) self.token_counts["tokens"] += int(inputs["attention_mask"].sum()) self.token_counts["target_tokens"] += int(target.numel()) return (loss, {"loss": loss}) if return_outputs else loss return TargetOnlyTrainer def make_metrics_callback(path, trainer_ref): from transformers import TrainerCallback class JsonlMetrics(TrainerCallback): """Appends one JSON object per log/eval event; adds throughput and memory.""" def __init__(self): self.t_last = time.time() self.step_last = 0 self.step_t0 = None self.step_times = [] self.device_peak = 0.0 def on_step_begin(self, args, state, control, **kw): self.step_t0 = time.time() def on_step_end(self, args, state, control, **kw): if self.step_t0 is not None: self.step_times.append(time.time() - self.step_t0) # torch keeps freed blocks reserved, so sampling after each step still sees the peak self.device_peak = max(self.device_peak, device_used_gb()) def on_log(self, args, state, control, logs=None, **kw): if not state.is_world_process_zero: return now = time.time() tr = trainer_ref() counts = tr.token_counts dt = max(now - self.t_last, 1e-6) rec = {"time": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(now)), "step": state.global_step, "epoch": round(state.epoch or 0, 4), **(logs or {})} if "loss" in (logs or {}) and self.step_times: ws = args.world_size rec.update({ "step_time_s": round(sum(self.step_times) / len(self.step_times), 3), # rank 0 counts its own batches; x world_size estimates the total "tokens_per_s_rank0": round(counts["tokens"] / dt, 1), "tokens_per_s_total_est": round(counts["tokens"] * ws / dt, 1), "target_tokens_per_s_total_est": round(counts["target_tokens"] * ws / dt, 1), }) self.step_times = [] counts["tokens"] = counts["target_tokens"] = 0 self.t_last = now rec["mem_alloc_peak_gb_rank0"] = round(torch.cuda.max_memory_allocated() / 2**30, 2) rec["mem_reserved_peak_gb_rank0"] = round(torch.cuda.max_memory_reserved() / 2**30, 2) rec["device_used_peak_gb_rank0"] = max(self.device_peak, device_used_gb()) # all processes on the card with open(path, "a") as f: f.write(json.dumps(rec) + "\n") return JsonlMetrics() # ── adapter export ──────────────────────────────────────────────────────────── def export_vllm_adapter(src, dst): """Copy a PEFT adapter trained on Qwen3_5ForCausalLM (`model.layers.N...`) to the module names of the full checkpoint (`model.language_model.layers.N...`). vLLM loads Qwen3.5 as Qwen3_5ForConditionalGeneration and maps `model.language_model.` -> `language_model.model.`. It only checks the last name part of each LoRA module, so an adapter with the wrong prefix loads without error and changes nothing. Always serve the exported copy.""" from safetensors.torch import load_file, save_file src, dst = Path(src), Path(dst) dst.mkdir(parents=True, exist_ok=True) tensors = load_file(str(src / "adapter_model.safetensors")) old, new = "base_model.model.model.layers.", "base_model.model.model.language_model.layers." out = {} for k, v in tensors.items(): if not k.startswith(old): raise ValueError(f"unexpected adapter key {k}") out[new + k[len(old):]] = v.contiguous() save_file(out, str(dst / "adapter_model.safetensors"), metadata={"format": "pt"}) cfg = json.loads((src / "adapter_config.json").read_text()) (dst / "adapter_config.json").write_text(json.dumps(cfg, indent=2)) return len(out) # ── main ────────────────────────────────────────────────────────────────────── def main(): ap = argparse.ArgumentParser() ap.add_argument("--config", required=True) ap.add_argument("--run-dir", help="default: /") ap.add_argument("--overwrite", action="store_true") a = ap.parse_args() cfg = yaml.safe_load(open(a.config)) rank, local_rank, world = rank_info() torch.cuda.set_device(local_rank) gpu = apply_memory_cap(cfg.get("memory_fraction")) # before any CUDA allocation from peft import LoraConfig, get_peft_model from transformers import AutoModelForCausalLM, AutoTokenizer, TrainingArguments, set_seed run_dir = Path(a.run_dir or Path(cfg["output_root"]) / cfg["run_name"]) if rank == 0: if (run_dir / "adapter").exists() and not a.overwrite: raise SystemExit(f"{run_dir} already has a finished adapter; use --overwrite or another --run-dir") run_dir.mkdir(parents=True, exist_ok=True) shutil.copy(a.config, run_dir / "config.yaml") tcfg = cfg["train"] set_seed(tcfg.get("seed", 42)) model_path = cfg["model_path"] tok = AutoTokenizer.from_pretrained(model_path) im_end_id = tok.convert_tokens_to_ids("<|im_end|>") # Data: explicit eval file, or a seeded split of the train file. train_rows = load_jsonl(cfg["train_file"]) if cfg.get("eval_file"): eval_rows = load_jsonl(cfg["eval_file"]) else: rng = random.Random(tcfg.get("seed", 42)) rng.shuffle(train_rows) n_eval = max(1, int(len(train_rows) * cfg.get("eval_fraction", 0.05))) eval_rows, train_rows = train_rows[:n_eval], train_rows[n_eval:] max_len = cfg.get("max_seq_len", 4096) train_ds, train_stats = build_dataset(train_rows, tok, im_end_id, max_len, "train") eval_ds, eval_stats = build_dataset(eval_rows, tok, im_end_id, max_len, "eval") # Model: text-only causal LM in bf16; SDPA for the 8 full-attention layers. model = AutoModelForCausalLM.from_pretrained(model_path, dtype=torch.bfloat16, attn_implementation="sdpa") model.config.use_cache = False assert type(model).__name__ == "Qwen3_5ForCausalLM", type(model).__name__ assert not any("visual" in n for n, _ in model.named_parameters()), "vision weights loaded" lcfg = cfg["lora"] lora = LoraConfig(r=lcfg["r"], lora_alpha=lcfg["alpha"], lora_dropout=lcfg.get("dropout", 0.0), target_modules=lcfg["target_modules"], bias="none", task_type="CAUSAL_LM") model = get_peft_model(model, lora) trainable = sum(p.numel() for p in model.parameters() if p.requires_grad) total = sum(p.numel() for p in model.parameters()) wrapped = sorted({n.split(".lora_A")[0].rsplit(".", 1)[-1] for n, _ in model.named_parameters() if ".lora_A" in n}) log(f"LoRA r={lora.r} alpha={lora.lora_alpha} trainable={trainable/1e6:.1f}M of {total/1e9:.3f}B; modules={wrapped}") args = TrainingArguments( output_dir=str(run_dir / "checkpoints"), learning_rate=float(tcfg["learning_rate"]), num_train_epochs=tcfg.get("num_train_epochs", 2), max_steps=tcfg.get("max_steps", -1), per_device_train_batch_size=tcfg["per_device_train_batch_size"], per_device_eval_batch_size=tcfg.get("per_device_eval_batch_size", tcfg["per_device_train_batch_size"]), gradient_accumulation_steps=tcfg.get("gradient_accumulation_steps", 1), lr_scheduler_type=tcfg.get("lr_scheduler_type", "cosine"), warmup_steps=tcfg.get("warmup_steps", 0), weight_decay=tcfg.get("weight_decay", 0.0), max_grad_norm=tcfg.get("max_grad_norm", 1.0), bf16=True, gradient_checkpointing=True, gradient_checkpointing_kwargs={"use_reentrant": False}, train_sampling_strategy="group_by_length" if tcfg.get("group_by_length", True) else "random", eval_strategy="steps", eval_steps=tcfg["eval_steps"], save_strategy="steps", save_steps=tcfg["eval_steps"], save_total_limit=tcfg.get("save_total_limit", 2), load_best_model_at_end=True, metric_for_best_model="eval_loss", greater_is_better=False, logging_steps=tcfg.get("logging_steps", 10), logging_first_step=True, label_names=["labels"], prediction_loss_only=True, remove_unused_columns=False, dataloader_num_workers=2, ddp_find_unused_parameters=False, optim=tcfg.get("optim", "adamw_torch_fused"), seed=tcfg.get("seed", 42), report_to=[], disable_tqdm=True, ) meta = { "run_dir": str(run_dir), "started_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "base_model": model_path, "base_revision": base_revision(model_path, cfg), "train_file": cfg["train_file"], "train_sha256": sha256_file(cfg["train_file"]), "eval_file": cfg.get("eval_file"), "eval_sha256": sha256_file(cfg["eval_file"]) if cfg.get("eval_file") else None, "train_stats": train_stats, "eval_stats": eval_stats, "config": cfg, "git_commit": git_commit(), "world_size": world, "gpu_rank0": gpu, "trainable_params": trainable, "lora_modules": wrapped, "versions": {m: __import__(m).__version__ for m in ("torch", "transformers", "peft", "accelerate", "datasets")}, } for opt in ("fla", "causal_conv1d", "trl"): try: meta["versions"][opt] = __import__(opt).__version__ except Exception: meta["versions"][opt] = None if rank == 0: (run_dir / "run_meta.json").write_text(json.dumps(meta, indent=2)) Trainer = make_trainer_class() trainer = Trainer(model=model, args=args, train_dataset=train_ds, eval_dataset=eval_ds, data_collator=PadCollator(tok.pad_token_id), processing_class=tok) import weakref trainer.add_callback(make_metrics_callback(run_dir / "metrics.jsonl", weakref.ref(trainer))) t0 = time.time() result = trainer.train() final_eval = trainer.evaluate() # on the best checkpoint (load_best_model_at_end) peak = {"alloc_gb": round(torch.cuda.max_memory_allocated() / 2**30, 2), "reserved_gb": round(torch.cuda.max_memory_reserved() / 2**30, 2), "device_used_gb_all_processes": device_used_gb(), "device_total_gb": gpu["total_gb"], "device_uuid": gpu["uuid"], "shared": gpu["shared"], "memory_fraction": gpu["memory_fraction"], "cap_gb": gpu["cap_gb"]} run_dir.mkdir(parents=True, exist_ok=True) (run_dir / f"peak_memory_rank{rank}.json").write_text(json.dumps(peak)) if trainer.is_world_process_zero(): trainer.model.save_pretrained(str(run_dir / "adapter")) tok.save_pretrained(str(run_dir / "adapter")) n = export_vllm_adapter(run_dir / "adapter", run_dir / "adapter_vllm") summary = { "finished_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "wall_s": round(time.time() - t0, 1), "train_metrics": result.metrics, "best_checkpoint": trainer.state.best_model_checkpoint, "best_eval_loss": trainer.state.best_metric, "final_eval": final_eval, "peak_memory_rank0": peak, "adapter_sha256": sha256_file(run_dir / "adapter" / "adapter_model.safetensors"), "adapter_vllm_tensors": n, } (run_dir / "summary.json").write_text(json.dumps(summary, indent=2)) log(f"done: {json.dumps(summary)}") if __name__ == "__main__": main()