""" Fine-tuning mode: "what can I TRAIN on this machine?" — the mirror of the inference advisor. Running a model and fine-tuning it have wildly different memory costs (a 7B chats in ~5 GB but QLoRA-trains in ~12 GB, LoRA in ~21 GB, full fine-tune in ~130 GB), so the honest answer is a different one. The memory model is deterministic and conservative, mirroring the inference engine's philosophy. Per-parameter byte constants and the calibration are sourced (see FINETUNE-RESEARCH below); FitCheck deliberately lands at or above Unsloth's published VRAM minimums, because most users run the vanilla PEFT/TRL/bitsandbytes stack, not Unsloth's memory-optimised kernels. Sources: - Full FT = 16 bytes/trainable-param (fp16 weight+grad + fp32 master/mom/var): EleutherAI Transformer Math; Google Cloud GPU-memory guide. - QLoRA 4-bit NF4+double-quant base ~= 4.5 bits/param; paged-8bit optimiser on the <1% trainable adapter: Dettmers et al. 2023 (arXiv:2305.14314). - Activation term + gradient-checkpointing divisor calibrated so QLoRA totals sit above Unsloth's requirements table. """ from .hardware import HardwareSpec from .real_advisor import ( USE_CASES, _SAFETY_FILL, catalogue, _by_use_case, catalogue_date, _C_MODEL, _C_WORK, _VERDICT_WORD, ) # Per-parameter core cost (weights + gradients + optimiser state), bytes/param. # GPU-independent (set by the model + method, not the card). LoRA = 16-bit frozen # base + tiny adapter; QLoRA = ~4.5-bit base + adapter; full = fp16 weight+grad + # fp32 master/momentum/variance. _CORE_BYTES = {"full": 16.0, "lora": 2.0 + 0.16, "qlora": 0.5625 + 0.16} _METHOD_PLAIN = { "qlora": "QLoRA (4-bit base + adapters)", "lora": "LoRA (16-bit base + adapters)", "full": "Full fine-tune (all weights)", } # Activation-memory model, CALIBRATED on a first-party RTX 5090 sweep (57 unique # Qwen2.5 configs; scripts/measure_finetune_vram.py) with LIMITED external spot # checks against a few published anchors on other GPUs / one other family # (scripts/fit_finetune_vram.py). This is NOT a fold-based cross-validation and # is single-GPU/single-family calibration -- treat it as such, not as proven # cross-hardware accuracy. # Architecture-aware (NOT a flat GB/param), because the dominant activation is # the cross-entropy logits tensor (batch*seq*VOCAB*4 bytes) which is # param-INDEPENDENT and vocab-driven -- the exact term a per-param formula misses # and the reason the old 0.6 GB/B constant under-predicted long-context/large- # batch runs by up to ~21 GB. Basis: nvidia-smi peak, VANILLA HF stack (eager # attention + standard fp32 cross-entropy) = the conservative HIGH end of the # band. `efficient=True` (flash/SDPA + fused cross-entropy, e.g. Unsloth/Liger) # drops the logits + eager-attention terms = the LOW end. # Fit by scripts/fit_finetune_vram.py on 57 UNIQUE (model x method x seq x batch) # RTX 5090 configs -- de-duplicated (repeated measurements collapsed to their # median), so a config is weighted by information, not by how often it happened # to be re-run. Re-run that script to refresh these if the sweep grows. _ACT_COEF = {"logits": 4.054, "act": 1.474, "attn": 3.213, "perparam": 0.280, "fixed": 0.342} # Typical dense-LLM architecture by parameter count, for when an entry has no # captured arch (interpolated; vocab defaults to a modern ~150k, conservative). _ARCH_ANCHORS = [ # (params_b, hidden, layers, heads) (0.5, 896, 24, 14), (1.5, 1536, 28, 12), (3.0, 2048, 36, 16), (7.0, 3584, 28, 28), (13.0, 5120, 40, 40), (32.0, 6656, 64, 52), (70.0, 8192, 80, 64), (120.0, 12288, 96, 96), ] _DEFAULT_VOCAB = 150000 def _approx_arch(params_b: float) -> dict: """Interpolate a representative architecture for a bare parameter count.""" a = _ARCH_ANCHORS p = max(params_b, a[0][0]) for (p0, h0, l0, hd0), (p1, h1, l1, hd1) in zip(a, a[1:]): if p <= p1: t = (p - p0) / (p1 - p0) if p1 > p0 else 0.0 return {"hidden": h0 + t * (h1 - h0), "n_layers": l0 + t * (l1 - l0), "n_heads": hd0 + t * (hd1 - hd0), "vocab": _DEFAULT_VOCAB} _, h, l, hd = a[-1] return {"hidden": h, "n_layers": l, "n_heads": hd, "vocab": _DEFAULT_VOCAB} def _activation_gb(arch: dict, seq_len: int, batch_size: int, params_b: float, grad_checkpointing: bool = True, efficient: bool = False) -> float: """Calibrated activation + overhead memory (GB) for one fine-tune step.""" hidden = float(arch.get("hidden") or 0) layers = float(arch.get("n_layers") or 0) heads = float(arch.get("n_heads") or 0) vocab = float(arch.get("vocab") or _DEFAULT_VOCAB) bs = batch_size * seq_len c = _ACT_COEF logits = 0.0 if efficient else c["logits"] * bs * vocab * 4 / 1e9 act = c["act"] * bs * hidden * layers * 2 / 1e9 if not grad_checkpointing: act *= 5.0 # without checkpointing, all layers stay live attn = 0.0 if efficient else c["attn"] * batch_size * heads * seq_len * seq_len * 2 / 1e9 return logits + act + attn + c["perparam"] * params_b + c["fixed"] # Slight conservatism on the headline number: cross-validation showed the # vanilla fit lands a touch UNDER some external anchors (the OOM-risk direction), # so we nudge up. The band's low end keeps the efficient-stack figure honest. _FT_SAFETY = 1.08 def estimate_finetune_vram(params_b: float, method: str = "qlora", seq_len: int = 2048, batch_size: int = 1, grad_checkpointing: bool = True, arch: dict | None = None, efficient: bool = False) -> float: """Conservative PEAK fine-tuning VRAM in GB (the vanilla-stack HIGH end by default). Architecture-aware when `arch` (hidden, n_layers, n_heads, vocab) is supplied; otherwise a representative arch is interpolated from params_b. See `estimate_finetune_vram_band` for the efficient-stack low end.""" p = max(params_b, 0.0) if not (arch and arch.get("hidden") and arch.get("vocab")): arch = _approx_arch(p) total = _CORE_BYTES[method] * p + _activation_gb( arch, seq_len, batch_size, p, grad_checkpointing, efficient) return round(total * _FT_SAFETY, 1) def estimate_finetune_vram_band(params_b: float, method: str = "qlora", seq_len: int = 2048, batch_size: int = 1, grad_checkpointing: bool = True, arch: dict | None = None) -> dict: """(low, high) VRAM band in GB: low = memory-optimised stack (flash-attention + fused cross-entropy, e.g. Unsloth/Liger); high = vanilla PEFT/bnb (eager attention + standard cross-entropy). The honest range a real setup lands in.""" high = estimate_finetune_vram(params_b, method, seq_len, batch_size, grad_checkpointing, arch, efficient=False) low = estimate_finetune_vram(params_b, method, seq_len, batch_size, grad_checkpointing, arch, efficient=True) return {"low": low, "high": high} # -------------------------------------------------------------------------- # Non-LLM categories: family-level fine-tune feasibility (researched). # These families don't share the LLM QLoRA formula; each has its own tooling # and memory floor. Numbers are conservative consumer-hardware minimums. # -------------------------------------------------------------------------- _CATEGORY_FT = { "vision": { "method": "Transfer learning (no LoRA — freeze the backbone)", "min_vram": 8.0, "tools": [ {"name": "Ultralytics YOLO", "what": "Fine-tune a pretrained checkpoint on your own images. Use freeze=N to cut memory.", "install": "pip install ultralytics", "tag": "Start here"}, {"name": "PyTorch / timm", "what": "Full control for custom vision training.", "install": "pytorch.org", "tag": "Advanced"}, ], "commands": [{"label": "Fine-tune YOLO on your dataset", "code": "yolo detect train model=yolo11n.pt data=mydata.yaml epochs=100 imgsz=640"}], "note": ("Vision models fine-tune by transfer learning from a pretrained checkpoint — " "there is no LoRA here. The memory lever is freeze=N (freeze the backbone) " "and a smaller batch. The n/s sizes train on ~8 GB; m/l want 12-16 GB."), "pointer": "https://docs.ultralytics.com/modes/train", }, "imagegen": { "method": "LoRA / DreamBooth (the dominant method for diffusion)", "min_vram": 8.0, "tools": [ {"name": "kohya_ss", "what": "The community standard GUI/scripts for SD, SDXL and Flux LoRA training.", "install": "github.com/bmaltais/kohya_ss", "tag": "Start here"}, {"name": "diffusers", "what": "Hugging Face's training scripts (train_dreambooth_lora_*).", "install": "pip install diffusers", "tag": "Scriptable"}, {"name": "OneTrainer", "what": "All-in-one desktop trainer for diffusion models.", "install": "github.com/Nerogar/OneTrainer", "tag": "GUI"}, ], "commands": [], "note": ("Diffusion fine-tuning is LoRA-first. SD 1.5 LoRA trains from ~6-8 GB, " "SDXL LoRA wants ~10-12 GB (16-24 comfortable), and Flux needs " "QLoRA/NF4 to fit ~9-16 GB (full FP16 wants 24 GB). A full fine-tune of " "SDXL/Flux is datacentre-only (40 GB+)."), "pointer": "https://huggingface.co/docs/diffusers/en/training/lora", }, "audio": { "method": "LoRA / PEFT (STT) or full fine-tune (small TTS)", "min_vram": 8.0, "tools": [ {"name": "HF Transformers", "what": "Fine-tune Whisper with Seq2SeqTrainer + PEFT (int8 + LoRA).", "install": "pip install transformers peft bitsandbytes", "tag": "Start here"}, {"name": "Coqui XTTS", "what": "Voice-cloning / TTS fine-tuning with a Gradio pipeline.", "install": "github.com/idiap/coqui-ai-TTS", "tag": "TTS"}, ], "commands": [], "note": ("Whisper fine-tunes with int8 + LoRA in under 8 GB (even large-v2 on a free " "Colab T4); tiny/base/small train comfortably on consumer GPUs. TTS " "(SpeechT5, XTTS v2) wants ~12-16 GB."), "pointer": "https://huggingface.co/blog/fine-tune-whisper", }, "embed": { "method": "Full fine-tune (these models are small)", "min_vram": 6.0, "tools": [ {"name": "sentence-transformers", "what": "Fine-tune embeddings with a few lines (MultipleNegativesRankingLoss).", "install": "pip install sentence-transformers", "tag": "Start here"}, ], "commands": [], "note": ("Embedding models are small — most fine-tune fully on ~6 GB. No quantisation " "or LoRA needed for typical sizes."), "pointer": "https://www.sbert.net/docs/training/overview.html", }, "data": { "method": "Full fine-tune / retrain (small models)", "min_vram": 4.0, "tools": [ {"name": "Python + the model's library", "what": "Forecasting/tabular models retrain from a small script.", "install": "pip install (see the model card)", "tag": "Start here"}, ], "commands": [], "note": "Time-series and tabular models are small and retrain on a CPU or a modest GPU.", "pointer": "", }, } # -------------------------------------------------------------------------- # Cloud fallback: when local hardware can't train the model they want. # (Prices/limits drift — labelled as guidance, confirmed live at the links.) # -------------------------------------------------------------------------- _CLOUD = [ {"name": "Google Colab (free)", "what": "Free T4 (16 GB). Comfortable for 7-9B QLoRA. Open an Unsloth notebook and Run all.", "cost": "Free", "link": "https://unsloth.ai/docs/get-started/unsloth-notebooks"}, {"name": "Kaggle (free)", "what": "Free 2x T4 (32 GB total), ~30 GPU-hours/week — more VRAM than Colab free.", "cost": "Free", "link": "https://www.kaggle.com/code"}, {"name": "Modal", "what": "Serverless GPUs from a Python script (gpu=\"A100-80GB\"). Per-second billing.", "cost": "~$30/mo free credit", "link": "https://modal.com/docs/examples"}, {"name": "RunPod", "what": "Cheapest raw GPU-hours: rent an A100/H100 for a few dollars for a one-off run.", "cost": "From ~$0.34/hr", "link": "https://www.runpod.io/pricing"}, ] def _qlora_command(repo_id: str) -> list[dict]: """The minimal real Unsloth QLoRA recipe + the GGUF export step, so the fine-tuned model can then be run in Ollama / LM Studio.""" model = repo_id or "unsloth/Qwen2.5-7B-Instruct" code = ( "from unsloth import FastLanguageModel\n" "from trl import SFTTrainer, SFTConfig\n" "from datasets import load_dataset\n\n" f'model, tok = FastLanguageModel.from_pretrained(\n' f' "{model}", max_seq_length=2048, load_in_4bit=True) # 4-bit = QLoRA\n' "model = FastLanguageModel.get_peft_model(\n" ' model, r=16, lora_alpha=16, use_gradient_checkpointing="unsloth",\n' ' target_modules=["q_proj","k_proj","v_proj","o_proj",\n' ' "gate_proj","up_proj","down_proj"])\n' 'ds = load_dataset("your/dataset", split="train") # chat/messages JSONL\n' "SFTTrainer(model=model, tokenizer=tok, train_dataset=ds,\n" " args=SFTConfig(per_device_train_batch_size=2, gradient_accumulation_steps=8,\n" ' num_train_epochs=1, learning_rate=2e-4, optim="adamw_8bit",\n' ' bf16=True, output_dir="out")).train()' ) export = ('model.save_pretrained_gguf("model", tok, quantization_method="q4_k_m")\n' "# then: ollama create my-model -f Modelfile (FROM ./model-Q4_K_M.gguf)") return [ {"label": "QLoRA fine-tune with Unsloth (lowest VRAM)", "code": code}, {"label": "Export to GGUF to run it in Ollama / LM Studio", "code": export}, ] def _llm_method_for(params_b: float, spec: HardwareSpec, arch: dict | None = None) -> dict: """Pick the lightest fine-tune method that fits, with a verdict.""" fast = spec.fast_budget_gb total = spec.total_budget_gb qlora = estimate_finetune_vram(params_b, "qlora", arch=arch) lora = estimate_finetune_vram(params_b, "lora", arch=arch) full = estimate_finetune_vram(params_b, "full", arch=arch) if fast and qlora <= fast * _SAFETY_FILL: # On the GPU. If LoRA also fits, mention it as the higher-quality option. method = "lora" if lora <= fast * _SAFETY_FILL else "qlora" need = lora if method == "lora" else qlora # report the SELECTED method's memory return {"verdict": "great", "method": method, "need": need, "qlora": qlora, "lora": lora, "full": full} # "Tight" = the QLoRA job spills GPU->system RAM via a paged optimiser. That # path NEEDS a CUDA GPU; a CPU-only machine cannot run the Unsloth/CUDA # commands we emit, so it must not be told it can train (route to cloud). if fast and qlora <= total * _SAFETY_FILL: return {"verdict": "tight", "method": "qlora", "need": qlora, "qlora": qlora, "lora": lora, "full": full} return {"verdict": "no", "method": "qlora", "need": qlora, "qlora": qlora, "lora": lora, "full": full} def _ft_option(entry: dict, m: dict) -> dict: feel = {"great": "Fits your GPU", "tight": "Works via system RAM — slow", "no": "Too big locally — use the cloud"}[m["verdict"]] return { "verdict": m["verdict"], "model": entry["name"], "desc": entry.get("good_for", ""), "setting": _METHOD_PLAIN[m["method"]], "memory": "Too big" if m["verdict"] == "no" else f"{m['need']:g} GB", "feel": feel, "params_b": entry.get("params_b"), "active_params_b": entry.get("active_params_b"), "url": (entry.get("links") or {}).get("hf") or (entry.get("links") or {}).get("home", ""), "license": entry.get("license", ""), "license_note": entry.get("license_note", ""), "gated": entry.get("gated", False), "run": {}, "provenance": "estimated", "stale": entry.get("stale", False), } def _llm_finetune(uc, candidates, spec, focus) -> dict: fast, total = spec.fast_budget_gb, spec.total_budget_gb evald = [(e, _llm_method_for(e.get("params_b", 1.0), spec, e.get("arch"))) for e in candidates] # Headline = the LARGEST model you can QLoRA locally (great); else largest # that works tight; else the smallest (so we can show the cloud path). great = [(e, m) for e, m in evald if m["verdict"] == "great"] tight = [(e, m) for e, m in evald if m["verdict"] == "tight"] def by_params(pair): return pair[0].get("params_b", 0) if focus: hit = next((pair for pair in evald if pair[0]["name"] == focus or str(pair[0].get("repo_id", "")).lower() == focus.lower()), None) chosen = hit or (max(great, key=by_params) if great else None) else: chosen = (max(great, key=by_params) if great else max(tight, key=by_params) if tight else min(evald, key=lambda pr: pr[1]["need"]) if evald else None) options = [_ft_option(e, m) for e, m in evald] repo = "" if chosen: e, m = chosen repo = e.get("repo_id", "") hv, need = m["verdict"], m["need"] if hv == "great": head = f"Yes — you can fine-tune {e['name']} on your machine." detail = ( f"The honest pick for training is {e['name']} with " f"{_METHOD_PLAIN[m['method']]}. It needs about {need:g} GB of GPU " f"memory (you have ~{fast:g} GB on the fast path). For reference the same " f"model is ~{m['lora']:g} GB with 16-bit LoRA and ~{m['full']:g} GB for a full " f"fine-tune — which is why QLoRA is the consumer answer." ) elif hv == "tight": head = f"Sort of — {e['name']} will fine-tune, but it spills into system RAM." detail = ( f"{e['name']} needs about {need:g} GB for QLoRA, more than your " f"~{fast:g} GB of GPU memory. A paged optimiser can borrow ordinary RAM " f"(you have ~{total:g} GB total) so it runs, but slowly. A bigger GPU — or a free " f"cloud notebook — would make this comfortable." ) else: head = f"Training is a stretch on this machine — here's the honest path." biggest_local = max((p for p in evald if p[1]["verdict"] != "no"), key=by_params, default=None) local_line = (f"Locally you can comfortably QLoRA up to {biggest_local[0]['name']}. " if biggest_local else "This machine has no GPU fast path for training. ") detail = ( f"{e['name']} needs about {need:g} GB to QLoRA, beyond what this machine " f"offers. {local_line}For anything bigger, a rented or free cloud GPU is the cheapest " f"path — see the options below." ) scale = max(fast or total, need, 1) * 1.05 has_fast = spec.has_fast_path if spec.is_apple_silicon: fast_label, total_label = "GPU can use", "Unified memory" elif has_fast: fast_label, total_label = "On the GPU (VRAM)", "GPU + system RAM" else: fast_label, total_label = "", "System RAM (no GPU)" gauge = { "need_gb": f"{need:g} GB needed to train", "fast_gb": f"{fast:g} GB", "total_gb": f"{total:g} GB", "fast_label": fast_label, "total_label": total_label, "has_fast": has_fast, "fill_pct": round(min(need / scale, 1.0) * 100, 1), "mark_pct": round(min((fast or total) / scale, 1.0) * 100, 1), "total_pct": round(min(total / scale, 1.0) * 100, 1), "breakdown": [ {"label": f"4-bit base {round(e.get('params_b',1)*0.5625,1):g} GB", "color": _C_MODEL}, {"label": f"Adapters, optimiser & activations {round(need - e.get('params_b',1)*0.5625,1):g} GB", "color": _C_WORK}, ], } commands = {"intro": "A real QLoRA recipe for the pick above, then the step to run your " "fine-tuned model locally.", "items": _qlora_command(repo)} provenance = ("Training memory is a conservative estimate (16 bytes/parameter for full " "fine-tuning; ~4.5-bit base for QLoRA) sized to land at or above Unsloth's " "published minimums — most setups use the vanilla PEFT/bitsandbytes stack.") else: hv = "no" head = "Nothing in the catalogue fits training on this machine yet." detail = "Try a smaller model, or use one of the free cloud notebooks below." gauge, commands, provenance = {}, {"intro": "", "items": []}, "" tools = [ {"name": "Unsloth", "what": "Fastest single-GPU QLoRA, lowest VRAM, built-in GGUF export. Beginner default.", "install": "pip install unsloth", "tag": "Start here"}, {"name": "Hugging Face TRL + PEFT", "what": "The reference stack: SFTTrainer + LoRA/QLoRA, maximum compatibility.", "install": "pip install trl peft bitsandbytes", "tag": "Reference"}, {"name": "Axolotl", "what": "One YAML config; best when you move to multi-GPU or long context.", "install": "github.com/axolotl-ai-cloud/axolotl", "tag": "Scale up"}, ] return { "verdict": hv, "verdict_word": _FT_VERDICT_WORD.get(hv, _VERDICT_WORD[hv]), "headline": head, "detail": detail, "gauge": gauge, "options": options, "tools": tools, "commands": commands, "provenance": provenance, "cloud": _CLOUD, "speed": None, "headline_model": chosen[0]["name"] if chosen else "", "focus": focus or "", } def _category_finetune(uc, candidates, spec) -> dict: fam = uc.family info = _CATEGORY_FT.get(fam) fast = spec.fast_budget_gb if not info: return {"verdict": "tight", "verdict_word": "Depends on the model", "headline": f"Fine-tuning {uc.plain_name.lower()} depends on the specific model.", "detail": "Paste a specific model id in the box above to check it.", "gauge": {}, "options": [], "tools": [], "commands": {"intro": "", "items": []}, "provenance": "", "cloud": _CLOUD, "speed": None, "headline_model": "", "focus": ""} min_vram = info["min_vram"] fits = fast >= min_vram hv = "great" if fits else "no" head = (f"Yes — you can fine-tune {uc.plain_name.lower()} on this machine." if fits else f"Fine-tuning {uc.plain_name.lower()} needs about {min_vram:g} GB of GPU memory — more than this machine has.") detail = info["note"] + (f' How to start.' if info.get("pointer") else "") options = [] for e in candidates[:12]: options.append({ "verdict": "great" if fits else "no", "model": e["name"], "desc": e.get("good_for", ""), "setting": info["method"], "memory": f"~{min_vram:g} GB to train" if fits else "Use the cloud", "feel": "Fits your GPU" if fits else "Too big locally", "params_b": e.get("params_b"), "active_params_b": e.get("active_params_b"), "url": (e.get("links") or {}).get("hf") or (e.get("links") or {}).get("home", ""), "license": e.get("license", ""), "license_note": e.get("license_note", ""), "gated": e.get("gated", False), "run": {}, "provenance": "estimated", "stale": e.get("stale", False), }) return { "verdict": hv, "verdict_word": _FT_VERDICT_WORD.get(hv, _VERDICT_WORD[hv]), "headline": head, "detail": detail, "gauge": {}, "options": options, "tools": info["tools"], "commands": {"intro": "", "items": info.get("commands", [])}, "provenance": ("These are conservative family-level minimums; the exact figure varies " "with model size, resolution and batch."), "cloud": _CLOUD, "speed": None, "headline_model": "", "focus": "", } _FT_VERDICT_WORD = {"great": "You can train this", "tight": "Trainable, but tight", "no": "Train in the cloud"} def advise_finetune(payload: dict, spec: HardwareSpec, extra_entries: list | None = None) -> dict: """Mirror of advise_real for fine-tuning. Same result shape so the same renderer draws it; adds a `cloud` section and omits the speed chart. extra_entries injects a live-looked-up model as a synthetic candidate.""" uc = USE_CASES.get(payload.get("usecase", "chat"), USE_CASES["chat"]) candidates = list(_by_use_case().get(uc.key, [])) if extra_entries: candidates = list(extra_entries) + candidates focus = (payload.get("focus") or "").strip() if not candidates: base = {"verdict": "tight", "verdict_word": "Not covered yet", "headline": "Our catalogue doesn't cover this goal yet.", "detail": "Paste a specific Hugging Face model id above to check it for training.", "gauge": {}, "options": [], "tools": [], "commands": {"intro": "", "items": []}, "provenance": "", "cloud": _CLOUD, "speed": None, "headline_model": "", "focus": ""} elif uc.family in ("llm", "vlm"): base = _llm_finetune(uc, candidates, spec, focus) else: base = _category_finetune(uc, candidates, spec) base.update({ "mode": "finetune", "catalogue_version": catalogue_date(), "use_case": uc.plain_name, "usecase": uc.key, "meets_goal": base["verdict"] in ("great", "tight"), "note": "", }) return base