"""Write the Hugging Face model card for each public checkpoint, from its training log and measured scores. Runs the upload on Modal, where the "huggingface" secret holds the write token (no local HF login needed). A card-only commit changes the repo's head, not the adapter: the pinned training commit in each card and in the README still loads the same weights. Usage: PYTHONIOENCODING=utf-8 modal run scripts/push_model_cards.py [--only v3-n12000,v4-n12000] """ import json from pathlib import Path import modal ROOT = Path(__file__).resolve().parent.parent USER = "debudash" CODE = "https://huggingface.co/debudash/slm-training" DATA = "https://huggingface.co/datasets/debudash/tz-convert-sft" SYSTEM = "You convert date-times between timezones under the tz-convert spec. Reply with exactly one line." BASE = {"spec": 12.0, "rob": 23.3, "drift": 0.0} # Measured on cases/ceiling.jsonl (spec-adherence, robustness) and cases/drift.jsonl (conversations held); # docs/results.md. None = not measured. SCORES = { "v1-n250": (12.0, 12.6, None), "v1-n500": (14.0, 24.3, None), "v1-n1000": (18.0, 31.1, None), "v1-n2000": (44.0, 47.6, None), "v2-n750": (20.0, 21.4, None), "v2-n1500": (38.0, 42.7, None), "v2-n3000": (40.0, 42.7, None), "v2-n6000": (58.0, 66.0, 12.9), "v3-n1500": (36.0, 46.6, 5.7), "v3-n3000": (40.0, 57.3, 20.0), "v3-n6000": (50.0, 71.8, 25.7), "v3-n12000": (72.0, 84.5, 75.7), "v4-n12000": (72.0, 79.6, 80.0), "v5-n18000": (72.0, 87.4, 91.4), } HEADLINE = "v5-n18000" app = modal.App("slm-training-cards", image=modal.Image.debian_slim(python_version="3.12").pip_install("huggingface_hub")) @app.function(secrets=[modal.Secret.from_name("huggingface")], timeout=600) def upload(cards: dict): from huggingface_hub import HfApi api, out = HfApi(), {} for repo, text in cards.items(): info = api.upload_file(path_or_fileobj=text.encode("utf-8"), path_in_repo="README.md", repo_id=repo, commit_message="Model card") out[repo] = info.oid return out def pct(v): return "–" if v is None else f"{v:.1f}%" def card(tag, log): repo, commit = log["repo"], log["commit"] spec, rob, drift = SCORES[tag] mix = tag.split("-")[0] h, lora = log["hparams"], log["lora"] front = ("---\nbase_model: Qwen/Qwen3-4B-Instruct-2507\nlibrary_name: peft\npipeline_tag: text-generation\n" f"datasets:\n- {USER}/tz-convert-sft\nlanguage:\n- en\nlicense: apache-2.0\n" "tags:\n- lora\n- qlora\n- timezone\n- tz-convert\n---\n\n") if tag != HEADLINE: return front + ( f"# qwen3-4b-tz-convert-{tag}\n\nOne point on the data-efficiency curve of the tz-convert project: a QLoRA " f"adapter on Qwen3-4B-Instruct-2507 trained on the first {log['n_examples']:,} examples of the " f"`{mix}` mix of [{USER}/tz-convert-sft]({DATA}). Same hyperparameters as every other checkpoint.\n\n" f"| Spec-adherence | Robustness | Drift: conversations held | Training commit |\n|---|---|---|---|\n" f"| {pct(spec)} | {pct(rob)} | {pct(drift)} | `{commit}` |\n\n" f"Base model with the full spec prompt: {BASE['spec']}% / {BASE['rob']}% / {BASE['drift']}%. " f"The best checkpoint and full card: [{USER}/qwen3-4b-tz-convert-{HEADLINE}]" f"(https://huggingface.co/{USER}/qwen3-4b-tz-convert-{HEADLINE}). Code, eval sets and reports: [{CODE}]({CODE}).\n") return front + f"""# qwen3-4b-tz-convert-{tag} A QLoRA adapter that makes Qwen3-4B-Instruct-2507 convert a date-time between timezones **in exactly one line, and never guess**. It replies with `YYYY-MM-DDTHH:MM:SS±HH:MM` (IANA tzdata 2026e), or `AMBIGUOUS: ` / `INVALID: ` when the request can't be converted without guessing, and it holds that across a conversation, including when the user insists. - **Behaviour spec:** [behaviour.spec]({CODE}/blob/main/behaviour.spec), over a closed world of 24 IANA zones, listed abbreviations and source dates 2020-2030. - **Training commit (pinned):** `{commit}` - **Data:** [{USER}/tz-convert-sft]({DATA}), config `{mix}`, {log['n_examples']:,} examples. A frontier teacher wrote the user-message wording only; every reply was computed by an exact oracle. ## Results Same LLM judge (Claude Haiku 5.5, against the oracle's reply) for every model. Single run, greedy decoding. | Model | Spec-adherence | Robustness | Drift: conversations held | |---|---|---|---| | Qwen3-4B-Instruct base, full spec + tables in the prompt | {BASE['spec']}% | {BASE['rob']}% | {BASE['drift']}% | | **This adapter** (one-line system prompt) | **{pct(spec)}** | **{pct(rob)}** | **{pct(drift)}** | | Claude Sonnet 5.5, best prompt, reasoning on | 88.7% | 90.0% | 76.4% | | Mistral Large 4, best prompt, reasoning on | 97.3% | 92.9% | 57.1% | Spec-adherence: 50 clean single-turn scenarios. Robustness: 47 adversarial and multi-turn scenarios (153 turns in total; differences under about 8 points are noise). Drift: 70 conversations, 485 turns (about 10 points of noise). ## Use The adapter was trained with exactly this system prompt; use it verbatim: ```text {SYSTEM} ``` ```python from vllm import LLM, SamplingParams from vllm.lora.request import LoRARequest from huggingface_hub import snapshot_download llm = LLM("Qwen/Qwen3-4B-Instruct-2507", enable_lora=True, max_lora_rank=64) lora = LoRARequest("tz", 1, snapshot_download("{repo}", revision="{commit}")) msgs = [{{"role": "system", "content": "{SYSTEM}"}}, {{"role": "user", "content": "Convert 2026-11-01 09:30 America/Chicago to Asia/Tokyo"}}] print(llm.chat(msgs, SamplingParams(temperature=0, max_tokens=64), lora_request=lora)[0].outputs[0].text) ``` One-command eval: `python eval.py --model {repo} --revision {commit} --eval-set cases/ceiling.jsonl` ([code]({CODE})). ## Training QLoRA: base in 4-bit NF4 (bf16 compute), LoRA r={lora['r']}, alpha={lora['lora_alpha']}, dropout={lora['lora_dropout']} on {", ".join(lora['target_modules'])}. Loss on assistant tokens only. {h['epochs']} epochs, lr {h['lr']} with {int(h['warmup_ratio'] * 100)}% warmup and cosine decay, batch {h['batch']} x grad-accum {h['grad_accum']}, max length {h['max_len']}, seed {h['seed']}. {log['train_minutes']} min on one {log.get('gpu') or 'GPU'} (Modal). Every checkpoint in the project used the same hyperparameters; each improvement came from a data change. ## Limitations - **Conversion arithmetic** is the main weakness: most remaining failures are a wrong hour (often exactly 1 h), which then carries through later turns of a conversation. - **Closed world:** zones outside the 24 listed, unlisted abbreviations and dates outside 2020-2030 are `INVALID` by design. - **In-distribution caveat:** the seven drift patterns appear in the training data (different wording and values), so the drift score tests the patterns in distribution. - **DST arithmetic (ISSUE-003, open):** on 80 conversions between two DST zones it gets 68 right (v3: 58; Sonnet and Mistral with reasoning: 74.5 and 79.5). Most misses are exactly 1 h off. Clean single-turn accuracy is 72%. - **Fixed in this version's data:** a zone change that doesn't say which zone it replaces ("Make that Asia/Tokyo.") after an ambiguous-source flag now stays `AMBIGUOUS` (8/8 on the ISSUE-002 probe; v3 0/8). """ @app.local_entrypoint() def main(only: str = ""): cards = {} for path in sorted((ROOT / "results/training").glob("qwen3-4b-tz-convert-*.json")): log = json.loads(path.read_text(encoding="utf-8")) tag = log["run"].removeprefix("qwen3-4b-tz-convert-") if tag not in SCORES or (only and tag not in only.split(",")): continue cards[log["repo"]] = card(tag, log) for repo, oid in upload.remote(cards).items(): print(f"{repo}: card pushed @ {oid[:8]}")