File size: 8,042 Bytes
dc23a98 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 | """Write the Hugging Face model card for each public checkpoint, from its training log and measured scores.
Runs the upload on Modal, where the "huggingface" secret holds the write token (no local HF login needed).
A card-only commit changes the repo's head, not the adapter: the pinned training commit in each card and in the
README still loads the same weights.
Usage: PYTHONIOENCODING=utf-8 modal run scripts/push_model_cards.py [--only v3-n12000,v4-n12000]
"""
import json
from pathlib import Path
import modal
ROOT = Path(__file__).resolve().parent.parent
USER = "debudash"
CODE = "https://huggingface.co/debudash/slm-training"
DATA = "https://huggingface.co/datasets/debudash/tz-convert-sft"
SYSTEM = "You convert date-times between timezones under the tz-convert spec. Reply with exactly one line."
BASE = {"spec": 12.0, "rob": 23.3, "drift": 0.0}
# Measured on cases/ceiling.jsonl (spec-adherence, robustness) and cases/drift.jsonl (conversations held);
# docs/results.md. None = not measured.
SCORES = {
"v1-n250": (12.0, 12.6, None), "v1-n500": (14.0, 24.3, None), "v1-n1000": (18.0, 31.1, None),
"v1-n2000": (44.0, 47.6, None), "v2-n750": (20.0, 21.4, None), "v2-n1500": (38.0, 42.7, None),
"v2-n3000": (40.0, 42.7, None), "v2-n6000": (58.0, 66.0, 12.9), "v3-n1500": (36.0, 46.6, 5.7),
"v3-n3000": (40.0, 57.3, 20.0), "v3-n6000": (50.0, 71.8, 25.7), "v3-n12000": (72.0, 84.5, 75.7),
"v4-n12000": (72.0, 79.6, 80.0), "v5-n18000": (72.0, 87.4, 91.4),
}
HEADLINE = "v5-n18000"
app = modal.App("slm-training-cards", image=modal.Image.debian_slim(python_version="3.12").pip_install("huggingface_hub"))
@app.function(secrets=[modal.Secret.from_name("huggingface")], timeout=600)
def upload(cards: dict):
from huggingface_hub import HfApi
api, out = HfApi(), {}
for repo, text in cards.items():
info = api.upload_file(path_or_fileobj=text.encode("utf-8"), path_in_repo="README.md", repo_id=repo,
commit_message="Model card")
out[repo] = info.oid
return out
def pct(v):
return "–" if v is None else f"{v:.1f}%"
def card(tag, log):
repo, commit = log["repo"], log["commit"]
spec, rob, drift = SCORES[tag]
mix = tag.split("-")[0]
h, lora = log["hparams"], log["lora"]
front = ("---\nbase_model: Qwen/Qwen3-4B-Instruct-2507\nlibrary_name: peft\npipeline_tag: text-generation\n"
f"datasets:\n- {USER}/tz-convert-sft\nlanguage:\n- en\nlicense: apache-2.0\n"
"tags:\n- lora\n- qlora\n- timezone\n- tz-convert\n---\n\n")
if tag != HEADLINE:
return front + (
f"# qwen3-4b-tz-convert-{tag}\n\nOne point on the data-efficiency curve of the tz-convert project: a QLoRA "
f"adapter on Qwen3-4B-Instruct-2507 trained on the first {log['n_examples']:,} examples of the "
f"`{mix}` mix of [{USER}/tz-convert-sft]({DATA}). Same hyperparameters as every other checkpoint.\n\n"
f"| Spec-adherence | Robustness | Drift: conversations held | Training commit |\n|---|---|---|---|\n"
f"| {pct(spec)} | {pct(rob)} | {pct(drift)} | `{commit}` |\n\n"
f"Base model with the full spec prompt: {BASE['spec']}% / {BASE['rob']}% / {BASE['drift']}%. "
f"The best checkpoint and full card: [{USER}/qwen3-4b-tz-convert-{HEADLINE}]"
f"(https://huggingface.co/{USER}/qwen3-4b-tz-convert-{HEADLINE}). Code, eval sets and reports: [{CODE}]({CODE}).\n")
return front + f"""# qwen3-4b-tz-convert-{tag}
A QLoRA adapter that makes Qwen3-4B-Instruct-2507 convert a date-time between timezones **in exactly one line,
and never guess**. It replies with `YYYY-MM-DDTHH:MM:SS±HH:MM` (IANA tzdata 2026e), or `AMBIGUOUS: <component>` /
`INVALID: <component>` when the request can't be converted without guessing, and it holds that across a
conversation, including when the user insists.
- **Behaviour spec:** [behaviour.spec]({CODE}/blob/main/behaviour.spec), over a closed world of 24 IANA zones,
listed abbreviations and source dates 2020-2030.
- **Training commit (pinned):** `{commit}`
- **Data:** [{USER}/tz-convert-sft]({DATA}), config `{mix}`, {log['n_examples']:,} examples. A frontier teacher
wrote the user-message wording only; every reply was computed by an exact oracle.
## Results
Same LLM judge (Claude Haiku 5.5, against the oracle's reply) for every model. Single run, greedy decoding.
| Model | Spec-adherence | Robustness | Drift: conversations held |
|---|---|---|---|
| Qwen3-4B-Instruct base, full spec + tables in the prompt | {BASE['spec']}% | {BASE['rob']}% | {BASE['drift']}% |
| **This adapter** (one-line system prompt) | **{pct(spec)}** | **{pct(rob)}** | **{pct(drift)}** |
| Claude Sonnet 5.5, best prompt, reasoning on | 88.7% | 90.0% | 76.4% |
| Mistral Large 4, best prompt, reasoning on | 97.3% | 92.9% | 57.1% |
Spec-adherence: 50 clean single-turn scenarios. Robustness: 47 adversarial and multi-turn scenarios (153 turns in
total; differences under about 8 points are noise). Drift: 70 conversations, 485 turns (about 10 points of noise).
## Use
The adapter was trained with exactly this system prompt; use it verbatim:
```text
{SYSTEM}
```
```python
from vllm import LLM, SamplingParams
from vllm.lora.request import LoRARequest
from huggingface_hub import snapshot_download
llm = LLM("Qwen/Qwen3-4B-Instruct-2507", enable_lora=True, max_lora_rank=64)
lora = LoRARequest("tz", 1, snapshot_download("{repo}", revision="{commit}"))
msgs = [{{"role": "system", "content": "{SYSTEM}"}},
{{"role": "user", "content": "Convert 2026-11-01 09:30 America/Chicago to Asia/Tokyo"}}]
print(llm.chat(msgs, SamplingParams(temperature=0, max_tokens=64), lora_request=lora)[0].outputs[0].text)
```
One-command eval: `python eval.py --model {repo} --revision {commit} --eval-set cases/ceiling.jsonl` ([code]({CODE})).
## Training
QLoRA: base in 4-bit NF4 (bf16 compute), LoRA r={lora['r']}, alpha={lora['lora_alpha']}, dropout={lora['lora_dropout']}
on {", ".join(lora['target_modules'])}. Loss on assistant tokens only. {h['epochs']} epochs, lr {h['lr']} with
{int(h['warmup_ratio'] * 100)}% warmup and cosine decay, batch {h['batch']} x grad-accum {h['grad_accum']}, max length
{h['max_len']}, seed {h['seed']}. {log['train_minutes']} min on one {log.get('gpu') or 'GPU'} (Modal). Every checkpoint
in the project used the same hyperparameters; each improvement came from a data change.
## Limitations
- **Conversion arithmetic** is the main weakness: most remaining failures are a wrong hour (often exactly 1 h), which
then carries through later turns of a conversation.
- **Closed world:** zones outside the 24 listed, unlisted abbreviations and dates outside 2020-2030 are `INVALID` by
design.
- **In-distribution caveat:** the seven drift patterns appear in the training data (different wording and values),
so the drift score tests the patterns in distribution.
- **DST arithmetic (ISSUE-003, open):** on 80 conversions between two DST zones it gets 68 right (v3: 58; Sonnet and
Mistral with reasoning: 74.5 and 79.5). Most misses are exactly 1 h off. Clean single-turn accuracy is 72%.
- **Fixed in this version's data:** a zone change that doesn't say which zone it replaces ("Make that Asia/Tokyo.")
after an ambiguous-source flag now stays `AMBIGUOUS` (8/8 on the ISSUE-002 probe; v3 0/8).
"""
@app.local_entrypoint()
def main(only: str = ""):
cards = {}
for path in sorted((ROOT / "results/training").glob("qwen3-4b-tz-convert-*.json")):
log = json.loads(path.read_text(encoding="utf-8"))
tag = log["run"].removeprefix("qwen3-4b-tz-convert-")
if tag not in SCORES or (only and tag not in only.split(",")):
continue
cards[log["repo"]] = card(tag, log)
for repo, oid in upload.remote(cards).items():
print(f"{repo}: card pushed @ {oid[:8]}")
|