Fleet-2 / fleet2.py
Lucasgitguy-co's picture
Fleet-2: Whistle student of Whisper-large-v3-turbo
15e183b verified
Raw History Blame Contribute Delete
45.8 kB
"""Fleet-2: Whistle student trained on Whisper-large-v3-turbo pseudo-labels (sequence-level distillation).
Stages (run through run.sh): prepare, filter, stage0, train, select, test, publish.
The teacher pass lives in teacher.py (separate MLX environment).
"""
import argparse
import hashlib
import io
import json
import math
import os
import shutil
import sys
import time
from dataclasses import replace
from pathlib import Path
HERE = Path(__file__).resolve().parent
RUN = Path(os.environ.get("FLEET2_RUN", HERE / "runs/main")).resolve()
SMOKE = os.environ.get("FLEET2_SMOKE") == "1"
DATA = RUN / "data"
PUB = RUN / "publish"
LANGS = dict(en="en_us", de="de_de", fr="fr_fr", es="es_419", it="it_it", nl="nl_nl", pl="pl_pl", ko="ko_kr")
WHISTLE_LANGS = ("en", "de", "fr", "es", "it", "nl", "pl")
SEED, SECONDS, TOKENS, EPOCHS, LR = 42, 12, 160, 2, 2e-5
MAX_IDS = TOKENS + 1 # bos + tag + text + eos must fit, same rule as Fleet-1
SPEAKER_CAP = 20
THRESH = dict(compression_ratio=2.4, avg_logprob=-1.0, no_speech_prob=0.6, error_rate=0.10)
if SMOKE:
SIZES = dict(candidates=3, keep=2, anchor=2, validation=2, test=2, ood=2)
else:
SIZES = dict(candidates=600, keep=400, anchor=200, validation=50, test=100, ood=50)
SKY = dict(repo="Sky-Kim/whistle-finetuned-ko", revision="b8a4f3188a828a0e87e2713b2d3e83d71d893068",
adapter="adapter/whistle_ko_lora.safetensors")
FLEET1 = dict(repo="Lucasgitguy-co/Fleet-1", revision="0cc581616f7e5493e2e3999e71c134cedc879a6d",
adapter="fleet_lora.safetensors")
WHISTLE = dict(repo="Cactus-Compute/whistle", revision="741118687e1f008f06d1351a402a731274789790",
checkpoint="checkpoints/whistle.safetensors")
CACTUS_CODE = "ee2ffc67fe0f9537d4928e158440c7d228a63100"
DEST = "Lucasgitguy-co/Fleet-2"
OWNER = "Lucasgitguy-co"
COLLECTION = "Lucasgitguy-co/speech-recognition-6aef2dcbe0ea208136b0ddc9"
VENDOR = HERE / "vendor"
# ---------------------------------------------------------------- helpers
def read(path):
return json.loads(Path(path).read_text(encoding="utf-8"))
def write(path, data):
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True)
tmp = path.with_suffix(path.suffix + ".tmp")
tmp.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
tmp.replace(path)
def immutable(path, data):
"""Write once; later runs must agree, so a resumed run never mixes settings."""
if Path(path).exists():
if read(path) != data:
raise RuntimeError(f"{path} records different settings from this run. Use a fresh run folder.")
return
write(path, data)
def sha(path):
h = hashlib.sha256()
with open(path, "rb") as f:
for block in iter(lambda: f.read(1 << 20), b""):
h.update(block)
return h.hexdigest()
def text_key(text):
return " ".join(text.casefold().split())
def rss():
import psutil
return psutil.Process().memory_info().rss
def hub(repo, name, revision):
from huggingface_hub import hf_hub_download
return hf_hub_download(repo, name, revision=revision)
def tokenizer():
from cactus.model.checkpoints import read_adapter
from cactus.model.whistle_finetune import WhistleTokenizer
a = read_adapter(hub(SKY["repo"], SKY["adapter"], SKY["revision"]))
return WhistleTokenizer(a["pieces"], a["languages"])
def error_units(reference, hypothesis, lang):
"""Fleet-1 scoring: CER for Korean (spaces removed), normalized WER otherwise."""
from cactus.model.whistle_finetune import errors, _normalizer
if lang == "ko":
import editdistance
norm = _normalizer(lang)
ref, hyp = norm(reference).replace(" ", ""), norm(hypothesis).replace(" ", "")
return editdistance.eval(ref, hyp), len(ref)
return errors(reference, hypothesis, lang)
def decode_audio(audio):
import numpy as np
import soundfile as sf
import soxr
wav, rate = sf.read(io.BytesIO(audio["bytes"]) if audio.get("bytes") else audio["path"], dtype="float32",
always_2d=True)
wav = wav.mean(axis=1)
if rate != 16000:
wav = soxr.resample(wav, rate, 16000)
return np.ascontiguousarray(wav, dtype=np.float32)
def save_wav(path, wav):
import soundfile as sf
path.parent.mkdir(parents=True, exist_ok=True)
sf.write(path, wav, 16000, subtype="PCM_16")
def stream(repo, revision, files):
from datasets import load_dataset, Audio
urls = [f"https://huggingface.co/datasets/{repo}/resolve/{revision}/{f}" for f in files]
return load_dataset("parquet", data_files={"x": urls}, split="x", streaming=True,
batch_size=8).cast_column("audio", Audio(decode=False))
# ---------------------------------------------------------------- prepare
def revisions():
from huggingface_hub import HfApi
path = DATA / "revisions.json"
if not path.exists():
api = HfApi()
write(path, {"google/fleurs": read(VENDOR / "training_manifest.json")["dataset_revision"],
"facebook/voxpopuli": api.dataset_info("facebook/voxpopuli").sha,
"Bingsu/zeroth-korean": api.dataset_info("Bingsu/zeroth-korean").sha})
return read(path)
def prepare_fleurs(revs, tok):
"""Reproduce Fleet-1's FLEURS rows exactly (sha256 + text checked)."""
from huggingface_hub import HfApi
manifest = read(VENDOR / "training_manifest.json")
rev = revs["google/fleurs"]
files = None
plan = {"train": SIZES["anchor"], "validation": SIZES["validation"], "test": SIZES["test"]}
for split, count in plan.items():
for lang, config in LANGS.items():
dest = DATA / f"fleurs-{split}-{lang}.json"
if dest.exists():
continue
wanted = [r for r in manifest["splits"][split] if r["language"] == lang][:count]
if len(wanted) != count:
raise RuntimeError(f"Fleet-1 manifest has only {len(wanted)} {split}/{lang} rows")
by_index = {r["row_index"]: r for r in wanted}
files = files or HfApi().list_repo_files("google/fleurs", repo_type="dataset", revision=rev)
paths = sorted(p for p in files if p.startswith(f"{config}/{split}/") and p.endswith(".parquet"))
print(f"FLEURS {split}/{lang}: reproducing {count} Fleet-1 rows", flush=True)
found = {}
for index, row in enumerate(stream("google/fleurs", rev, paths)):
if index > max(by_index):
break
if index not in by_index:
continue
ref = by_index[index]
text = (row.get("raw_transcription") or row.get("transcription") or "").strip()
path = DATA / "audio/fleurs" / split / lang / f"{index:06d}.wav"
save_wav(path, decode_audio(row["audio"]))
if text != ref["text"] or sha(path) != ref["sha256"]:
raise RuntimeError(f"FLEURS {split}/{lang}/{index} does not reproduce Fleet-1's published example")
found[index] = {"id": f"fleurs/{split}/{lang}/{index}", "source": "fleurs", "split": split,
"audio": str(path.relative_to(DATA)), "gold": text, "language": lang,
"row_index": index, "fleurs_id": ref.get("id"), "sha256": ref["sha256"],
"text_key": text_key(text)}
if set(found) != set(by_index):
raise RuntimeError(f"FLEURS {split}/{lang}: missing rows {sorted(set(by_index) - set(found))}")
write(dest, [found[r["row_index"]] for r in wanted])
def prepare_corpus(name, repo, rev, lang, split, count, files, gold_field, seen, tok, per_shard):
"""Stream the first eligible rows of each shard. Never downloads whole shards."""
out, speakers = [], {}
quota = math.ceil(count / len(files)) if per_shard else count
shards = files if per_shard else [files]
for shard_no, shard in enumerate(shards):
shard_files = [shard] if per_shard else shard
taken = 0
for index, row in enumerate(stream(repo, rev, shard_files)):
if len(out) >= count or taken >= quota:
break
text = (row.get(gold_field) or row.get("normalized_text") or "").strip()
key = text_key(text)
if not text or (lang, key) in seen:
continue
if len(tok.encode(text)) + 3 > MAX_IDS:
continue
speaker = row.get("speaker_id")
if speaker and speakers.get(speaker, 0) >= SPEAKER_CAP:
continue
wav = decode_audio(row["audio"])
if not 1 * 16000 <= len(wav) <= SECONDS * 16000:
continue
path = DATA / "audio" / name / split / lang / f"{shard_no:02d}-{index:06d}.wav"
save_wav(path, wav)
out.append({"id": f"{name}/{split}/{lang}/{shard_no}/{index}", "source": name, "split": split,
"audio": str(path.relative_to(DATA)), "gold": text, "language": lang,
"shard": shard_files[0].rsplit("/", 1)[-1], "row_index": index,
"audio_id": row.get("audio_id"), "speaker": speaker, "sha256": sha(path),
"seconds": round(len(wav) / 16000, 3), "text_key": key})
seen.add((lang, key))
if speaker:
speakers[speaker] = speakers.get(speaker, 0) + 1
taken += 1
if len(out) >= count:
break
return out
def corpus_files(repo, rev, lang, split):
from huggingface_hub import HfApi
files = HfApi().list_repo_files(repo, repo_type="dataset", revision=rev)
if repo == "facebook/voxpopuli":
prefix = f"{lang}/{split}-"
else:
prefix = f"data/{split}-"
chosen = sorted(f for f in files if f.startswith(prefix) and f.endswith(".parquet"))
if not chosen:
raise RuntimeError(f"No {split} parquet files for {repo} {lang}")
return chosen
def prepare():
DATA.mkdir(parents=True, exist_ok=True)
immutable(RUN / "settings.json", {"smoke": SMOKE, "sizes": SIZES, "seed": SEED, "seconds": SECONDS,
"tokens": TOKENS, "epochs": EPOCHS, "lr": LR, "thresholds": THRESH,
"speaker_cap": SPEAKER_CAP, "sky": SKY, "fleet1": FLEET1, "whistle": WHISTLE,
"cactus_code": CACTUS_CODE,
"teacher": read(HERE / "teacher_pin.json")})
revs = revisions()
tok = tokenizer()
prepare_fleurs(revs, tok)
# Every evaluation transcript is excluded from training data.
seen = set()
for split in ("validation", "test"):
for lang in LANGS:
seen |= {(r["language"], r["text_key"]) for r in read(DATA / f"fleurs-{split}-{lang}.json")}
# Out-of-domain test (report only), then training candidates.
for split, count, kind in (("test", SIZES["ood"], "ood"), ("train", SIZES["candidates"], "candidates")):
for lang in LANGS:
dest = DATA / f"{kind}-{lang}.json"
if not dest.exists():
if lang == "ko":
repo, name, field = "Bingsu/zeroth-korean", "zeroth", "text"
else:
repo, name, field = "facebook/voxpopuli", "voxpopuli", "raw_text"
rev = revs[repo]
files = corpus_files(repo, rev, lang, split)
print(f"{name} {split}/{lang}: streaming up to {count} clips from {len(files)} shard(s)", flush=True)
rows = prepare_corpus(name, repo, rev, lang, split, count, files, field, seen, tok,
per_shard=(split == "train"))
if len(rows) < (1 if SMOKE else count // 2):
raise RuntimeError(f"{name} {split}/{lang}: only {len(rows)} eligible clips")
write(dest, rows)
print(f" saved {len(rows)} clips", flush=True)
seen |= {(r["language"], r["text_key"]) for r in read(dest)}
candidates = [r for lang in LANGS for r in read(DATA / f"fleurs-train-{lang}.json")]
candidates += [r for lang in LANGS for r in read(DATA / f"candidates-{lang}.json")]
ids = [r["id"] for r in candidates]
if len(ids) != len(set(ids)):
raise RuntimeError("Duplicate candidate ids")
tmp = DATA / "candidates.jsonl.tmp"
tmp.write_text("".join(json.dumps(r, ensure_ascii=False) + "\n" for r in candidates), encoding="utf-8")
tmp.replace(DATA / "candidates.jsonl")
print(f"Prepared {len(candidates)} training candidates for the teacher.", flush=True)
# ---------------------------------------------------------------- filter
def filter_stage():
tok = tokenizer()
candidates = [json.loads(l) for l in (DATA / "candidates.jsonl").read_text(encoding="utf-8").splitlines() if l]
teacher = {}
for line in (DATA / "teacher.jsonl").read_text(encoding="utf-8").splitlines():
if line.strip():
row = json.loads(line)
teacher[row["id"]] = row
missing = [c["id"] for c in candidates if c["id"] not in teacher]
if missing:
raise RuntimeError(f"{len(missing)} candidates have no teacher label; rerun the teacher stage")
eval_keys = set()
for split in ("validation", "test"):
for lang in LANGS:
eval_keys |= {(r["language"], r["text_key"]) for r in read(DATA / f"fleurs-{split}-{lang}.json")}
for lang in LANGS:
eval_keys |= {(r["language"], r["text_key"]) for r in read(DATA / f"ood-{lang}.json")}
kept, stats, used = [], {}, set()
limits = {"fleurs": SIZES["anchor"], "voxpopuli": SIZES["keep"], "zeroth": SIZES["keep"]}
counts = {}
for c in candidates:
t = teacher[c["id"]]
group = f"{c['source']}/{c['language']}"
s = stats.setdefault(group, {"candidates": 0, "kept": 0, "rejected": {}})
s["candidates"] += 1
reason = None
text = t["text"]
segs = t["segments"]
if not text or not segs:
reason = "empty"
elif any(x["compression_ratio"] > THRESH["compression_ratio"] for x in segs):
reason = "compression_ratio"
elif any(x["avg_logprob"] < THRESH["avg_logprob"] for x in segs):
reason = "avg_logprob"
elif any(x["no_speech_prob"] > THRESH["no_speech_prob"] for x in segs):
reason = "no_speech_prob"
else:
e, n = error_units(c["gold"], text, c["language"])
rate = e / n if n else 1.0
if not n or rate > THRESH["error_rate"]:
reason = "disagrees_with_gold"
elif len(tok.encode(text)) + 3 > MAX_IDS:
reason = "too_many_tokens"
elif (c["language"], text_key(text)) in eval_keys or (c["language"], text_key(text)) in used:
reason = "duplicate_text"
elif counts.get(group, 0) >= limits[c["source"]]:
reason = "over_quota"
if reason:
s["rejected"][reason] = s["rejected"].get(reason, 0) + 1
continue
counts[group] = counts.get(group, 0) + 1
used.add((c["language"], text_key(text)))
s["kept"] += 1
kept.append({**c, "text": text, "teacher_error_rate": round(rate, 4)})
if SMOKE and not kept:
# Smoke only: keep the chain testable even if 2-clip samples all fail the filter.
kept = [{**c, "text": teacher[c["id"]]["text"] or c["gold"], "teacher_error_rate": None} for c in candidates]
for lang in LANGS:
new = sum(1 for r in kept if r["language"] == lang and r["source"] != "fleurs")
if not SMOKE and new < SIZES["keep"] // 4:
raise RuntimeError(f"Only {new} teacher-labeled {lang} clips passed the filter; see filter_stats.json")
write(DATA / "filter_stats.json", stats)
write(DATA / "train.json", kept)
for group, s in sorted(stats.items()):
print(f"{group:<16} kept {s['kept']:>4}/{s['candidates']:<4} rejected {s['rejected']}", flush=True)
print(f"Training set: {len(kept)} clips", flush=True)
# ---------------------------------------------------------------- evaluation
def evaluate(weights, rows, out):
"""Native .cact engine, automatic language detection (as Fleet-1). Cached by weights+rows hash."""
from cactus.agent.whistle import Whistle
identity = {"weights_sha256": sha(weights), "rows_sha256": hashlib.sha256(
json.dumps([r["id"] for r in rows]).encode()).hexdigest()}
out = Path(out)
if out.exists() and read(out).get("identity") == identity:
return read(out)["scores"]
model = Whistle(weights=str(weights))
predictions = []
for row in rows:
result = model.transcribe(str(DATA / row["audio"]))
e, n = error_units(row["gold"], result["text"], row["language"])
predictions.append({"id": row["id"], "language": row["language"], "reference": row["gold"],
"prediction": result["text"], "detected_language": result.get("language"),
"edits": int(e), "units": int(n)})
scores = {}
for lang in LANGS:
group = [p for p in predictions if p["language"] == lang]
if not group:
continue
units = sum(p["units"] for p in group)
if not units:
raise RuntimeError(f"No reference units for {lang}")
scores[lang] = {"metric": "CER" if lang == "ko" else "WER",
"error_rate": sum(p["edits"] for p in group) / units,
"edits": sum(p["edits"] for p in group), "reference_units": units, "clips": len(group),
"language_accuracy": sum(p["detected_language"] == lang for p in group) / len(group)}
out.parent.mkdir(parents=True, exist_ok=True)
with open(out.with_suffix(".jsonl"), "w", encoding="utf-8") as f:
for p in predictions:
f.write(json.dumps(p, ensure_ascii=False) + "\n")
write(out, {"identity": identity, "weights": str(weights), "scores": scores, "rss_bytes": rss()})
return scores
def macro(scores, langs=tuple(LANGS)):
return sum(scores[l]["error_rate"] for l in langs) / len(langs)
def rows_for(kind):
if kind in ("validation", "test"):
return [r for lang in LANGS for r in read(DATA / f"fleurs-{kind}-{lang}.json")]
return [r for lang in LANGS for r in read(DATA / f"ood-{lang}.json")]
def published(which):
if which == "whistle":
return hub(WHISTLE["repo"], "whistle.cact", WHISTLE["revision"])
spec = SKY if which == "sky" else FLEET1
return hub(spec["repo"], "whistle.cact", spec["revision"])
# ---------------------------------------------------------------- training
class Trainer:
def __init__(self, adapter_path, total):
import jax
import jax.numpy as jnp
import optax
from cactus.model import whistle_finetune as wf
from cactus.model.checkpoints import read_adapter
from cactus.model.whistle import Whistle
from cactus.model.quantize import configure_deploy, cq_ste_params, WEIGHT_BITS
self.jax, self.jnp, self.wf = jax, jnp, wf
self.adapter = read_adapter(adapter_path)
if not set(LANGS).issubset(self.adapter["languages"]) or not self.adapter["pieces"]:
raise ValueError("Start adapter lacks the Korean vocabulary or language tags")
self.tokenizer = wf.WhistleTokenizer(self.adapter["pieces"], self.adapter["languages"])
self.base = hub(WHISTLE["repo"], WHISTLE["checkpoint"], WHISTLE["revision"])
params, self.cfg = wf.load_base(self.base)
self.params = jax.tree.map(lambda x: jnp.asarray(x, dtype=jnp.float32), params)
self.tuned = replace(self.cfg, max_tokens=TOKENS, text=self.tokenizer.text,
languages=tuple(self.adapter["languages"]))
self.weights = {"lora": {tuple(k.split("/")): {n: jnp.asarray(v) for n, v in pair.items()}
for k, pair in self.adapter["lora"].items()},
"rows": jnp.asarray(self.adapter["rows"])}
configure_deploy(act_bits=8, kv_bits=8)
model = Whistle(self.tuned)
schedule = optax.warmup_cosine_decay_schedule(0., LR, max(1, total // 20), max(2, total))
self.optimizer = optax.chain(optax.clip_by_global_norm(1.), optax.adamw(schedule))
self.state = self.optimizer.init(self.weights)
scale, cfg, tok = self.adapter["scale"], self.cfg, self.tokenizer
def loss(weights, wav, lengths, ids):
merged = cq_ste_params(wf.tuned_params(self.params, cfg, tok, weights["lora"], scale, weights["rows"]),
WEIGHT_BITS)
logits = model.apply({"params": merged}, wav, lengths, ids[:, :-1], quant=True)
targets = ids[:, 1:]
mask = targets != cfg.pad
ce = optax.softmax_cross_entropy_with_integer_labels(logits, targets)
return (ce * mask).sum() / jnp.maximum(mask.sum(), 1.)
@jax.jit
def step(weights, state, wav, lengths, ids):
value, grad = jax.value_and_grad(loss)(weights, wav, lengths, ids)
updates, state = self.optimizer.update(grad, state, weights)
return optax.apply_updates(weights, updates), state, value
self.step_fn = step
def arrays(self, row):
import numpy as np
import soundfile as sf
wav, rate = sf.read(DATA / row["audio"], dtype="float32")
if rate != 16000 or wav.ndim != 1 or len(wav) > SECONDS * 16000:
raise ValueError(f"{row['id']}: expected <= {SECONDS} s of mono 16 kHz audio")
padded = np.zeros((1, SECONDS * 16000), np.float32)
padded[0, :len(wav)] = wav
ids = [self.cfg.bos, self.tuned.tag(row["language"]), *self.tokenizer.encode(row["text"]), self.cfg.eos]
if len(ids) > MAX_IDS:
raise ValueError(f"{row['id']}: transcript exceeds context")
tokens = np.full((1, MAX_IDS), self.cfg.pad, np.int32)
tokens[0, :len(ids)] = ids
return padded, np.array([len(wav)], np.int32), tokens
def step(self, row):
self.weights, self.state, value = self.step_fn(self.weights, self.state, *self.arrays(row))
value = float(value)
if not math.isfinite(value):
raise RuntimeError("Nonfinite training loss; stopping")
return value
def export(self, folder):
import numpy as np
from cactus.model.checkpoints import write_adapter, read_adapter
folder.mkdir(parents=True, exist_ok=True)
a = {**self.adapter, "base": f"{WHISTLE['repo']}@{WHISTLE['revision']}/{WHISTLE['checkpoint']}", "seed": SEED,
"lora": {"/".join(k): {n: np.asarray(v) for n, v in pair.items()}
for k, pair in self.weights["lora"].items()},
"rows": np.asarray(self.weights["rows"])}
tmp = folder / "adapter.tmp.safetensors"
write_adapter(str(tmp), a)
back = read_adapter(str(tmp))
if back["pieces"] != a["pieces"] or back["languages"] != a["languages"] or not np.array_equal(back["rows"], a["rows"]):
raise ValueError("Adapter round trip changed tokenizer or rows")
tmp.replace(folder / "fleet2_lora.safetensors")
params = self.wf.tuned_params(self.params, self.cfg, self.tokenizer, self.weights["lora"],
self.adapter["scale"], a["rows"])
self.wf.build(params, self.cfg, self.tokenizer, str(folder / "whistle.tmp.cact"))
(folder / "whistle.tmp.cact").replace(folder / "whistle.cact")
def save(self, path, meta):
import numpy as np
leaves = self.jax.tree.leaves((self.weights, self.state))
tmp = path.with_suffix(".tmp")
with tmp.open("wb") as f:
np.savez(f, metadata=np.array(json.dumps(meta)), **{f"a{i}": np.asarray(x) for i, x in enumerate(leaves)})
tmp.replace(path)
def restore(self, path):
import numpy as np
leaves, tree = self.jax.tree.flatten((self.weights, self.state))
with np.load(path, allow_pickle=False) as data:
values = [self.jnp.asarray(data[f"a{i}"]) for i in range(len(leaves))]
if any(a.shape != b.shape for a, b in zip(values, leaves)):
raise ValueError("Resume checkpoint does not match this model")
self.weights, self.state = self.jax.tree.unflatten(tree, values)
return json.loads(str(data["metadata"]))
def start_adapter(which):
spec = SKY if which == "sky" else FLEET1
return hub(spec["repo"], spec["adapter"], spec["revision"])
def stage0():
from cactus.model.checkpoints import read_adapter
sky, fleet = read_adapter(start_adapter("sky")), read_adapter(start_adapter("fleet1"))
if sky["pieces"] != fleet["pieces"] or sky["languages"] != fleet["languages"]:
raise RuntimeError("Sky-Kim and Fleet-1 adapters use different vocabularies")
val = rows_for("validation")
scores = {name: evaluate(published(name), val, RUN / f"eval/validation-{name}.json") for name in ("sky", "fleet1")}
scores["whistle"] = evaluate(published("whistle"), [r for r in val if r["language"] in WHISTLE_LANGS],
RUN / "eval/validation-whistle.json")
m = {name: macro(scores[name]) for name in ("sky", "fleet1")}
chosen = "fleet1" if m["fleet1"] < m["sky"] else "sky" # tie keeps the source
for name, s in scores.items():
print(name, {l: f"{v['error_rate']:.2%}" for l, v in s.items()}, flush=True)
print(f"Macro validation error: Sky-Kim {m['sky']:.4f}, Fleet-1 {m['fleet1']:.4f} -> start from {chosen}", flush=True)
# Audit: rebuild the chosen start model from base + adapter; predictions must match the published file.
trainer = Trainer(start_adapter(chosen), 1)
trainer.export(RUN / "checkpoints/start-rebuilt")
rebuilt = evaluate(RUN / "checkpoints/start-rebuilt/whistle.cact", val, RUN / "eval/validation-start-rebuilt.json")
a = [json.loads(l) for l in (RUN / f"eval/validation-{chosen}.jsonl").read_text(encoding="utf-8").splitlines()]
b = [json.loads(l) for l in (RUN / "eval/validation-start-rebuilt.jsonl").read_text(encoding="utf-8").splitlines()]
mismatches = [x["id"] for x, y in zip(a, b) if (x["prediction"], x["detected_language"]) != (y["prediction"], y["detected_language"])]
audit = {"chosen": chosen, "byte_identical": sha(published(chosen)) == sha(RUN / "checkpoints/start-rebuilt/whistle.cact"),
"prediction_mismatches": mismatches, "passed": not mismatches and len(a) == len(b)}
write(RUN / "export-audit.json", audit)
if not audit["passed"]:
raise RuntimeError("Rebuilt start model does not reproduce the published predictions; see export-audit.json")
immutable(RUN / "init.json", {"chosen": chosen, "macro_validation": m, "rule": "lower 8-language mean validation error; tie keeps Sky-Kim",
"adapter": SKY if chosen == "sky" else FLEET1, "rebuilt_macro": macro(rebuilt)})
print("Stage 0 audit passed.", flush=True)
def train():
import numpy as np
init = read(RUN / "init.json")
training = read(DATA / "train.json")
total = EPOCHS * len(training)
points = sorted({max(1, round(k * total / 4)) for k in (1, 2, 3, 4)})
immutable(RUN / "train-settings.json", {"init": init["chosen"], "clips": len(training), "total_steps": total,
"eval_points": points, "train_sha256": sha(DATA / "train.json")})
if (RUN / "trained.json").exists():
print("Training already complete.", flush=True)
return
trainer = Trainer(start_adapter(init["chosen"]), total)
checkpoint = RUN / "resume.npz"
meta = {"step": 0, "evals": []}
if checkpoint.exists():
meta = trainer.restore(checkpoint)
print(f"Resuming at step {meta['step']}/{total}", flush=True)
val = rows_for("validation")
print(f"CPU training: {len(training)} clips x {EPOCHS} epochs = {total} steps; eval at {points}. "
"First JAX compile takes a few minutes.", flush=True)
start, first = time.monotonic(), meta["step"]
while meta["step"] < total:
epoch, pos = divmod(meta["step"], len(training))
order = np.random.default_rng(SEED + epoch).permutation(len(training))
loss = trainer.step(training[int(order[pos])])
meta["step"] += 1
if meta["step"] % 25 == 0 or meta["step"] == first + 1:
done = meta["step"] - first
left = (time.monotonic() - start) / done * (total - meta["step"]) / 3600
print(f"step {meta['step']}/{total} loss {loss:.4f} ~{left:.1f} h left rss {rss() / 2**30:.1f} GB", flush=True)
if meta["step"] in points:
folder = RUN / f"checkpoints/step-{meta['step']}"
trainer.export(folder)
scores = evaluate(folder / "whistle.cact", val, RUN / f"eval/validation-step-{meta['step']}.json")
meta["evals"].append({"step": meta["step"], "macro": macro(scores),
"scores": {l: s["error_rate"] for l, s in scores.items()}})
print(f"eval step {meta['step']}: macro validation error {macro(scores):.4f}", flush=True)
trainer.save(checkpoint, meta)
elif meta["step"] % 200 == 0:
trainer.save(checkpoint, meta)
trainer.save(checkpoint, meta)
write(RUN / "trained.json", {**meta, "seconds_last_invocation": time.monotonic() - start})
def select():
init, trained = read(RUN / "init.json"), read(RUN / "trained.json")
baseline = init["macro_validation"][init["chosen"]]
best = min(trained["evals"], key=lambda e: (e["macro"], e["step"]))
if best["macro"] < baseline:
selection = {"step": best["step"], "macro_validation": best["macro"], "start_macro_validation": baseline,
"reason": "Lowest macro validation error, better than the start model"}
else:
# The user chose to publish no matter what (2026-11-15), so the best checkpoint always goes forward.
selection = {"step": best["step"], "macro_validation": best["macro"], "start_macro_validation": baseline,
"reason": "No checkpoint beat the start model on validation; best validation checkpoint used anyway"}
immutable(RUN / "selection.json", selection)
print(selection["reason"], f"(best {best['macro']:.4f} vs start {baseline:.4f})", flush=True)
def test():
selection = read(RUN / "selection.json")
report = {"selection": selection, "init": read(RUN / "init.json"), "filter": read(DATA / "filter_stats.json")}
if selection["step"] is None:
report["gate"] = {"passed": False, "reason": selection["reason"]}
write(RUN / "report.json", report)
print("No improvement on validation; nothing to test or publish.", flush=True)
return
weights = RUN / f"checkpoints/step-{selection['step']}/whistle.cact"
models = {"Fleet-2": weights, "Fleet-1": published("fleet1"), "Sky-Kim": published("sky"),
"Whistle": published("whistle")}
results = {}
for split in ("test", "ood"):
rows = rows_for(split)
results[split] = {}
for name, path in models.items():
use = [r for r in rows if name != "Whistle" or r["language"] in WHISTLE_LANGS]
results[split][name] = evaluate(path, use, RUN / f"eval/{split}-{name}.json")
if not SMOKE:
historical = read(VENDOR / "evaluation.json")
for name, key in (("Sky-Kim", "source"), ("Fleet-1", "Fleet-1")):
for lang in LANGS:
if abs(results["test"][name][lang]["error_rate"] - historical[key][lang]["error_rate"]) > 1e-10:
raise RuntimeError(f"{name} {lang} test score does not reproduce Fleet-1's published table")
f2, f1, sky = results["test"]["Fleet-2"], results["test"]["Fleet-1"], results["test"]["Sky-Kim"]
wins = [l for l in LANGS if f2[l]["error_rate"] < f1[l]["error_rate"]]
ko_ok = f2["ko"]["error_rate"] <= sky["ko"]["error_rate"]
passed = len(wins) >= 5 and ko_ok
report.update(results=results, gate={"passed": passed, "beats_fleet1_in": wins, "korean_not_worse_than_sky": ko_ok,
"rule": ">=5 of 8 languages better than Fleet-1 and Korean CER <= Sky-Kim, on Fleet-1's FLEURS test clips"})
write(RUN / "report.json", report)
for lang in LANGS:
row = " ".join(f"{n} {results['test'][n][lang]['error_rate']:.2%}" for n in models if lang in results["test"][n])
print(f"{lang} ({f2[lang]['metric']}): {row}", flush=True)
print(f"Gate {'PASSED' if passed else 'FAILED'}: better than Fleet-1 in {len(wins)}/8 ({', '.join(wins)}); "
f"Korean not worse than Sky-Kim: {ko_ok}", flush=True)
# ---------------------------------------------------------------- publish
def pct(x):
return f"{x:.2%}"
def card(report, init_repo):
res, f = report["results"], report["filter"]
t = res["test"]
lines = []
for lang in LANGS:
w = pct(t["Whistle"][lang]["error_rate"]) if lang in t["Whistle"] else "n/a"
lines.append(f"| {lang} | {t['Fleet-2'][lang]['metric']} | {w} | {pct(t['Sky-Kim'][lang]['error_rate'])} | "
f"{pct(t['Fleet-1'][lang]['error_rate'])} | **{pct(t['Fleet-2'][lang]['error_rate'])}** |")
o = res["ood"]
ood = []
for lang in LANGS:
w = pct(o["Whistle"][lang]["error_rate"]) if lang in o["Whistle"] else "n/a"
src = "Zeroth-Korean" if lang == "ko" else "VoxPopuli"
ood.append(f"| {lang} | {src} | {o['Fleet-2'][lang]['clips']} | {w} | {pct(o['Sky-Kim'][lang]['error_rate'])} | "
f"{pct(o['Fleet-1'][lang]['error_rate'])} | **{pct(o['Fleet-2'][lang]['error_rate'])}** |")
kept = sum(s["kept"] for s in f.values())
cand = sum(s["candidates"] for s in f.values())
new = sum(s["kept"] for g, s in f.items() if not g.startswith("fleurs"))
anchor = kept - new
gate = report["gate"]
sel = report["selection"]
trained = read(RUN / "train-settings.json")
wins = gate["beats_fleet1_in"]
status = (f"Fleet-2 beats Fleet-1 in {len(wins)} of 8 languages ({', '.join(wins) or 'none'}). "
f"Korean CER is no worse than Sky-Kim: {'yes' if gate['korean_not_worse_than_sky'] else 'no'}.")
if not gate["passed"]:
status += " **Improvement over Fleet-1 was NOT demonstrated. Check each language; some may regress.**"
if sel["macro_validation"] >= sel["start_macro_validation"]:
status += " No training checkpoint beat the start model on validation; the best one was published anyway."
return f"""---
license: apache-2.0
base_model: {init_repo}
language: [en, de, fr, es, it, nl, pl, ko]
pipeline_tag: automatic-speech-recognition
datasets: [facebook/voxpopuli, Bingsu/zeroth-korean, google/fleurs]
tags: [whistle, cactus, lora, speech-to-text, distillation, pseudo-labels]
---
# Fleet-2
A 30 MB on-device speech-to-text model (Cactus Whistle architecture) for English, German, French, Spanish, Italian, Dutch, Polish and Korean, trained on transcripts written by [openai/whisper-large-v3-turbo](https://huggingface.co/openai/whisper-large-v3-turbo).
**Collection:** [Speech recognition](https://huggingface.co/collections/{COLLECTION})
## What changed from Fleet-1
- **Teacher:** Whisper-large-v3-turbo transcribed new audio, and Fleet-2 learned from those transcripts. This is sequence-level distillation, as in Distil-Whisper. Whisper and Whistle use different tokenizers, so no logits were matched.
- **New audio:** VoxPopuli (European Parliament speech) for the seven European languages and Zeroth-Korean for Korean. Fleet-1 used FLEURS only.
- **Start point:** chosen by validation score between Sky-Kim's Korean Whistle and Fleet-1. This run started from `{init_repo}`.
## Evaluation
Each language has 100 short FLEURS test clips. These are the same clips, filtered the same way, as Fleet-1's table. All models ran in the same native Cactus engine with automatic language detection. Lower is better. Whistle has no Korean.
| Language | Metric | Whistle | Sky-Kim | Fleet-1 | Fleet-2 |
|---|---|---:|---:|---:|---:|
{chr(10).join(lines)}
{status}
<details>
<summary>Out-of-domain clips (VoxPopuli / Zeroth-Korean test splits)</summary>
| Language | Source | Clips | Whistle | Sky-Kim | Fleet-1 | Fleet-2 |
|---|---|---:|---:|---:|---:|---:|
{chr(10).join(ood)}
Fleet-2 trained on other clips from these corpora, so this table shows in-corpus generalisation, not a fully independent test.
</details>
These results come from short clips of 12 s or less. They are not full benchmark scores and not significance tests. Accuracy on long, noisy or conversational audio was not measured. Upstream models may have seen FLEURS training data before.
## Training
- **Data:** {kept} clips kept out of {cand} that the teacher labeled: {new} new VoxPopuli/Zeroth clips and {anchor} of Fleet-1's FLEURS training clips. All clips are 12 s or less.
- **Teacher decoding:** language forced, temperature 0, no conditioning on previous text. Runtime: the MLX conversion `mlx-community/whisper-large-v3-turbo`.
- **Filter:** a clip was kept only if all of these held:
- compression ratio ≤ {THRESH['compression_ratio']}
- average log-probability ≥ {THRESH['avg_logprob']}
- no-speech probability ≤ {THRESH['no_speech_prob']}
- teacher text within {THRESH['error_rate']:.0%} WER of the human transcript (CER for Korean)
The training target is the teacher's text.
- **Excluded from training:** validation and test clips, and any transcript that matches an evaluation transcript.
- **Optimisation:** the start adapter (LoRA plus Korean vocabulary rows) continued with AdamW at {LR}, warmup/cosine schedule, batch 1, {EPOCHS} epochs ({trained['total_steps']} steps), clip 1.0, W4/A8 quantisation-aware training on CPU (Apple M4).
- **Checkpoint selection:** step {sel['step']}, chosen by macro validation error on Fleet-1's FLEURS validation clips ({sel['macro_validation']:.4f}, against {sel['start_macro_validation']:.4f} for the start model). Test clips were not used for selection.
- **Release:** published whatever the scores. The comparison above is reported as measured, including any regressions.
`training_manifest.json` records every clip, teacher transcript and pinned revision. `evaluation.json` has every score. `fleet2.py`, `teacher.py` and `run.sh` reproduce the run.
## Usage
```python
import cactus
from huggingface_hub import hf_hub_download
path = hf_hub_download("{DEST}", "whistle.cact")
print(cactus.transcribe("clip.wav", weights=path)["text"])
```
Use 16 kHz mono WAV and `cactus-compute` at revision `{CACTUS_CODE}`. This is a Cactus model, not a Transformers checkpoint.
## License and attribution
- **Model:** Apache License 2.0. The original Whistle LICENSE is kept verbatim. Architecture and pretrained weights: Cactus Compute's Whistle (Mroz et al., 2026). Korean vocabulary and adapter lineage: [Sky-Kim/whistle-finetuned-ko](https://huggingface.co/Sky-Kim/whistle-finetuned-ko) and [Lucasgitguy-co/Fleet-1](https://huggingface.co/Lucasgitguy-co/Fleet-1). See `UPSTREAM_MODEL_CARD.md`.
- **Teacher:** [openai/whisper-large-v3-turbo](https://huggingface.co/openai/whisper-large-v3-turbo), MIT License. Only its transcripts were used; none of its weights are included.
- **Data:**
- [VoxPopuli](https://huggingface.co/datasets/facebook/voxpopuli): CC0
- [Zeroth-Korean](https://www.openslr.org/40/): CC BY 4.0
- [FLEURS](https://huggingface.co/datasets/google/fleurs) (Conneau et al., 2022): CC BY 4.0
No audio is included in this repository. See `NOTICE`.
"""
NOTICE = """Fleet-2
Copyright 2026 Lucasgitguy-co. Licensed under the Apache License, Version 2.0 (see LICENSE).
This model is a derivative of:
- Whistle by Cactus Compute (Apache-2.0): architecture, base weights, runtime.
- Sky-Kim/whistle-finetuned-ko (Apache-2.0): Korean vocabulary, language tags, LoRA adapter.
- Lucasgitguy-co/Fleet-1 (Apache-2.0): adapter lineage.
Training targets were produced by openai/whisper-large-v3-turbo (MIT License, Copyright (c) 2022 OpenAI),
run through the mlx-community/whisper-large-v3-turbo conversion. No Whisper weights are distributed here.
Training and evaluation audio (not redistributed):
- VoxPopuli, Wang et al. (2021), CC0 1.0.
- Zeroth-Korean, Atlas Guide Inc. and Gridspace Inc., OpenSLR 40, CC BY 4.0. Transcripts were replaced by teacher output.
- FLEURS, Conneau et al. (2022), CC BY 4.0. Audio converted to 16 kHz mono PCM and subsetted.
"""
def assemble(report):
init = read(RUN / "init.json")
spec = SKY if init["chosen"] == "sky" else FLEET1
step = report["selection"]["step"]
src = RUN / f"checkpoints/step-{step}"
if PUB.exists():
shutil.rmtree(PUB)
PUB.mkdir(parents=True)
shutil.copyfile(src / "whistle.cact", PUB / "whistle.cact")
shutil.copyfile(src / "fleet2_lora.safetensors", PUB / "fleet2_lora.safetensors")
shutil.copyfile(hub(SKY["repo"], "LICENSE", SKY["revision"]), PUB / "LICENSE")
shutil.copyfile(hub(SKY["repo"], "README.md", SKY["revision"]), PUB / "UPSTREAM_MODEL_CARD.md")
for name in ("fleet2.py", "teacher.py", "run.sh", "teacher_pin.json", "requirements-student.lock",
"requirements-teacher.lock"):
shutil.copyfile(HERE / name, PUB / name)
(PUB / "NOTICE").write_text(NOTICE, encoding="utf-8")
cfg = read(hub(spec["repo"], "config.json", spec["revision"]))
cfg.update(name_or_path=DEST, base_model=spec["repo"])
cfg["finetune"] = {"method": "continued LoRA on teacher pseudo-labels", "teacher": "openai/whisper-large-v3-turbo",
"start": f"{spec['repo']}@{spec['revision']}", "steps": step, "adapter": "fleet2_lora.safetensors"}
write(PUB / "config.json", cfg)
write(PUB / "evaluation.json", report)
train_rows = [{k: r.get(k) for k in ("id", "source", "language", "text", "gold", "teacher_error_rate", "audio_id",
"fleurs_id", "shard", "row_index", "speaker", "seconds", "sha256")}
for r in read(DATA / "train.json")]
write(PUB / "training_manifest.json", {
"settings": read(RUN / "settings.json"), "dataset_revisions": read(DATA / "revisions.json"),
"init": init, "train_settings": read(RUN / "train-settings.json"), "selection": report["selection"],
"eval_history": read(RUN / "trained.json")["evals"], "filter": read(DATA / "filter_stats.json"),
"train": train_rows,
"validation_ids": [r["id"] for r in rows_for("validation")], "test_ids": [r["id"] for r in rows_for("test")],
"ood_ids": [r["id"] for r in rows_for("ood")]})
(PUB / "README.md").write_text(card(report, spec["repo"]), encoding="utf-8")
return spec
def publish():
from huggingface_hub import HfApi
report = read(RUN / "report.json")
if not report.get("results"):
raise RuntimeError("report.json has no test results; run the test stage first")
if not report["gate"]["passed"]:
print("Comparison check failed (Fleet-2 does not clearly beat Fleet-1); publishing anyway with the results on the card.", flush=True)
assemble(report)
api = HfApi()
who = api.whoami()
if who["name"] != OWNER:
raise RuntimeError(f"Signed in to Hugging Face as {who['name']}, not {OWNER}; run `hf auth login` first")
marker = RUN / "destination.json"
exists = api.repo_exists(DEST)
if exists and not marker.exists():
raise RuntimeError(f"{DEST} already exists and was not created by this run; refusing to overwrite")
if SMOKE:
print(f"Smoke dry run OK: signed in as {who['name']}, {DEST} exists={exists}, publish folder assembled "
f"({len(list(PUB.iterdir()))} files). Nothing uploaded.", flush=True)
return
write(marker, {"repo": DEST, "run": str(RUN)})
api.create_repo(DEST, private=False, exist_ok=True)
commit = api.upload_folder(repo_id=DEST, folder_path=str(PUB), commit_message="Fleet-2: Whistle student of Whisper-large-v3-turbo")
try:
api.add_collection_item(COLLECTION, item_id=DEST, item_type="model", exists_ok=True)
except Exception as error: # the model is published either way
print(f"Could not add to the collection ({error}); add it by hand.", flush=True)
# Verify the published file works: download from the Hub and transcribe one test clip.
from huggingface_hub import hf_hub_download
from cactus.agent.whistle import Whistle
path = hf_hub_download(DEST, "whistle.cact", force_download=True)
if sha(path) != sha(PUB / "whistle.cact"):
raise RuntimeError("Downloaded whistle.cact differs from the local file")
row = rows_for("test")[0]
text = Whistle(weights=path).transcribe(str(DATA / row["audio"]))["text"]
write(RUN / "published.json", {"repo": DEST, "commit": getattr(commit, "oid", str(commit)), "check_clip": row["id"], "check_text": text})
print(f"Published https://huggingface.co/{DEST} (check transcription: {text!r})", flush=True)
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument("stage", choices=["prepare", "filter", "stage0", "train", "select", "test", "publish"])
a = p.parse_args()
RUN.mkdir(parents=True, exist_ok=True)
{"prepare": prepare, "filter": filter_stage, "stage0": stage0, "train": train, "select": select,
"test": test, "publish": publish}[a.stage]()
if __name__ == "__main__":
main()