"""Fleet-2: Whistle student trained on Whisper-large-v3-turbo pseudo-labels (sequence-level distillation). Stages (run through run.sh): prepare, filter, stage0, train, select, test, publish. The teacher pass lives in teacher.py (separate MLX environment). """ import argparse import hashlib import io import json import math import os import shutil import sys import time from dataclasses import replace from pathlib import Path HERE = Path(__file__).resolve().parent RUN = Path(os.environ.get("FLEET2_RUN", HERE / "runs/main")).resolve() SMOKE = os.environ.get("FLEET2_SMOKE") == "1" DATA = RUN / "data" PUB = RUN / "publish" LANGS = dict(en="en_us", de="de_de", fr="fr_fr", es="es_419", it="it_it", nl="nl_nl", pl="pl_pl", ko="ko_kr") WHISTLE_LANGS = ("en", "de", "fr", "es", "it", "nl", "pl") SEED, SECONDS, TOKENS, EPOCHS, LR = 42, 12, 160, 2, 2e-5 MAX_IDS = TOKENS + 1 # bos + tag + text + eos must fit, same rule as Fleet-1 SPEAKER_CAP = 20 THRESH = dict(compression_ratio=2.4, avg_logprob=-1.0, no_speech_prob=0.6, error_rate=0.10) if SMOKE: SIZES = dict(candidates=3, keep=2, anchor=2, validation=2, test=2, ood=2) else: SIZES = dict(candidates=600, keep=400, anchor=200, validation=50, test=100, ood=50) SKY = dict(repo="Sky-Kim/whistle-finetuned-ko", revision="b8a4f3188a828a0e87e2713b2d3e83d71d893068", adapter="adapter/whistle_ko_lora.safetensors") FLEET1 = dict(repo="Lucasgitguy-co/Fleet-1", revision="0cc581616f7e5493e2e3999e71c134cedc879a6d", adapter="fleet_lora.safetensors") WHISTLE = dict(repo="Cactus-Compute/whistle", revision="741118687e1f008f06d1351a402a731274789790", checkpoint="checkpoints/whistle.safetensors") CACTUS_CODE = "ee2ffc67fe0f9537d4928e158440c7d228a63100" DEST = "Lucasgitguy-co/Fleet-2" OWNER = "Lucasgitguy-co" COLLECTION = "Lucasgitguy-co/speech-recognition-6aef2dcbe0ea208136b0ddc9" VENDOR = HERE / "vendor" # ---------------------------------------------------------------- helpers def read(path): return json.loads(Path(path).read_text(encoding="utf-8")) def write(path, data): path = Path(path) path.parent.mkdir(parents=True, exist_ok=True) tmp = path.with_suffix(path.suffix + ".tmp") tmp.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") tmp.replace(path) def immutable(path, data): """Write once; later runs must agree, so a resumed run never mixes settings.""" if Path(path).exists(): if read(path) != data: raise RuntimeError(f"{path} records different settings from this run. Use a fresh run folder.") return write(path, data) def sha(path): h = hashlib.sha256() with open(path, "rb") as f: for block in iter(lambda: f.read(1 << 20), b""): h.update(block) return h.hexdigest() def text_key(text): return " ".join(text.casefold().split()) def rss(): import psutil return psutil.Process().memory_info().rss def hub(repo, name, revision): from huggingface_hub import hf_hub_download return hf_hub_download(repo, name, revision=revision) def tokenizer(): from cactus.model.checkpoints import read_adapter from cactus.model.whistle_finetune import WhistleTokenizer a = read_adapter(hub(SKY["repo"], SKY["adapter"], SKY["revision"])) return WhistleTokenizer(a["pieces"], a["languages"]) def error_units(reference, hypothesis, lang): """Fleet-1 scoring: CER for Korean (spaces removed), normalized WER otherwise.""" from cactus.model.whistle_finetune import errors, _normalizer if lang == "ko": import editdistance norm = _normalizer(lang) ref, hyp = norm(reference).replace(" ", ""), norm(hypothesis).replace(" ", "") return editdistance.eval(ref, hyp), len(ref) return errors(reference, hypothesis, lang) def decode_audio(audio): import numpy as np import soundfile as sf import soxr wav, rate = sf.read(io.BytesIO(audio["bytes"]) if audio.get("bytes") else audio["path"], dtype="float32", always_2d=True) wav = wav.mean(axis=1) if rate != 16000: wav = soxr.resample(wav, rate, 16000) return np.ascontiguousarray(wav, dtype=np.float32) def save_wav(path, wav): import soundfile as sf path.parent.mkdir(parents=True, exist_ok=True) sf.write(path, wav, 16000, subtype="PCM_16") def stream(repo, revision, files): from datasets import load_dataset, Audio urls = [f"https://huggingface.co/datasets/{repo}/resolve/{revision}/{f}" for f in files] return load_dataset("parquet", data_files={"x": urls}, split="x", streaming=True, batch_size=8).cast_column("audio", Audio(decode=False)) # ---------------------------------------------------------------- prepare def revisions(): from huggingface_hub import HfApi path = DATA / "revisions.json" if not path.exists(): api = HfApi() write(path, {"google/fleurs": read(VENDOR / "training_manifest.json")["dataset_revision"], "facebook/voxpopuli": api.dataset_info("facebook/voxpopuli").sha, "Bingsu/zeroth-korean": api.dataset_info("Bingsu/zeroth-korean").sha}) return read(path) def prepare_fleurs(revs, tok): """Reproduce Fleet-1's FLEURS rows exactly (sha256 + text checked).""" from huggingface_hub import HfApi manifest = read(VENDOR / "training_manifest.json") rev = revs["google/fleurs"] files = None plan = {"train": SIZES["anchor"], "validation": SIZES["validation"], "test": SIZES["test"]} for split, count in plan.items(): for lang, config in LANGS.items(): dest = DATA / f"fleurs-{split}-{lang}.json" if dest.exists(): continue wanted = [r for r in manifest["splits"][split] if r["language"] == lang][:count] if len(wanted) != count: raise RuntimeError(f"Fleet-1 manifest has only {len(wanted)} {split}/{lang} rows") by_index = {r["row_index"]: r for r in wanted} files = files or HfApi().list_repo_files("google/fleurs", repo_type="dataset", revision=rev) paths = sorted(p for p in files if p.startswith(f"{config}/{split}/") and p.endswith(".parquet")) print(f"FLEURS {split}/{lang}: reproducing {count} Fleet-1 rows", flush=True) found = {} for index, row in enumerate(stream("google/fleurs", rev, paths)): if index > max(by_index): break if index not in by_index: continue ref = by_index[index] text = (row.get("raw_transcription") or row.get("transcription") or "").strip() path = DATA / "audio/fleurs" / split / lang / f"{index:06d}.wav" save_wav(path, decode_audio(row["audio"])) if text != ref["text"] or sha(path) != ref["sha256"]: raise RuntimeError(f"FLEURS {split}/{lang}/{index} does not reproduce Fleet-1's published example") found[index] = {"id": f"fleurs/{split}/{lang}/{index}", "source": "fleurs", "split": split, "audio": str(path.relative_to(DATA)), "gold": text, "language": lang, "row_index": index, "fleurs_id": ref.get("id"), "sha256": ref["sha256"], "text_key": text_key(text)} if set(found) != set(by_index): raise RuntimeError(f"FLEURS {split}/{lang}: missing rows {sorted(set(by_index) - set(found))}") write(dest, [found[r["row_index"]] for r in wanted]) def prepare_corpus(name, repo, rev, lang, split, count, files, gold_field, seen, tok, per_shard): """Stream the first eligible rows of each shard. Never downloads whole shards.""" out, speakers = [], {} quota = math.ceil(count / len(files)) if per_shard else count shards = files if per_shard else [files] for shard_no, shard in enumerate(shards): shard_files = [shard] if per_shard else shard taken = 0 for index, row in enumerate(stream(repo, rev, shard_files)): if len(out) >= count or taken >= quota: break text = (row.get(gold_field) or row.get("normalized_text") or "").strip() key = text_key(text) if not text or (lang, key) in seen: continue if len(tok.encode(text)) + 3 > MAX_IDS: continue speaker = row.get("speaker_id") if speaker and speakers.get(speaker, 0) >= SPEAKER_CAP: continue wav = decode_audio(row["audio"]) if not 1 * 16000 <= len(wav) <= SECONDS * 16000: continue path = DATA / "audio" / name / split / lang / f"{shard_no:02d}-{index:06d}.wav" save_wav(path, wav) out.append({"id": f"{name}/{split}/{lang}/{shard_no}/{index}", "source": name, "split": split, "audio": str(path.relative_to(DATA)), "gold": text, "language": lang, "shard": shard_files[0].rsplit("/", 1)[-1], "row_index": index, "audio_id": row.get("audio_id"), "speaker": speaker, "sha256": sha(path), "seconds": round(len(wav) / 16000, 3), "text_key": key}) seen.add((lang, key)) if speaker: speakers[speaker] = speakers.get(speaker, 0) + 1 taken += 1 if len(out) >= count: break return out def corpus_files(repo, rev, lang, split): from huggingface_hub import HfApi files = HfApi().list_repo_files(repo, repo_type="dataset", revision=rev) if repo == "facebook/voxpopuli": prefix = f"{lang}/{split}-" else: prefix = f"data/{split}-" chosen = sorted(f for f in files if f.startswith(prefix) and f.endswith(".parquet")) if not chosen: raise RuntimeError(f"No {split} parquet files for {repo} {lang}") return chosen def prepare(): DATA.mkdir(parents=True, exist_ok=True) immutable(RUN / "settings.json", {"smoke": SMOKE, "sizes": SIZES, "seed": SEED, "seconds": SECONDS, "tokens": TOKENS, "epochs": EPOCHS, "lr": LR, "thresholds": THRESH, "speaker_cap": SPEAKER_CAP, "sky": SKY, "fleet1": FLEET1, "whistle": WHISTLE, "cactus_code": CACTUS_CODE, "teacher": read(HERE / "teacher_pin.json")}) revs = revisions() tok = tokenizer() prepare_fleurs(revs, tok) # Every evaluation transcript is excluded from training data. seen = set() for split in ("validation", "test"): for lang in LANGS: seen |= {(r["language"], r["text_key"]) for r in read(DATA / f"fleurs-{split}-{lang}.json")} # Out-of-domain test (report only), then training candidates. for split, count, kind in (("test", SIZES["ood"], "ood"), ("train", SIZES["candidates"], "candidates")): for lang in LANGS: dest = DATA / f"{kind}-{lang}.json" if not dest.exists(): if lang == "ko": repo, name, field = "Bingsu/zeroth-korean", "zeroth", "text" else: repo, name, field = "facebook/voxpopuli", "voxpopuli", "raw_text" rev = revs[repo] files = corpus_files(repo, rev, lang, split) print(f"{name} {split}/{lang}: streaming up to {count} clips from {len(files)} shard(s)", flush=True) rows = prepare_corpus(name, repo, rev, lang, split, count, files, field, seen, tok, per_shard=(split == "train")) if len(rows) < (1 if SMOKE else count // 2): raise RuntimeError(f"{name} {split}/{lang}: only {len(rows)} eligible clips") write(dest, rows) print(f" saved {len(rows)} clips", flush=True) seen |= {(r["language"], r["text_key"]) for r in read(dest)} candidates = [r for lang in LANGS for r in read(DATA / f"fleurs-train-{lang}.json")] candidates += [r for lang in LANGS for r in read(DATA / f"candidates-{lang}.json")] ids = [r["id"] for r in candidates] if len(ids) != len(set(ids)): raise RuntimeError("Duplicate candidate ids") tmp = DATA / "candidates.jsonl.tmp" tmp.write_text("".join(json.dumps(r, ensure_ascii=False) + "\n" for r in candidates), encoding="utf-8") tmp.replace(DATA / "candidates.jsonl") print(f"Prepared {len(candidates)} training candidates for the teacher.", flush=True) # ---------------------------------------------------------------- filter def filter_stage(): tok = tokenizer() candidates = [json.loads(l) for l in (DATA / "candidates.jsonl").read_text(encoding="utf-8").splitlines() if l] teacher = {} for line in (DATA / "teacher.jsonl").read_text(encoding="utf-8").splitlines(): if line.strip(): row = json.loads(line) teacher[row["id"]] = row missing = [c["id"] for c in candidates if c["id"] not in teacher] if missing: raise RuntimeError(f"{len(missing)} candidates have no teacher label; rerun the teacher stage") eval_keys = set() for split in ("validation", "test"): for lang in LANGS: eval_keys |= {(r["language"], r["text_key"]) for r in read(DATA / f"fleurs-{split}-{lang}.json")} for lang in LANGS: eval_keys |= {(r["language"], r["text_key"]) for r in read(DATA / f"ood-{lang}.json")} kept, stats, used = [], {}, set() limits = {"fleurs": SIZES["anchor"], "voxpopuli": SIZES["keep"], "zeroth": SIZES["keep"]} counts = {} for c in candidates: t = teacher[c["id"]] group = f"{c['source']}/{c['language']}" s = stats.setdefault(group, {"candidates": 0, "kept": 0, "rejected": {}}) s["candidates"] += 1 reason = None text = t["text"] segs = t["segments"] if not text or not segs: reason = "empty" elif any(x["compression_ratio"] > THRESH["compression_ratio"] for x in segs): reason = "compression_ratio" elif any(x["avg_logprob"] < THRESH["avg_logprob"] for x in segs): reason = "avg_logprob" elif any(x["no_speech_prob"] > THRESH["no_speech_prob"] for x in segs): reason = "no_speech_prob" else: e, n = error_units(c["gold"], text, c["language"]) rate = e / n if n else 1.0 if not n or rate > THRESH["error_rate"]: reason = "disagrees_with_gold" elif len(tok.encode(text)) + 3 > MAX_IDS: reason = "too_many_tokens" elif (c["language"], text_key(text)) in eval_keys or (c["language"], text_key(text)) in used: reason = "duplicate_text" elif counts.get(group, 0) >= limits[c["source"]]: reason = "over_quota" if reason: s["rejected"][reason] = s["rejected"].get(reason, 0) + 1 continue counts[group] = counts.get(group, 0) + 1 used.add((c["language"], text_key(text))) s["kept"] += 1 kept.append({**c, "text": text, "teacher_error_rate": round(rate, 4)}) if SMOKE and not kept: # Smoke only: keep the chain testable even if 2-clip samples all fail the filter. kept = [{**c, "text": teacher[c["id"]]["text"] or c["gold"], "teacher_error_rate": None} for c in candidates] for lang in LANGS: new = sum(1 for r in kept if r["language"] == lang and r["source"] != "fleurs") if not SMOKE and new < SIZES["keep"] // 4: raise RuntimeError(f"Only {new} teacher-labeled {lang} clips passed the filter; see filter_stats.json") write(DATA / "filter_stats.json", stats) write(DATA / "train.json", kept) for group, s in sorted(stats.items()): print(f"{group:<16} kept {s['kept']:>4}/{s['candidates']:<4} rejected {s['rejected']}", flush=True) print(f"Training set: {len(kept)} clips", flush=True) # ---------------------------------------------------------------- evaluation def evaluate(weights, rows, out): """Native .cact engine, automatic language detection (as Fleet-1). Cached by weights+rows hash.""" from cactus.agent.whistle import Whistle identity = {"weights_sha256": sha(weights), "rows_sha256": hashlib.sha256( json.dumps([r["id"] for r in rows]).encode()).hexdigest()} out = Path(out) if out.exists() and read(out).get("identity") == identity: return read(out)["scores"] model = Whistle(weights=str(weights)) predictions = [] for row in rows: result = model.transcribe(str(DATA / row["audio"])) e, n = error_units(row["gold"], result["text"], row["language"]) predictions.append({"id": row["id"], "language": row["language"], "reference": row["gold"], "prediction": result["text"], "detected_language": result.get("language"), "edits": int(e), "units": int(n)}) scores = {} for lang in LANGS: group = [p for p in predictions if p["language"] == lang] if not group: continue units = sum(p["units"] for p in group) if not units: raise RuntimeError(f"No reference units for {lang}") scores[lang] = {"metric": "CER" if lang == "ko" else "WER", "error_rate": sum(p["edits"] for p in group) / units, "edits": sum(p["edits"] for p in group), "reference_units": units, "clips": len(group), "language_accuracy": sum(p["detected_language"] == lang for p in group) / len(group)} out.parent.mkdir(parents=True, exist_ok=True) with open(out.with_suffix(".jsonl"), "w", encoding="utf-8") as f: for p in predictions: f.write(json.dumps(p, ensure_ascii=False) + "\n") write(out, {"identity": identity, "weights": str(weights), "scores": scores, "rss_bytes": rss()}) return scores def macro(scores, langs=tuple(LANGS)): return sum(scores[l]["error_rate"] for l in langs) / len(langs) def rows_for(kind): if kind in ("validation", "test"): return [r for lang in LANGS for r in read(DATA / f"fleurs-{kind}-{lang}.json")] return [r for lang in LANGS for r in read(DATA / f"ood-{lang}.json")] def published(which): if which == "whistle": return hub(WHISTLE["repo"], "whistle.cact", WHISTLE["revision"]) spec = SKY if which == "sky" else FLEET1 return hub(spec["repo"], "whistle.cact", spec["revision"]) # ---------------------------------------------------------------- training class Trainer: def __init__(self, adapter_path, total): import jax import jax.numpy as jnp import optax from cactus.model import whistle_finetune as wf from cactus.model.checkpoints import read_adapter from cactus.model.whistle import Whistle from cactus.model.quantize import configure_deploy, cq_ste_params, WEIGHT_BITS self.jax, self.jnp, self.wf = jax, jnp, wf self.adapter = read_adapter(adapter_path) if not set(LANGS).issubset(self.adapter["languages"]) or not self.adapter["pieces"]: raise ValueError("Start adapter lacks the Korean vocabulary or language tags") self.tokenizer = wf.WhistleTokenizer(self.adapter["pieces"], self.adapter["languages"]) self.base = hub(WHISTLE["repo"], WHISTLE["checkpoint"], WHISTLE["revision"]) params, self.cfg = wf.load_base(self.base) self.params = jax.tree.map(lambda x: jnp.asarray(x, dtype=jnp.float32), params) self.tuned = replace(self.cfg, max_tokens=TOKENS, text=self.tokenizer.text, languages=tuple(self.adapter["languages"])) self.weights = {"lora": {tuple(k.split("/")): {n: jnp.asarray(v) for n, v in pair.items()} for k, pair in self.adapter["lora"].items()}, "rows": jnp.asarray(self.adapter["rows"])} configure_deploy(act_bits=8, kv_bits=8) model = Whistle(self.tuned) schedule = optax.warmup_cosine_decay_schedule(0., LR, max(1, total // 20), max(2, total)) self.optimizer = optax.chain(optax.clip_by_global_norm(1.), optax.adamw(schedule)) self.state = self.optimizer.init(self.weights) scale, cfg, tok = self.adapter["scale"], self.cfg, self.tokenizer def loss(weights, wav, lengths, ids): merged = cq_ste_params(wf.tuned_params(self.params, cfg, tok, weights["lora"], scale, weights["rows"]), WEIGHT_BITS) logits = model.apply({"params": merged}, wav, lengths, ids[:, :-1], quant=True) targets = ids[:, 1:] mask = targets != cfg.pad ce = optax.softmax_cross_entropy_with_integer_labels(logits, targets) return (ce * mask).sum() / jnp.maximum(mask.sum(), 1.) @jax.jit def step(weights, state, wav, lengths, ids): value, grad = jax.value_and_grad(loss)(weights, wav, lengths, ids) updates, state = self.optimizer.update(grad, state, weights) return optax.apply_updates(weights, updates), state, value self.step_fn = step def arrays(self, row): import numpy as np import soundfile as sf wav, rate = sf.read(DATA / row["audio"], dtype="float32") if rate != 16000 or wav.ndim != 1 or len(wav) > SECONDS * 16000: raise ValueError(f"{row['id']}: expected <= {SECONDS} s of mono 16 kHz audio") padded = np.zeros((1, SECONDS * 16000), np.float32) padded[0, :len(wav)] = wav ids = [self.cfg.bos, self.tuned.tag(row["language"]), *self.tokenizer.encode(row["text"]), self.cfg.eos] if len(ids) > MAX_IDS: raise ValueError(f"{row['id']}: transcript exceeds context") tokens = np.full((1, MAX_IDS), self.cfg.pad, np.int32) tokens[0, :len(ids)] = ids return padded, np.array([len(wav)], np.int32), tokens def step(self, row): self.weights, self.state, value = self.step_fn(self.weights, self.state, *self.arrays(row)) value = float(value) if not math.isfinite(value): raise RuntimeError("Nonfinite training loss; stopping") return value def export(self, folder): import numpy as np from cactus.model.checkpoints import write_adapter, read_adapter folder.mkdir(parents=True, exist_ok=True) a = {**self.adapter, "base": f"{WHISTLE['repo']}@{WHISTLE['revision']}/{WHISTLE['checkpoint']}", "seed": SEED, "lora": {"/".join(k): {n: np.asarray(v) for n, v in pair.items()} for k, pair in self.weights["lora"].items()}, "rows": np.asarray(self.weights["rows"])} tmp = folder / "adapter.tmp.safetensors" write_adapter(str(tmp), a) back = read_adapter(str(tmp)) if back["pieces"] != a["pieces"] or back["languages"] != a["languages"] or not np.array_equal(back["rows"], a["rows"]): raise ValueError("Adapter round trip changed tokenizer or rows") tmp.replace(folder / "fleet2_lora.safetensors") params = self.wf.tuned_params(self.params, self.cfg, self.tokenizer, self.weights["lora"], self.adapter["scale"], a["rows"]) self.wf.build(params, self.cfg, self.tokenizer, str(folder / "whistle.tmp.cact")) (folder / "whistle.tmp.cact").replace(folder / "whistle.cact") def save(self, path, meta): import numpy as np leaves = self.jax.tree.leaves((self.weights, self.state)) tmp = path.with_suffix(".tmp") with tmp.open("wb") as f: np.savez(f, metadata=np.array(json.dumps(meta)), **{f"a{i}": np.asarray(x) for i, x in enumerate(leaves)}) tmp.replace(path) def restore(self, path): import numpy as np leaves, tree = self.jax.tree.flatten((self.weights, self.state)) with np.load(path, allow_pickle=False) as data: values = [self.jnp.asarray(data[f"a{i}"]) for i in range(len(leaves))] if any(a.shape != b.shape for a, b in zip(values, leaves)): raise ValueError("Resume checkpoint does not match this model") self.weights, self.state = self.jax.tree.unflatten(tree, values) return json.loads(str(data["metadata"])) def start_adapter(which): spec = SKY if which == "sky" else FLEET1 return hub(spec["repo"], spec["adapter"], spec["revision"]) def stage0(): from cactus.model.checkpoints import read_adapter sky, fleet = read_adapter(start_adapter("sky")), read_adapter(start_adapter("fleet1")) if sky["pieces"] != fleet["pieces"] or sky["languages"] != fleet["languages"]: raise RuntimeError("Sky-Kim and Fleet-1 adapters use different vocabularies") val = rows_for("validation") scores = {name: evaluate(published(name), val, RUN / f"eval/validation-{name}.json") for name in ("sky", "fleet1")} scores["whistle"] = evaluate(published("whistle"), [r for r in val if r["language"] in WHISTLE_LANGS], RUN / "eval/validation-whistle.json") m = {name: macro(scores[name]) for name in ("sky", "fleet1")} chosen = "fleet1" if m["fleet1"] < m["sky"] else "sky" # tie keeps the source for name, s in scores.items(): print(name, {l: f"{v['error_rate']:.2%}" for l, v in s.items()}, flush=True) print(f"Macro validation error: Sky-Kim {m['sky']:.4f}, Fleet-1 {m['fleet1']:.4f} -> start from {chosen}", flush=True) # Audit: rebuild the chosen start model from base + adapter; predictions must match the published file. trainer = Trainer(start_adapter(chosen), 1) trainer.export(RUN / "checkpoints/start-rebuilt") rebuilt = evaluate(RUN / "checkpoints/start-rebuilt/whistle.cact", val, RUN / "eval/validation-start-rebuilt.json") a = [json.loads(l) for l in (RUN / f"eval/validation-{chosen}.jsonl").read_text(encoding="utf-8").splitlines()] b = [json.loads(l) for l in (RUN / "eval/validation-start-rebuilt.jsonl").read_text(encoding="utf-8").splitlines()] mismatches = [x["id"] for x, y in zip(a, b) if (x["prediction"], x["detected_language"]) != (y["prediction"], y["detected_language"])] audit = {"chosen": chosen, "byte_identical": sha(published(chosen)) == sha(RUN / "checkpoints/start-rebuilt/whistle.cact"), "prediction_mismatches": mismatches, "passed": not mismatches and len(a) == len(b)} write(RUN / "export-audit.json", audit) if not audit["passed"]: raise RuntimeError("Rebuilt start model does not reproduce the published predictions; see export-audit.json") immutable(RUN / "init.json", {"chosen": chosen, "macro_validation": m, "rule": "lower 8-language mean validation error; tie keeps Sky-Kim", "adapter": SKY if chosen == "sky" else FLEET1, "rebuilt_macro": macro(rebuilt)}) print("Stage 0 audit passed.", flush=True) def train(): import numpy as np init = read(RUN / "init.json") training = read(DATA / "train.json") total = EPOCHS * len(training) points = sorted({max(1, round(k * total / 4)) for k in (1, 2, 3, 4)}) immutable(RUN / "train-settings.json", {"init": init["chosen"], "clips": len(training), "total_steps": total, "eval_points": points, "train_sha256": sha(DATA / "train.json")}) if (RUN / "trained.json").exists(): print("Training already complete.", flush=True) return trainer = Trainer(start_adapter(init["chosen"]), total) checkpoint = RUN / "resume.npz" meta = {"step": 0, "evals": []} if checkpoint.exists(): meta = trainer.restore(checkpoint) print(f"Resuming at step {meta['step']}/{total}", flush=True) val = rows_for("validation") print(f"CPU training: {len(training)} clips x {EPOCHS} epochs = {total} steps; eval at {points}. " "First JAX compile takes a few minutes.", flush=True) start, first = time.monotonic(), meta["step"] while meta["step"] < total: epoch, pos = divmod(meta["step"], len(training)) order = np.random.default_rng(SEED + epoch).permutation(len(training)) loss = trainer.step(training[int(order[pos])]) meta["step"] += 1 if meta["step"] % 25 == 0 or meta["step"] == first + 1: done = meta["step"] - first left = (time.monotonic() - start) / done * (total - meta["step"]) / 3600 print(f"step {meta['step']}/{total} loss {loss:.4f} ~{left:.1f} h left rss {rss() / 2**30:.1f} GB", flush=True) if meta["step"] in points: folder = RUN / f"checkpoints/step-{meta['step']}" trainer.export(folder) scores = evaluate(folder / "whistle.cact", val, RUN / f"eval/validation-step-{meta['step']}.json") meta["evals"].append({"step": meta["step"], "macro": macro(scores), "scores": {l: s["error_rate"] for l, s in scores.items()}}) print(f"eval step {meta['step']}: macro validation error {macro(scores):.4f}", flush=True) trainer.save(checkpoint, meta) elif meta["step"] % 200 == 0: trainer.save(checkpoint, meta) trainer.save(checkpoint, meta) write(RUN / "trained.json", {**meta, "seconds_last_invocation": time.monotonic() - start}) def select(): init, trained = read(RUN / "init.json"), read(RUN / "trained.json") baseline = init["macro_validation"][init["chosen"]] best = min(trained["evals"], key=lambda e: (e["macro"], e["step"])) if best["macro"] < baseline: selection = {"step": best["step"], "macro_validation": best["macro"], "start_macro_validation": baseline, "reason": "Lowest macro validation error, better than the start model"} else: # The user chose to publish no matter what (2026-11-15), so the best checkpoint always goes forward. selection = {"step": best["step"], "macro_validation": best["macro"], "start_macro_validation": baseline, "reason": "No checkpoint beat the start model on validation; best validation checkpoint used anyway"} immutable(RUN / "selection.json", selection) print(selection["reason"], f"(best {best['macro']:.4f} vs start {baseline:.4f})", flush=True) def test(): selection = read(RUN / "selection.json") report = {"selection": selection, "init": read(RUN / "init.json"), "filter": read(DATA / "filter_stats.json")} if selection["step"] is None: report["gate"] = {"passed": False, "reason": selection["reason"]} write(RUN / "report.json", report) print("No improvement on validation; nothing to test or publish.", flush=True) return weights = RUN / f"checkpoints/step-{selection['step']}/whistle.cact" models = {"Fleet-2": weights, "Fleet-1": published("fleet1"), "Sky-Kim": published("sky"), "Whistle": published("whistle")} results = {} for split in ("test", "ood"): rows = rows_for(split) results[split] = {} for name, path in models.items(): use = [r for r in rows if name != "Whistle" or r["language"] in WHISTLE_LANGS] results[split][name] = evaluate(path, use, RUN / f"eval/{split}-{name}.json") if not SMOKE: historical = read(VENDOR / "evaluation.json") for name, key in (("Sky-Kim", "source"), ("Fleet-1", "Fleet-1")): for lang in LANGS: if abs(results["test"][name][lang]["error_rate"] - historical[key][lang]["error_rate"]) > 1e-10: raise RuntimeError(f"{name} {lang} test score does not reproduce Fleet-1's published table") f2, f1, sky = results["test"]["Fleet-2"], results["test"]["Fleet-1"], results["test"]["Sky-Kim"] wins = [l for l in LANGS if f2[l]["error_rate"] < f1[l]["error_rate"]] ko_ok = f2["ko"]["error_rate"] <= sky["ko"]["error_rate"] passed = len(wins) >= 5 and ko_ok report.update(results=results, gate={"passed": passed, "beats_fleet1_in": wins, "korean_not_worse_than_sky": ko_ok, "rule": ">=5 of 8 languages better than Fleet-1 and Korean CER <= Sky-Kim, on Fleet-1's FLEURS test clips"}) write(RUN / "report.json", report) for lang in LANGS: row = " ".join(f"{n} {results['test'][n][lang]['error_rate']:.2%}" for n in models if lang in results["test"][n]) print(f"{lang} ({f2[lang]['metric']}): {row}", flush=True) print(f"Gate {'PASSED' if passed else 'FAILED'}: better than Fleet-1 in {len(wins)}/8 ({', '.join(wins)}); " f"Korean not worse than Sky-Kim: {ko_ok}", flush=True) # ---------------------------------------------------------------- publish def pct(x): return f"{x:.2%}" def card(report, init_repo): res, f = report["results"], report["filter"] t = res["test"] lines = [] for lang in LANGS: w = pct(t["Whistle"][lang]["error_rate"]) if lang in t["Whistle"] else "n/a" lines.append(f"| {lang} | {t['Fleet-2'][lang]['metric']} | {w} | {pct(t['Sky-Kim'][lang]['error_rate'])} | " f"{pct(t['Fleet-1'][lang]['error_rate'])} | **{pct(t['Fleet-2'][lang]['error_rate'])}** |") o = res["ood"] ood = [] for lang in LANGS: w = pct(o["Whistle"][lang]["error_rate"]) if lang in o["Whistle"] else "n/a" src = "Zeroth-Korean" if lang == "ko" else "VoxPopuli" ood.append(f"| {lang} | {src} | {o['Fleet-2'][lang]['clips']} | {w} | {pct(o['Sky-Kim'][lang]['error_rate'])} | " f"{pct(o['Fleet-1'][lang]['error_rate'])} | **{pct(o['Fleet-2'][lang]['error_rate'])}** |") kept = sum(s["kept"] for s in f.values()) cand = sum(s["candidates"] for s in f.values()) new = sum(s["kept"] for g, s in f.items() if not g.startswith("fleurs")) anchor = kept - new gate = report["gate"] sel = report["selection"] trained = read(RUN / "train-settings.json") wins = gate["beats_fleet1_in"] status = (f"Fleet-2 beats Fleet-1 in {len(wins)} of 8 languages ({', '.join(wins) or 'none'}). " f"Korean CER is no worse than Sky-Kim: {'yes' if gate['korean_not_worse_than_sky'] else 'no'}.") if not gate["passed"]: status += " **Improvement over Fleet-1 was NOT demonstrated. Check each language; some may regress.**" if sel["macro_validation"] >= sel["start_macro_validation"]: status += " No training checkpoint beat the start model on validation; the best one was published anyway." return f"""--- license: apache-2.0 base_model: {init_repo} language: [en, de, fr, es, it, nl, pl, ko] pipeline_tag: automatic-speech-recognition datasets: [facebook/voxpopuli, Bingsu/zeroth-korean, google/fleurs] tags: [whistle, cactus, lora, speech-to-text, distillation, pseudo-labels] --- # Fleet-2 A 30 MB on-device speech-to-text model (Cactus Whistle architecture) for English, German, French, Spanish, Italian, Dutch, Polish and Korean, trained on transcripts written by [openai/whisper-large-v3-turbo](https://huggingface.co/openai/whisper-large-v3-turbo). **Collection:** [Speech recognition](https://huggingface.co/collections/{COLLECTION}) ## What changed from Fleet-1 - **Teacher:** Whisper-large-v3-turbo transcribed new audio, and Fleet-2 learned from those transcripts. This is sequence-level distillation, as in Distil-Whisper. Whisper and Whistle use different tokenizers, so no logits were matched. - **New audio:** VoxPopuli (European Parliament speech) for the seven European languages and Zeroth-Korean for Korean. Fleet-1 used FLEURS only. - **Start point:** chosen by validation score between Sky-Kim's Korean Whistle and Fleet-1. This run started from `{init_repo}`. ## Evaluation Each language has 100 short FLEURS test clips. These are the same clips, filtered the same way, as Fleet-1's table. All models ran in the same native Cactus engine with automatic language detection. Lower is better. Whistle has no Korean. | Language | Metric | Whistle | Sky-Kim | Fleet-1 | Fleet-2 | |---|---|---:|---:|---:|---:| {chr(10).join(lines)} {status}
Out-of-domain clips (VoxPopuli / Zeroth-Korean test splits) | Language | Source | Clips | Whistle | Sky-Kim | Fleet-1 | Fleet-2 | |---|---|---:|---:|---:|---:|---:| {chr(10).join(ood)} Fleet-2 trained on other clips from these corpora, so this table shows in-corpus generalisation, not a fully independent test.
These results come from short clips of 12 s or less. They are not full benchmark scores and not significance tests. Accuracy on long, noisy or conversational audio was not measured. Upstream models may have seen FLEURS training data before. ## Training - **Data:** {kept} clips kept out of {cand} that the teacher labeled: {new} new VoxPopuli/Zeroth clips and {anchor} of Fleet-1's FLEURS training clips. All clips are 12 s or less. - **Teacher decoding:** language forced, temperature 0, no conditioning on previous text. Runtime: the MLX conversion `mlx-community/whisper-large-v3-turbo`. - **Filter:** a clip was kept only if all of these held: - compression ratio ≤ {THRESH['compression_ratio']} - average log-probability ≥ {THRESH['avg_logprob']} - no-speech probability ≤ {THRESH['no_speech_prob']} - teacher text within {THRESH['error_rate']:.0%} WER of the human transcript (CER for Korean) The training target is the teacher's text. - **Excluded from training:** validation and test clips, and any transcript that matches an evaluation transcript. - **Optimisation:** the start adapter (LoRA plus Korean vocabulary rows) continued with AdamW at {LR}, warmup/cosine schedule, batch 1, {EPOCHS} epochs ({trained['total_steps']} steps), clip 1.0, W4/A8 quantisation-aware training on CPU (Apple M4). - **Checkpoint selection:** step {sel['step']}, chosen by macro validation error on Fleet-1's FLEURS validation clips ({sel['macro_validation']:.4f}, against {sel['start_macro_validation']:.4f} for the start model). Test clips were not used for selection. - **Release:** published whatever the scores. The comparison above is reported as measured, including any regressions. `training_manifest.json` records every clip, teacher transcript and pinned revision. `evaluation.json` has every score. `fleet2.py`, `teacher.py` and `run.sh` reproduce the run. ## Usage ```python import cactus from huggingface_hub import hf_hub_download path = hf_hub_download("{DEST}", "whistle.cact") print(cactus.transcribe("clip.wav", weights=path)["text"]) ``` Use 16 kHz mono WAV and `cactus-compute` at revision `{CACTUS_CODE}`. This is a Cactus model, not a Transformers checkpoint. ## License and attribution - **Model:** Apache License 2.0. The original Whistle LICENSE is kept verbatim. Architecture and pretrained weights: Cactus Compute's Whistle (Mroz et al., 2026). Korean vocabulary and adapter lineage: [Sky-Kim/whistle-finetuned-ko](https://huggingface.co/Sky-Kim/whistle-finetuned-ko) and [Lucasgitguy-co/Fleet-1](https://huggingface.co/Lucasgitguy-co/Fleet-1). See `UPSTREAM_MODEL_CARD.md`. - **Teacher:** [openai/whisper-large-v3-turbo](https://huggingface.co/openai/whisper-large-v3-turbo), MIT License. Only its transcripts were used; none of its weights are included. - **Data:** - [VoxPopuli](https://huggingface.co/datasets/facebook/voxpopuli): CC0 - [Zeroth-Korean](https://www.openslr.org/40/): CC BY 4.0 - [FLEURS](https://huggingface.co/datasets/google/fleurs) (Conneau et al., 2022): CC BY 4.0 No audio is included in this repository. See `NOTICE`. """ NOTICE = """Fleet-2 Copyright 2026 Lucasgitguy-co. Licensed under the Apache License, Version 2.0 (see LICENSE). This model is a derivative of: - Whistle by Cactus Compute (Apache-2.0): architecture, base weights, runtime. - Sky-Kim/whistle-finetuned-ko (Apache-2.0): Korean vocabulary, language tags, LoRA adapter. - Lucasgitguy-co/Fleet-1 (Apache-2.0): adapter lineage. Training targets were produced by openai/whisper-large-v3-turbo (MIT License, Copyright (c) 2022 OpenAI), run through the mlx-community/whisper-large-v3-turbo conversion. No Whisper weights are distributed here. Training and evaluation audio (not redistributed): - VoxPopuli, Wang et al. (2021), CC0 1.0. - Zeroth-Korean, Atlas Guide Inc. and Gridspace Inc., OpenSLR 40, CC BY 4.0. Transcripts were replaced by teacher output. - FLEURS, Conneau et al. (2022), CC BY 4.0. Audio converted to 16 kHz mono PCM and subsetted. """ def assemble(report): init = read(RUN / "init.json") spec = SKY if init["chosen"] == "sky" else FLEET1 step = report["selection"]["step"] src = RUN / f"checkpoints/step-{step}" if PUB.exists(): shutil.rmtree(PUB) PUB.mkdir(parents=True) shutil.copyfile(src / "whistle.cact", PUB / "whistle.cact") shutil.copyfile(src / "fleet2_lora.safetensors", PUB / "fleet2_lora.safetensors") shutil.copyfile(hub(SKY["repo"], "LICENSE", SKY["revision"]), PUB / "LICENSE") shutil.copyfile(hub(SKY["repo"], "README.md", SKY["revision"]), PUB / "UPSTREAM_MODEL_CARD.md") for name in ("fleet2.py", "teacher.py", "run.sh", "teacher_pin.json", "requirements-student.lock", "requirements-teacher.lock"): shutil.copyfile(HERE / name, PUB / name) (PUB / "NOTICE").write_text(NOTICE, encoding="utf-8") cfg = read(hub(spec["repo"], "config.json", spec["revision"])) cfg.update(name_or_path=DEST, base_model=spec["repo"]) cfg["finetune"] = {"method": "continued LoRA on teacher pseudo-labels", "teacher": "openai/whisper-large-v3-turbo", "start": f"{spec['repo']}@{spec['revision']}", "steps": step, "adapter": "fleet2_lora.safetensors"} write(PUB / "config.json", cfg) write(PUB / "evaluation.json", report) train_rows = [{k: r.get(k) for k in ("id", "source", "language", "text", "gold", "teacher_error_rate", "audio_id", "fleurs_id", "shard", "row_index", "speaker", "seconds", "sha256")} for r in read(DATA / "train.json")] write(PUB / "training_manifest.json", { "settings": read(RUN / "settings.json"), "dataset_revisions": read(DATA / "revisions.json"), "init": init, "train_settings": read(RUN / "train-settings.json"), "selection": report["selection"], "eval_history": read(RUN / "trained.json")["evals"], "filter": read(DATA / "filter_stats.json"), "train": train_rows, "validation_ids": [r["id"] for r in rows_for("validation")], "test_ids": [r["id"] for r in rows_for("test")], "ood_ids": [r["id"] for r in rows_for("ood")]}) (PUB / "README.md").write_text(card(report, spec["repo"]), encoding="utf-8") return spec def publish(): from huggingface_hub import HfApi report = read(RUN / "report.json") if not report.get("results"): raise RuntimeError("report.json has no test results; run the test stage first") if not report["gate"]["passed"]: print("Comparison check failed (Fleet-2 does not clearly beat Fleet-1); publishing anyway with the results on the card.", flush=True) assemble(report) api = HfApi() who = api.whoami() if who["name"] != OWNER: raise RuntimeError(f"Signed in to Hugging Face as {who['name']}, not {OWNER}; run `hf auth login` first") marker = RUN / "destination.json" exists = api.repo_exists(DEST) if exists and not marker.exists(): raise RuntimeError(f"{DEST} already exists and was not created by this run; refusing to overwrite") if SMOKE: print(f"Smoke dry run OK: signed in as {who['name']}, {DEST} exists={exists}, publish folder assembled " f"({len(list(PUB.iterdir()))} files). Nothing uploaded.", flush=True) return write(marker, {"repo": DEST, "run": str(RUN)}) api.create_repo(DEST, private=False, exist_ok=True) commit = api.upload_folder(repo_id=DEST, folder_path=str(PUB), commit_message="Fleet-2: Whistle student of Whisper-large-v3-turbo") try: api.add_collection_item(COLLECTION, item_id=DEST, item_type="model", exists_ok=True) except Exception as error: # the model is published either way print(f"Could not add to the collection ({error}); add it by hand.", flush=True) # Verify the published file works: download from the Hub and transcribe one test clip. from huggingface_hub import hf_hub_download from cactus.agent.whistle import Whistle path = hf_hub_download(DEST, "whistle.cact", force_download=True) if sha(path) != sha(PUB / "whistle.cact"): raise RuntimeError("Downloaded whistle.cact differs from the local file") row = rows_for("test")[0] text = Whistle(weights=path).transcribe(str(DATA / row["audio"]))["text"] write(RUN / "published.json", {"repo": DEST, "commit": getattr(commit, "oid", str(commit)), "check_clip": row["id"], "check_text": text}) print(f"Published https://huggingface.co/{DEST} (check transcription: {text!r})", flush=True) def main(): p = argparse.ArgumentParser(description=__doc__) p.add_argument("stage", choices=["prepare", "filter", "stage0", "train", "select", "test", "publish"]) a = p.parse_args() RUN.mkdir(parents=True, exist_ok=True) {"prepare": prepare, "filter": filter_stage, "stage0": stage0, "train": train, "select": select, "test": test, "publish": publish}[a.stage]() if __name__ == "__main__": main()