# /// script # requires-python = ">=3.11,<3.14" # dependencies = [ # "torch>=2.6,<2.8", # "datasets>=3.0", # "numpy<3", # "onnx>=1.17", # "onnxruntime>=1.20", # "huggingface_hub>=0.30", # ] # /// """Train a small model that spots correctly spelled wrong words, like "don" typed for "do".""" import argparse import hashlib import json import os import re import time import urllib.request from collections import Counter, defaultdict from pathlib import Path import numpy as np import onnxruntime as ort import torch import torch.nn as nn import torch.nn.functional as F from datasets import load_dataset from huggingface_hub import HfApi from onnxruntime.quantization import QuantType, quantize_dynamic LEFT_WINDOW = 16 RIGHT_WINDOW = 4 MAX_NEIGHBORS = 8 CANDIDATES = MAX_NEIGHBORS + 1 MIN_TYPED_LENGTH = 3 PAD, UNK, NUMBER, START, NAME = 0, 1, 2, 3, 4 SPECIAL_TOKENS = ["", "", "", "", ""] REAL_MISTAKE_SHARE = 0.6 HOMOPHONE_WEIGHT = 20 MIN_REAL_MISTAKE_COUNT = 2 HOMOPHONES = [ ("their", "there", "they're"), ("your", "you're"), ("its", "it's"), ("too", "two"), ("then", "than"), ("were", "where", "we're"), ("whose", "who's"), ("hear", "here"), ("buy", "bye"), ("weather", "whether"), ("peace", "piece"), ("affect", "effect"), ("lose", "loose"), ("accept", "except"), ("advice", "advise"), ("quiet", "quite"), ("through", "though", "thorough"), ("passed", "past"), ("principal", "principle"), ("complement", "compliment"), ("brake", "break"), ("waist", "waste"), ("right", "write"), ("sight", "site", "cite"), ("wear", "where"), ("led", "lead"), ("meet", "meat"), ("week", "weak"), ("wood", "would"), ("hole", "whole"), ("male", "mail"), ("sale", "sell"), ("role", "roll"), ("aloud", "allowed"), ("board", "bored"), ("fair", "fare"), ] TYPO_PAIRS_DATASET = "chirunder/github_typo_corrections" # Sound-alikes and classic mix-ups. Every member can be typed for every other member. EXTENDED_GROUPS = HOMOPHONES + [ ("to", "too", "two"), ("of", "off"), ("no", "know", "now"), ("by", "buy", "bye"), ("for", "four"), ("one", "won"), ("new", "knew"), ("wait", "weight"), ("way", "weigh"), ("plane", "plain"), ("made", "maid"), ("pair", "pear"), ("tale", "tail"), ("flour", "flower"), ("threw", "through"), ("son", "sun"), ("bought", "brought"), ("wear", "were"), ("bare", "bear"), ("dear", "deer"), ("hour", "our"), ("knight", "night"), ("knot", "not"), ("sea", "see"), ("steal", "steel"), ("stair", "stare"), ("tide", "tied"), ("which", "witch"), ("course", "coarse"), ("sent", "scent", "cent"), ("die", "dye"), ("flew", "flu"), ("great", "grate"), ("heal", "heel"), ("heard", "herd"), ("lessen", "lesson"), ("main", "mane"), ("medal", "meddle"), ("missed", "mist"), ("morning", "mourning"), ("pain", "pane"), ("pause", "paws"), ("peak", "peek"), ("pole", "poll"), ("poor", "pour", "pore"), ("pray", "prey"), ("rain", "reign", "rein"), ("read", "red"), ("real", "reel"), ("road", "rode"), ("root", "route"), ("rose", "rows"), ("scene", "seen"), ("seam", "seem"), ("sole", "soul"), ("some", "sum"), ("stationary", "stationery"), ("throne", "thrown"), ("vain", "vein"), ("waive", "wave"), ("ate", "eight"), ("blew", "blue"), ("cell", "sell"), ("cereal", "serial"), ("council", "counsel"), ("desert", "dessert"), ("dual", "duel"), ("fourth", "forth"), ("guessed", "guest"), ("hall", "haul"), ("higher", "hire"), ("loan", "lone"), ("hours", "ours"), ("presence", "presents"), ("sail", "sale"), ("stake", "steak"), ("breath", "breathe"), ("chose", "choose"), ("clothes", "cloths"), ("dose", "does"), ("later", "latter"), ("personal", "personnel"), ("precede", "proceed"), ("quite", "quit"), ("woman", "women"), ("form", "from"), ("lose", "loss"), ("whit", "with"), ("grown", "groan"), ("idle", "idol"), ("sweet", "suite"), ("ensure", "insure"), ("emigrate", "immigrate"), ("we", "wee"), ("so", "sew"), ] # Mix-ups that only make sense one way ("could of" for "could have"). ONE_WAY = [("of", "have"), ("then", "than"), ("loose", "lose"), ("defiantly", "definitely")] GROUP_SETS = {"shipped": HOMOPHONES, "extended": EXTENDED_GROUPS} SLIP_WORDS = ["fro", "nit", "grate", "manger", "bye", "whit", "wee", "thee", "lest", "sew"] WORD_LIST_URL = "https://raw.githubusercontent.com/wolfgarbe/SymSpell/master/SymSpell.FrequencyDictionary/en-80k.txt" TOKEN = re.compile(r"[A-Za-z]+(?:'[A-Za-z]+)?|\d+(?:[.,]\d+)*|[^\s\w]") SENTENCE_OPENERS = {".", "!", "?", ":", '"'} WORD = re.compile(r"^[a-z]+(?:'[a-z]+)?$") DIGITS = re.compile(r"^\d") def tokenize(text: str) -> list[str]: """Lowercases words, except capitalized words mid-sentence, which become a single name token.""" tokens = [] for raw in TOKEN.findall(text.replace("’", "'")): at_sentence_start = not tokens or tokens[-1] in SENTENCE_OPENERS is_pronoun = raw == "I" or raw.startswith("I'") tokens.append("" if raw[0].isupper() and not at_sentence_start and not is_pronoun else raw.lower()) return tokens def fineweb_documents(): stream = load_dataset("HuggingFaceFW/fineweb-edu", split="train", streaming=True).shuffle(seed=7, buffer_size=20000) for row in stream: yield row["text"] def smoltalk_documents(): stream = load_dataset("HuggingFaceTB/smoltalk", "all", split="train", streaming=True).shuffle(seed=7, buffer_size=20000) for row in stream: yield "\n".join(message["content"] for message in row["messages"] if message["role"] != "system") def open_assistant_documents(): for row in load_dataset("OpenAssistant/oasst1", split="train"): if row["lang"] == "en": yield row["text"] def load_corpus(token_budgets: dict[str, int]) -> tuple[list[np.ndarray], dict[str, int]]: sources = {"fineweb": fineweb_documents, "smoltalk": smoltalk_documents, "oasst": open_assistant_documents} raw_ids = {"": 0} documents = [] for name, budget in token_budgets.items(): started, used = time.time(), 0 for text in sources[name](): tokens = tokenize(text) if len(tokens) < 8: continue documents.append(np.array([0] + [raw_ids.setdefault(token, len(raw_ids)) for token in tokens], dtype=np.int32)) used += len(tokens) if used >= budget: break print(f"{name}: {used:,} tokens in {time.time() - started:.0f}s", flush=True) return documents, raw_ids CACHE_REPO = "cfahlgren1/tinyfix-lab" def load_corpus_cached(token_budgets: dict[str, int], api: HfApi) -> tuple[list[np.ndarray], dict[str, int]]: """Tokenizing 120M tokens takes a while, so the first run saves the result to the private lab repo.""" folder = f"data/corpus-{token_budgets['fineweb']}-{token_budgets['smoltalk']}" try: paths = {name: api.hf_hub_download(CACHE_REPO, f"{folder}/{name}") for name in ("tokens.npy", "lengths.npy", "raw_tokens.json")} tokens, lengths = np.load(paths["tokens.npy"]), np.load(paths["lengths.npy"]) raw_tokens = json.loads(Path(paths["raw_tokens.json"]).read_text()) print(f"loaded cached corpus {folder}: {len(tokens):,} tokens", flush=True) return np.split(tokens, np.cumsum(lengths)[:-1]), {token: index for index, token in enumerate(raw_tokens)} except Exception as error: print(f"no cached corpus ({type(error).__name__}); building it", flush=True) documents, raw_ids = load_corpus(token_budgets) cache = Path("/tmp/corpus-cache") cache.mkdir(exist_ok=True) np.save(cache / "tokens.npy", np.concatenate(documents)) np.save(cache / "lengths.npy", np.array([len(document) for document in documents], dtype=np.int64)) (cache / "raw_tokens.json").write_text(json.dumps(list(raw_ids))) api.create_repo(CACHE_REPO, repo_type="model", private=True, exist_ok=True) api.upload_folder(folder_path=str(cache), repo_id=CACHE_REPO, path_in_repo=folder, repo_type="model") print(f"cached corpus to {CACHE_REPO}/{folder}", flush=True) return documents, raw_ids def slip_words(ranked: list[str], counts_by_token: dict[str, int], start: int, how_many: int, min_count: int) -> list[str]: """Words just outside the vocabulary that are one edit from a common word, so a slip that lands on them can be judged.""" common = ranked[:3000] buckets = defaultdict(list) for word in common: if len(word) < 3: continue for position in range(len(word)): buckets[word[:position] + word[position + 1 :]].append(word) buckets[word].append(word) found = [] for word in ranked[start:]: if len(found) >= how_many or counts_by_token[word] < min_count: break if len(word) < 3 or "'" in word: continue keys = {word} | {word[:position] + word[position + 1 :] for position in range(len(word))} if any(within_one_edit(word, other) for key in keys for other in buckets.get(key, [])): found.append(word) return found def build_vocabulary(documents: list[np.ndarray], raw_ids: dict[str, int], size: int, forced: list[str], slips: int) -> tuple[list[str], np.ndarray]: with urllib.request.urlopen(WORD_LIST_URL) as response: dictionary = {line.split()[0] for line in response.read().decode().splitlines() if line.strip()} counts = np.bincount(np.concatenate(documents), minlength=len(raw_ids)) allowed = [(token, identifier) for token, identifier in raw_ids.items() if (WORD.match(token) and token in dictionary) or (len(token) == 1 and not token.isalnum())] allowed = [pair for pair in allowed if pair[0] != ""] allowed.sort(key=lambda pair: (-counts[pair[1]], pair[0])) ranked = [token for token, _ in allowed] room = size - len(SPECIAL_TOKENS) extra = [word for word in forced if word in raw_ids and word not in ranked[:room]] if slips: counts_by_token = {token: int(counts[identifier]) for token, identifier in allowed} extra += [word for word in slip_words(ranked, counts_by_token, room, slips, 20) if word not in extra] missing = [word for word in forced if word not in raw_ids] print(f"vocabulary: forcing {len(extra)} extra words, e.g. {extra[:40]}; not in corpus: {missing}", flush=True) vocabulary = SPECIAL_TOKENS + ranked[: room - len(extra)] + extra token_to_id = {token: index for index, token in enumerate(vocabulary)} lookup = np.full(len(raw_ids), UNK, dtype=np.int32) for token, identifier in raw_ids.items(): if token == "": lookup[identifier] = START elif token == "": lookup[identifier] = NAME elif DIGITS.match(token): lookup[identifier] = NUMBER elif token in token_to_id: lookup[identifier] = token_to_id[token] return vocabulary, lookup def is_target_word(token: str) -> bool: return len(token) >= 2 and token not in SPECIAL_TOKENS and WORD.match(token) is not None def within_one_edit(first: str, second: str) -> bool: if first == second or abs(len(first) - len(second)) > 1: return False if len(first) == len(second): differences = [index for index, (a, b) in enumerate(zip(first, second)) if a != b] if len(differences) == 1: return True return len(differences) == 2 and differences[1] == differences[0] + 1 and first[differences[0]] == second[differences[1]] and first[differences[1]] == second[differences[0]] shorter, longer = sorted((first, second), key=len) return any(longer[:index] + longer[index + 1 :] == shorter for index in range(len(longer))) def strip_punctuation(token: str) -> str: return token.strip("\"'()[]{}<>.,;:!?*_`#-=/\\") def real_mistakes_for_training(vocabulary: set[str]) -> Counter: """Counts real typed-for-expected word swaps, using only the half of the dev split that eval doesn't tune on.""" counts = Counter() for row in load_dataset(TYPO_PAIRS_DATASET, split="train"): typed_tokens, fixed_tokens = (row["text"] or "").split(), (row["correction"] or "").split() if len(typed_tokens) != len(fixed_tokens): continue different = [index for index, (first, second) in enumerate(zip(typed_tokens, fixed_tokens)) if first != second] if len(different) != 1: continue typed, expected = strip_punctuation(typed_tokens[different[0]]), strip_punctuation(fixed_tokens[different[0]]) digest = int(hashlib.sha1(f"{typed}|{expected}".lower().encode()).hexdigest(), 16) if digest % 2 != 0 or (digest >> 1) % 2 != 0: continue typed, expected = typed.lower(), expected.lower() if typed != expected and len(typed) >= MIN_TYPED_LENGTH and typed in vocabulary and expected in vocabulary and is_target_word(typed) and is_target_word(expected): counts[(typed, expected)] += 1 return Counter({pair: count for pair, count in counts.items() if count >= MIN_REAL_MISTAKE_COUNT}) PAIRS_FILE: list[tuple[str, str]] = [] def load_pairs_file(api: HfApi, repo: str, path: str) -> list[tuple[str, str]]: """Reads a mix-up list ("typed meant1 meant2 ...") from the lab repo, so training and the engine use the same pairs.""" text = Path(api.hf_hub_download(repo, path)).read_text() pairs = [(parts[0], meant) for parts in (line.split() for line in text.splitlines()) if parts for meant in parts[1:]] return list(dict.fromkeys(pairs)) def group_pairs(groups: str) -> list[tuple[str, str]]: if PAIRS_FILE: return PAIRS_FILE pairs = [(typed, expected) for group in GROUP_SETS[groups] for typed in group for expected in group if typed != expected] if groups != "shipped": pairs += ONE_WAY return list(dict.fromkeys(pairs)) def build_confusions(vocabulary: list[str], use_real_mistakes: bool, groups: str, min_typed: int) -> dict[str, list[tuple[str, int]]]: """Maps a typed word to words people really meant by it, most likely first.""" known = set(vocabulary) counts = real_mistakes_for_training(known) if use_real_mistakes else Counter() print(f"real mistake pairs={len(counts):,}", flush=True) for typed, expected in group_pairs(groups): if typed in known and expected in known and len(typed) >= min_typed: counts[(typed, expected)] += HOMOPHONE_WEIGHT confusions = defaultdict(list) for (typed, expected), count in sorted(counts.items(), key=lambda item: (-item[1], item[0])): confusions[typed].append((expected, count)) return {typed: options[:MAX_NEIGHBORS] for typed, options in confusions.items()} def build_real_mistake_tables(vocabulary: list[str], confusions: dict[str, list[tuple[str, int]]]) -> tuple[np.ndarray, np.ndarray]: """For each intended word, lists the words typed by mistake and how often, for sampling.""" token_to_id = {token: index for index, token in enumerate(vocabulary)} mistakes = defaultdict(list) for typed, options in confusions.items(): for expected, count in options: mistakes[token_to_id[expected]].append((token_to_id[typed], count)) typed_table = np.zeros((len(vocabulary), MAX_NEIGHBORS), dtype=np.int32) weights = np.zeros((len(vocabulary), MAX_NEIGHBORS), dtype=np.float32) for expected, options in mistakes.items(): options = sorted(options, key=lambda option: -option[1])[:MAX_NEIGHBORS] typed_table[expected, : len(options)] = [typed for typed, _ in options] weights[expected, : len(options)] = [count**0.5 for _, count in options] return typed_table, weights def build_neighbors(vocabulary: list[str], confusions: dict[str, list[tuple[str, int]]]) -> np.ndarray: """Lists known mix-ups first, then the most common words one edit away, the same order the browser uses.""" token_to_id = {token: index for index, token in enumerate(vocabulary)} buckets = defaultdict(set) for index, token in enumerate(vocabulary): if not is_target_word(token): continue buckets[token].add(index) for position in range(len(token)): buckets[token[:position] + token[position + 1 :]].add(index) table = np.zeros((len(vocabulary), MAX_NEIGHBORS), dtype=np.int32) for index, token in enumerate(vocabulary): if not is_target_word(token): continue nearby = set(buckets[token]) for position in range(len(token)): nearby |= buckets[token[:position] + token[position + 1 :]] one_edit = sorted(other for other in nearby if within_one_edit(token, vocabulary[other])) known_mixups = [token_to_id[expected] for expected, _ in confusions.get(token, [])] found = list(dict.fromkeys(known_mixups + one_edit)) table[index, : min(len(found), MAX_NEIGHBORS)] = found[:MAX_NEIGHBORS] return table class MatMulLinear(nn.Module): """Linear layer that exports as MatMul so ONNX Runtime dynamic quantization applies.""" def __init__(self, inputs: int, outputs: int): super().__init__() self.weight = nn.Parameter(nn.init.xavier_uniform_(torch.empty(inputs, outputs))) self.bias = nn.Parameter(torch.zeros(outputs)) def forward(self, values): return values.matmul(self.weight) + self.bias class Gru(nn.Module): """GRU built from MatMul layers because ONNX Runtime cannot quantize its GRU operator.""" def __init__(self, inputs: int, hidden: int): super().__init__() self.hidden = hidden self.reset_gate = MatMulLinear(inputs + hidden, hidden) self.update_gate = MatMulLinear(inputs + hidden, hidden) self.proposal_input = MatMulLinear(inputs, hidden) self.proposal_state = MatMulLinear(hidden, hidden) def forward(self, sequence): state = sequence[:, 0, :].sum(-1, keepdim=True).mul(0).repeat(1, self.hidden) for step in range(sequence.shape[1]): current = sequence[:, step, :] joined = torch.cat([current, state], dim=-1) reset = torch.sigmoid(self.reset_gate(joined)) update = torch.sigmoid(self.update_gate(joined)) proposal = torch.tanh(self.proposal_input(current) + reset * self.proposal_state(state)) state = (1 - update) * proposal + update * state return state class WrongWordModel(nn.Module): """Scores the typed word and its neighbors against the words on both sides.""" def __init__(self, vocabulary_size: int, embedding: int, hidden: int): super().__init__() self.embedding = nn.Embedding(vocabulary_size, embedding, padding_idx=PAD) self.word_bias = nn.Embedding(vocabulary_size, 1, padding_idx=PAD) self.left = Gru(embedding, hidden) self.right = Gru(embedding, hidden) self.mix = MatMulLinear(hidden * 2, hidden) self.project = MatMulLinear(hidden, embedding) self.typed_bonus = nn.Parameter(torch.tensor(1.0)) self.register_buffer("typed_position", F.one_hot(torch.tensor(0), CANDIDATES).float()) def forward(self, left, right, candidates): context = torch.cat([self.left(self.embedding(left)), self.right(self.embedding(right))], dim=-1) summary = self.project(torch.tanh(self.mix(context))) scores = (self.embedding(candidates) * summary.unsqueeze(1)).sum(-1) + self.word_bias(candidates).squeeze(-1) return scores + self.typed_bonus * self.typed_position INFLECTION_ENDINGS = {"", "s", "es", "d", "ed", "ing", "r", "er", "ly"} def is_inflection(first: str, second: str) -> bool: """True for pairs like friendship/friendships and continued/continues that only differ in an ending.""" shared = 0 while shared < min(len(first), len(second)) and first[shared] == second[shared]: shared += 1 return shared >= 3 and first[shared:] in INFLECTION_ENDINGS and second[shared:] in INFLECTION_ENDINGS class Sampler: """Draws training cases by swapping real words for nearby real words.""" def __init__(self, tokens: torch.Tensor, vocabulary: list[str], neighbors: np.ndarray, real_typed: np.ndarray, real_weights: np.ndarray, device: str, options: dict): self.init_args = (vocabulary, neighbors, real_typed, real_weights, options) self.tokens = tokens self.device = device self.vocabulary = vocabulary self.options = options self.real_typed = torch.from_numpy(real_typed).to(device).long() self.real_weights = torch.from_numpy(real_weights).to(device) self.has_real = self.real_weights.sum(1) > 0 self.neighbors = torch.from_numpy(neighbors).to(device).long() self.neighbor_counts = (self.neighbors != PAD).sum(1) neighbor_weights = (neighbors != PAD).astype(np.float32) if options["inflection_weight"] != 1: for row in np.nonzero(neighbors[:, 0] != PAD)[0]: for column in range(MAX_NEIGHBORS): other = neighbors[row, column] if other != PAD and is_inflection(vocabulary[row], vocabulary[other]): neighbor_weights[row, column] = options["inflection_weight"] neighbor_weights[neighbor_weights.sum(1) == 0] = 1 self.neighbor_weights = torch.from_numpy(neighbor_weights).to(device) lengths = torch.tensor([len(token) for token in vocabulary], device=device) typed_ok = lengths >= options["min_typed"] if options["short_listed"]: typed_ok |= (lengths >= 2) & self.has_listed(real_typed, len(vocabulary), device) self.typed_ok = typed_ok self.is_target = torch.tensor([is_target_word(token) for token in vocabulary], device=device) self.group_share = options["group_share"] if self.group_share > 0: group_ids = torch.tensor(sorted(options["group_ids"]), device=device) inside = torch.zeros(len(tokens), dtype=torch.bool, device=device) inside[LEFT_WINDOW + 1 : len(tokens) - RIGHT_WINDOW - 1] = True where = torch.nonzero(torch.isin(tokens, group_ids) & inside).squeeze(1) order = torch.argsort(tokens[where], stable=True) self.group_positions = where[order] type_counts = torch.bincount(tokens[self.group_positions], minlength=len(vocabulary)) self.group_starts = torch.cumsum(type_counts, 0) - type_counts self.group_counts = type_counts self.group_type_weights = type_counts.float().pow(options["group_power"]) print(f"group positions={len(where):,} types={(type_counts > 0).sum().item()}", flush=True) @staticmethod def has_listed(real_typed: np.ndarray, size: int, device: str) -> torch.Tensor: listed = torch.zeros(size, dtype=torch.bool, device=device) typed = torch.from_numpy(real_typed[real_typed != PAD]).to(device).long() listed[typed] = True return listed def to(self, device: str) -> "Sampler": return Sampler(self.tokens.to(device), *self.init_args[:4], device, self.init_args[4]) def positions(self, size: int) -> torch.Tensor: positions = torch.randint(LEFT_WINDOW + 1, len(self.tokens) - RIGHT_WINDOW - 1, (size,), device=self.device) if self.group_share <= 0: return positions from_groups = torch.rand(size, device=self.device) < self.group_share types = torch.multinomial(self.group_type_weights, size, replacement=True) offsets = (torch.rand(size, device=self.device) * self.group_counts[types]).long() chosen = self.group_positions[(self.group_starts[types] + offsets).clamp(max=len(self.group_positions) - 1)] return torch.where(from_groups, chosen, positions) def batch(self, size: int, corrupt_rate: float): positions = self.positions(size) target = self.tokens[positions] counts = self.neighbor_counts[target] choice = torch.multinomial(self.neighbor_weights[target], 1).squeeze(1) corrupted = self.neighbors[target, choice] weights = self.real_weights[target] weights = weights.where(self.has_real[target].unsqueeze(1), torch.ones_like(weights)) real_choice = torch.multinomial(weights, 1).squeeze(1) uses_real = self.has_real[target] & (torch.rand(size, device=self.device) < self.options["real_share"]) corrupted = torch.where(uses_real, self.real_typed[target, real_choice], corrupted) counts = counts.where(~uses_real, torch.ones_like(counts)) is_corrupted = (torch.rand(size, device=self.device) < corrupt_rate) & (counts > 0) typed = torch.where(is_corrupted, corrupted, target) candidates = torch.cat([typed.unsqueeze(1), self.neighbors[typed]], dim=1) matches = candidates == target.unsqueeze(1) valid = self.is_target[target] & self.typed_ok[typed] & (self.neighbor_counts[typed] > 0) & matches.any(1) positions, candidates, matches, is_corrupted = positions[valid], candidates[valid], matches[valid], is_corrupted[valid] count = len(positions) left_offsets = torch.arange(-LEFT_WINDOW, 0, device=self.device) left = self.tokens[positions.unsqueeze(1) + left_offsets] right = self.tokens[positions.unsqueeze(1) + torch.arange(1, RIGHT_WINDOW + 1, device=self.device)] visible_right = torch.randint(0, RIGHT_WINDOW + 1, (count,), device=self.device).unsqueeze(1) right = right.where(torch.arange(RIGHT_WINDOW, device=self.device).unsqueeze(0) < visible_right, torch.tensor(PAD, device=self.device)).flip(1) is_short = torch.rand(count, device=self.device) < self.options["short_share"] kept = torch.randint(0, LEFT_WINDOW, (count,), device=self.device).unsqueeze(1) distance_from_end = (-left_offsets).unsqueeze(0) hidden_left = is_short.unsqueeze(1) & (distance_from_end > kept + 1) marks_start = is_short.unsqueeze(1) & (distance_from_end == kept + 1) left = left.where(~hidden_left, torch.tensor(PAD, device=self.device)).where(~marks_start, torch.tensor(START, device=self.device)) return left, right, candidates, matches.long().argmax(1), is_corrupted def mask_empty(scores, candidates): return scores.where(candidates != PAD, torch.tensor(-1e4, device=scores.device)) @torch.no_grad() def evaluate(model, sampler, batches: int, batch_size: int) -> dict: model.eval() clean_margins, corrupted_margins, corrupted_right = [], [], [] for _ in range(batches): left, right, candidates, label, is_corrupted = sampler.batch(batch_size, 0.5) scores = mask_empty(model(left, right, candidates), candidates) best = scores[:, 1:].max(1) margin = best.values - scores[:, 0] clean_margins.append(margin[~is_corrupted]) corrupted_margins.append(margin[is_corrupted]) corrupted_right.append(best.indices[is_corrupted] + 1 == label[is_corrupted]) model.train() clean = torch.cat(clean_margins) margin = torch.cat(corrupted_margins) right = torch.cat(corrupted_right) results = {"clean_cases": len(clean), "corrupted_cases": len(margin)} for false_rate in (0.005, 0.001, 0.0005): threshold = torch.quantile(clean[:200000].float(), 1 - false_rate).item() results[f"caught_at_false_{false_rate}"] = round(((margin >= threshold) & right).float().mean().item(), 4) results[f"threshold_at_false_{false_rate}"] = round(threshold, 2) return results def export_onnx(model: WrongWordModel, output: Path, sampler: Sampler) -> dict: model = model.cpu().eval() float_path = output / "model.onnx" example = (torch.ones(2, LEFT_WINDOW, dtype=torch.long), torch.ones(2, RIGHT_WINDOW, dtype=torch.long), torch.ones(2, CANDIDATES, dtype=torch.long)) torch.onnx.export( model, example, float_path, input_names=["left", "right", "candidates"], output_names=["scores"], dynamic_axes={"left": {0: "batch"}, "right": {0: "batch"}, "candidates": {0: "batch"}, "scores": {0: "batch"}}, opset_version=17, dynamo=False, ) quantized = { "model_quint8.onnx": ["MatMul", "Gather"], "model_matmul_quint8.onnx": ["MatMul"], } for name, operators in quantized.items(): quantize_dynamic(float_path, output / name, weight_type=QuantType.QUInt8, op_types_to_quantize=operators) sampler_cpu = sampler.to("cpu") left, right, candidates, label, _ = sampler_cpu.batch(4096, 0.5) inputs = {"left": left.numpy().astype(np.int64), "right": right.numpy().astype(np.int64), "candidates": candidates.numpy().astype(np.int64)} with torch.no_grad(): reference = model(left, right, candidates).numpy() def run(path: Path) -> np.ndarray: return ort.InferenceSession(str(path), providers=["CPUExecutionProvider"]).run(None, inputs)[0] def margins(scores: np.ndarray) -> np.ndarray: scores = np.where(inputs["candidates"] != PAD, scores, -1e4) return scores[:, 1:].max(1) - scores[:, 0] report = {"float_vs_torch_max_difference": float(np.abs(run(float_path) - reference).max())} report["float_mb"] = round(float_path.stat().st_size / 1e6, 2) for name in quantized: scores = run(output / name) report[name] = { "mb": round((output / name).stat().st_size / 1e6, 2), "pick_agreement": round(float((scores.argmax(1) == reference.argmax(1)).mean()), 4), "mean_margin_difference": round(float(np.abs(margins(scores) - margins(reference)).mean()), 3), } return report def main(): parser = argparse.ArgumentParser() parser.add_argument("--repo", default="cfahlgren1/tinyfix-lab") parser.add_argument("--run-name", required=True) parser.add_argument("--groups", choices=sorted(GROUP_SETS), default="shipped") parser.add_argument("--min-typed", type=int, default=MIN_TYPED_LENGTH) parser.add_argument("--short-listed", action="store_true", help="Also train two-letter typed words that have listed mix-ups.") parser.add_argument("--real-share", type=float, default=REAL_MISTAKE_SHARE) parser.add_argument("--group-share", type=float, default=0.0, help="Share of cases centered on a word from a mix-up group.") parser.add_argument("--group-power", type=float, default=0.5) parser.add_argument("--inflection-weight", type=float, default=1.0) parser.add_argument("--short-share", type=float, default=0.2) parser.add_argument("--corrupt-rate", type=float, default=0.5) parser.add_argument("--force-group-words", action="store_true") parser.add_argument("--slip-words", type=int, default=0) parser.add_argument("--real-mistakes", action="store_true", help="Also train on GitHub word swaps. Scored worse than homophones alone in v3.") parser.add_argument("--fineweb-tokens", type=int, default=70_000_000) parser.add_argument("--smoltalk-tokens", type=int, default=50_000_000) parser.add_argument("--vocabulary", type=int, default=20_000) parser.add_argument("--embedding", type=int, default=64) parser.add_argument("--hidden", type=int, default=256) parser.add_argument("--steps", type=int, default=60_000) parser.add_argument("--batch", type=int, default=4096) parser.add_argument("--learning-rate", type=float, default=2e-3) parser.add_argument("--output", default="/tmp/tinyfix-realword") parser.add_argument("--no-upload", action="store_true") parser.add_argument("--pairs-file", default="", help="Path in --repo of a mix-up list to train on instead of --groups.") parser.add_argument("--seed", type=int, default=0) args = parser.parse_args() torch.manual_seed(args.seed) device = "cuda" if torch.cuda.is_available() else "cpu" output = Path(args.output) output.mkdir(parents=True, exist_ok=True) print(f"device={device} args={vars(args)}", flush=True) api = HfApi(token=os.environ["HF_TOKEN"]) if args.pairs_file: PAIRS_FILE.extend(load_pairs_file(api, args.repo, args.pairs_file)) print(f"pairs from {args.pairs_file}: {len(PAIRS_FILE)}", flush=True) documents, raw_ids = load_corpus_cached({"fineweb": args.fineweb_tokens, "smoltalk": args.smoltalk_tokens, "oasst": 10**12}, api) pairs = group_pairs(args.groups) forced = list(dict.fromkeys(word for pair in pairs for word in pair)) if args.force_group_words else [] if args.slip_words: forced += [word for word in SLIP_WORDS if word not in forced] vocabulary, lookup = build_vocabulary(documents, raw_ids, args.vocabulary, forced, args.slip_words) print(f"documents={len(documents):,} raw_types={len(raw_ids):,} vocabulary={len(vocabulary):,}", flush=True) order = np.random.default_rng(7).permutation(len(documents)) held_out = max(1, len(documents) // 20) train_tokens = torch.from_numpy(lookup[np.concatenate([documents[index] for index in order[held_out:]])]).to(device).long() validation_tokens = torch.from_numpy(lookup[np.concatenate([documents[index] for index in order[:held_out]])]).to(device).long() print(f"train={len(train_tokens):,} validation={len(validation_tokens):,} unknown={(train_tokens == UNK).float().mean().item():.3f}", flush=True) del documents min_typed = 2 if args.short_listed else args.min_typed confusions = build_confusions(vocabulary, args.real_mistakes, args.groups, min_typed) (output / "confusions.txt").write_text("".join(f"{typed} {' '.join(expected for expected, _ in options)}\n" for typed, options in confusions.items())) neighbors = build_neighbors(vocabulary, confusions) real_typed, real_weights = build_real_mistake_tables(vocabulary, confusions) print(f"words with neighbors={(neighbors[:, 0] != PAD).sum():,}", flush=True) (output / "vocab.txt").write_text("\n".join(vocabulary) + "\n") token_to_id = {token: index for index, token in enumerate(vocabulary)} options = { "min_typed": args.min_typed, "short_listed": args.short_listed, "inflection_weight": args.inflection_weight, "group_share": args.group_share, "group_power": args.group_power, "real_share": args.real_share, "short_share": args.short_share, "group_ids": {token_to_id[word] for pair in pairs for word in pair if word in token_to_id}, } train_sampler = Sampler(train_tokens, vocabulary, neighbors, real_typed, real_weights, device, options) validation_sampler = Sampler(validation_tokens, vocabulary, neighbors, real_typed, real_weights, device, {**options, "group_share": 0.0}) model = WrongWordModel(len(vocabulary), args.embedding, args.hidden).to(device) parameters = sum(parameter.numel() for parameter in model.parameters()) print(f"parameters={parameters:,}", flush=True) optimizer = torch.optim.AdamW(model.parameters(), lr=args.learning_rate, weight_decay=1e-6) schedule = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr=args.learning_rate, total_steps=args.steps, pct_start=0.1) started = time.time() running_loss = 0.0 report_every = max(args.steps // 40, 1) for step in range(1, args.steps + 1): left, right, candidates, label, _ = train_sampler.batch(args.batch, args.corrupt_rate) loss = F.cross_entropy(mask_empty(model(left, right, candidates), candidates), label) optimizer.zero_grad(set_to_none=True) loss.backward() torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0) optimizer.step() schedule.step() running_loss += loss.item() if step % report_every == 0: print(f"step {step}/{args.steps} loss={running_loss / report_every:.4f} elapsed={time.time() - started:.0f}s", flush=True) running_loss = 0.0 if step % (args.steps // 5 or 1) == 0: print(f"validation step {step}: {evaluate(model, validation_sampler, 60, args.batch)}", flush=True) validation = evaluate(model, validation_sampler, 200, args.batch) print(f"final validation: {validation}", flush=True) torch.save(model.state_dict(), output / "model.pt") export_report = export_onnx(model, output, validation_sampler) print(f"export: {export_report}", flush=True) metrics = { "args": vars(args), "parameters": parameters, "validation": validation, "export": export_report, "train_tokens": len(train_tokens), "train_minutes": round((time.time() - started) / 60, 1), } (output / "metrics.json").write_text(json.dumps(metrics, indent=2) + "\n") print(json.dumps(metrics, indent=2), flush=True) if args.no_upload: print(f"Skipping upload; artifacts in {output}", flush=True) return api.create_repo(args.repo, repo_type="model", private=True, exist_ok=True) api.upload_folder(folder_path=str(output), repo_id=args.repo, path_in_repo=f"combine/{args.run_name}", repo_type="model") print(f"Uploaded to {args.repo}/combine/{args.run_name}", flush=True) if __name__ == "__main__": main()