ONNX
English
autocorrect
spelling-correction
browser
offline-page
tinyfix / train_realword.py
cfahlgren1's picture
cfahlgren1 HF Staff
retrain on more mix-up groups: fixes about 40% of wrong words on the new test bench, up from 24%
f4cab9b verified
Raw History Blame Contribute Delete
37.1 kB
# /// script
# requires-python = ">=3.11,<3.14"
# dependencies = [
# "torch>=2.6,<2.8",
# "datasets>=3.0",
# "numpy<3",
# "onnx>=1.17",
# "onnxruntime>=1.20",
# "huggingface_hub>=0.30",
# ]
# ///
"""Train a small model that spots correctly spelled wrong words, like "don" typed for "do"."""
import argparse
import hashlib
import json
import os
import re
import time
import urllib.request
from collections import Counter, defaultdict
from pathlib import Path
import numpy as np
import onnxruntime as ort
import torch
import torch.nn as nn
import torch.nn.functional as F
from datasets import load_dataset
from huggingface_hub import HfApi
from onnxruntime.quantization import QuantType, quantize_dynamic
LEFT_WINDOW = 16
RIGHT_WINDOW = 4
MAX_NEIGHBORS = 8
CANDIDATES = MAX_NEIGHBORS + 1
MIN_TYPED_LENGTH = 3
PAD, UNK, NUMBER, START, NAME = 0, 1, 2, 3, 4
SPECIAL_TOKENS = ["<pad>", "<unk>", "<num>", "<s>", "<name>"]
REAL_MISTAKE_SHARE = 0.6
HOMOPHONE_WEIGHT = 20
MIN_REAL_MISTAKE_COUNT = 2
HOMOPHONES = [
("their", "there", "they're"), ("your", "you're"), ("its", "it's"), ("too", "two"), ("then", "than"),
("were", "where", "we're"), ("whose", "who's"), ("hear", "here"), ("buy", "bye"), ("weather", "whether"),
("peace", "piece"), ("affect", "effect"), ("lose", "loose"), ("accept", "except"), ("advice", "advise"),
("quiet", "quite"), ("through", "though", "thorough"), ("passed", "past"), ("principal", "principle"),
("complement", "compliment"), ("brake", "break"), ("waist", "waste"), ("right", "write"), ("sight", "site", "cite"),
("wear", "where"), ("led", "lead"), ("meet", "meat"), ("week", "weak"), ("wood", "would"), ("hole", "whole"),
("male", "mail"), ("sale", "sell"), ("role", "roll"), ("aloud", "allowed"), ("board", "bored"), ("fair", "fare"),
]
TYPO_PAIRS_DATASET = "chirunder/github_typo_corrections"
# Sound-alikes and classic mix-ups. Every member can be typed for every other member.
EXTENDED_GROUPS = HOMOPHONES + [
("to", "too", "two"), ("of", "off"), ("no", "know", "now"), ("by", "buy", "bye"), ("for", "four"), ("one", "won"),
("new", "knew"), ("wait", "weight"), ("way", "weigh"), ("plane", "plain"), ("made", "maid"), ("pair", "pear"),
("tale", "tail"), ("flour", "flower"), ("threw", "through"), ("son", "sun"), ("bought", "brought"), ("wear", "were"),
("bare", "bear"), ("dear", "deer"), ("hour", "our"), ("knight", "night"), ("knot", "not"), ("sea", "see"),
("steal", "steel"), ("stair", "stare"), ("tide", "tied"), ("which", "witch"), ("course", "coarse"), ("sent", "scent", "cent"),
("die", "dye"), ("flew", "flu"), ("great", "grate"), ("heal", "heel"), ("heard", "herd"), ("lessen", "lesson"),
("main", "mane"), ("medal", "meddle"), ("missed", "mist"), ("morning", "mourning"), ("pain", "pane"), ("pause", "paws"),
("peak", "peek"), ("pole", "poll"), ("poor", "pour", "pore"), ("pray", "prey"), ("rain", "reign", "rein"), ("read", "red"),
("real", "reel"), ("road", "rode"), ("root", "route"), ("rose", "rows"), ("scene", "seen"), ("seam", "seem"),
("sole", "soul"), ("some", "sum"), ("stationary", "stationery"), ("throne", "thrown"), ("vain", "vein"), ("waive", "wave"),
("ate", "eight"), ("blew", "blue"), ("cell", "sell"), ("cereal", "serial"), ("council", "counsel"), ("desert", "dessert"),
("dual", "duel"), ("fourth", "forth"), ("guessed", "guest"), ("hall", "haul"), ("higher", "hire"), ("loan", "lone"),
("hours", "ours"), ("presence", "presents"), ("sail", "sale"), ("stake", "steak"), ("breath", "breathe"), ("chose", "choose"),
("clothes", "cloths"), ("dose", "does"), ("later", "latter"), ("personal", "personnel"), ("precede", "proceed"),
("quite", "quit"), ("woman", "women"), ("form", "from"), ("lose", "loss"), ("whit", "with"),
("grown", "groan"), ("idle", "idol"), ("sweet", "suite"),
("ensure", "insure"), ("emigrate", "immigrate"),
("we", "wee"), ("so", "sew"),
]
# Mix-ups that only make sense one way ("could of" for "could have").
ONE_WAY = [("of", "have"), ("then", "than"), ("loose", "lose"), ("defiantly", "definitely")]
GROUP_SETS = {"shipped": HOMOPHONES, "extended": EXTENDED_GROUPS}
SLIP_WORDS = ["fro", "nit", "grate", "manger", "bye", "whit", "wee", "thee", "lest", "sew"]
WORD_LIST_URL = "https://raw.githubusercontent.com/wolfgarbe/SymSpell/master/SymSpell.FrequencyDictionary/en-80k.txt"
TOKEN = re.compile(r"[A-Za-z]+(?:'[A-Za-z]+)?|\d+(?:[.,]\d+)*|[^\s\w]")
SENTENCE_OPENERS = {".", "!", "?", ":", '"'}
WORD = re.compile(r"^[a-z]+(?:'[a-z]+)?$")
DIGITS = re.compile(r"^\d")
def tokenize(text: str) -> list[str]:
"""Lowercases words, except capitalized words mid-sentence, which become a single name token."""
tokens = []
for raw in TOKEN.findall(text.replace("’", "'")):
at_sentence_start = not tokens or tokens[-1] in SENTENCE_OPENERS
is_pronoun = raw == "I" or raw.startswith("I'")
tokens.append("<name>" if raw[0].isupper() and not at_sentence_start and not is_pronoun else raw.lower())
return tokens
def fineweb_documents():
stream = load_dataset("HuggingFaceFW/fineweb-edu", split="train", streaming=True).shuffle(seed=7, buffer_size=20000)
for row in stream:
yield row["text"]
def smoltalk_documents():
stream = load_dataset("HuggingFaceTB/smoltalk", "all", split="train", streaming=True).shuffle(seed=7, buffer_size=20000)
for row in stream:
yield "\n".join(message["content"] for message in row["messages"] if message["role"] != "system")
def open_assistant_documents():
for row in load_dataset("OpenAssistant/oasst1", split="train"):
if row["lang"] == "en":
yield row["text"]
def load_corpus(token_budgets: dict[str, int]) -> tuple[list[np.ndarray], dict[str, int]]:
sources = {"fineweb": fineweb_documents, "smoltalk": smoltalk_documents, "oasst": open_assistant_documents}
raw_ids = {"<s>": 0}
documents = []
for name, budget in token_budgets.items():
started, used = time.time(), 0
for text in sources[name]():
tokens = tokenize(text)
if len(tokens) < 8:
continue
documents.append(np.array([0] + [raw_ids.setdefault(token, len(raw_ids)) for token in tokens], dtype=np.int32))
used += len(tokens)
if used >= budget:
break
print(f"{name}: {used:,} tokens in {time.time() - started:.0f}s", flush=True)
return documents, raw_ids
CACHE_REPO = "cfahlgren1/tinyfix-lab"
def load_corpus_cached(token_budgets: dict[str, int], api: HfApi) -> tuple[list[np.ndarray], dict[str, int]]:
"""Tokenizing 120M tokens takes a while, so the first run saves the result to the private lab repo."""
folder = f"data/corpus-{token_budgets['fineweb']}-{token_budgets['smoltalk']}"
try:
paths = {name: api.hf_hub_download(CACHE_REPO, f"{folder}/{name}") for name in ("tokens.npy", "lengths.npy", "raw_tokens.json")}
tokens, lengths = np.load(paths["tokens.npy"]), np.load(paths["lengths.npy"])
raw_tokens = json.loads(Path(paths["raw_tokens.json"]).read_text())
print(f"loaded cached corpus {folder}: {len(tokens):,} tokens", flush=True)
return np.split(tokens, np.cumsum(lengths)[:-1]), {token: index for index, token in enumerate(raw_tokens)}
except Exception as error:
print(f"no cached corpus ({type(error).__name__}); building it", flush=True)
documents, raw_ids = load_corpus(token_budgets)
cache = Path("/tmp/corpus-cache")
cache.mkdir(exist_ok=True)
np.save(cache / "tokens.npy", np.concatenate(documents))
np.save(cache / "lengths.npy", np.array([len(document) for document in documents], dtype=np.int64))
(cache / "raw_tokens.json").write_text(json.dumps(list(raw_ids)))
api.create_repo(CACHE_REPO, repo_type="model", private=True, exist_ok=True)
api.upload_folder(folder_path=str(cache), repo_id=CACHE_REPO, path_in_repo=folder, repo_type="model")
print(f"cached corpus to {CACHE_REPO}/{folder}", flush=True)
return documents, raw_ids
def slip_words(ranked: list[str], counts_by_token: dict[str, int], start: int, how_many: int, min_count: int) -> list[str]:
"""Words just outside the vocabulary that are one edit from a common word, so a slip that lands on them can be judged."""
common = ranked[:3000]
buckets = defaultdict(list)
for word in common:
if len(word) < 3:
continue
for position in range(len(word)):
buckets[word[:position] + word[position + 1 :]].append(word)
buckets[word].append(word)
found = []
for word in ranked[start:]:
if len(found) >= how_many or counts_by_token[word] < min_count:
break
if len(word) < 3 or "'" in word:
continue
keys = {word} | {word[:position] + word[position + 1 :] for position in range(len(word))}
if any(within_one_edit(word, other) for key in keys for other in buckets.get(key, [])):
found.append(word)
return found
def build_vocabulary(documents: list[np.ndarray], raw_ids: dict[str, int], size: int, forced: list[str], slips: int) -> tuple[list[str], np.ndarray]:
with urllib.request.urlopen(WORD_LIST_URL) as response:
dictionary = {line.split()[0] for line in response.read().decode().splitlines() if line.strip()}
counts = np.bincount(np.concatenate(documents), minlength=len(raw_ids))
allowed = [(token, identifier) for token, identifier in raw_ids.items() if (WORD.match(token) and token in dictionary) or (len(token) == 1 and not token.isalnum())]
allowed = [pair for pair in allowed if pair[0] != "<s>"]
allowed.sort(key=lambda pair: (-counts[pair[1]], pair[0]))
ranked = [token for token, _ in allowed]
room = size - len(SPECIAL_TOKENS)
extra = [word for word in forced if word in raw_ids and word not in ranked[:room]]
if slips:
counts_by_token = {token: int(counts[identifier]) for token, identifier in allowed}
extra += [word for word in slip_words(ranked, counts_by_token, room, slips, 20) if word not in extra]
missing = [word for word in forced if word not in raw_ids]
print(f"vocabulary: forcing {len(extra)} extra words, e.g. {extra[:40]}; not in corpus: {missing}", flush=True)
vocabulary = SPECIAL_TOKENS + ranked[: room - len(extra)] + extra
token_to_id = {token: index for index, token in enumerate(vocabulary)}
lookup = np.full(len(raw_ids), UNK, dtype=np.int32)
for token, identifier in raw_ids.items():
if token == "<s>":
lookup[identifier] = START
elif token == "<name>":
lookup[identifier] = NAME
elif DIGITS.match(token):
lookup[identifier] = NUMBER
elif token in token_to_id:
lookup[identifier] = token_to_id[token]
return vocabulary, lookup
def is_target_word(token: str) -> bool:
return len(token) >= 2 and token not in SPECIAL_TOKENS and WORD.match(token) is not None
def within_one_edit(first: str, second: str) -> bool:
if first == second or abs(len(first) - len(second)) > 1:
return False
if len(first) == len(second):
differences = [index for index, (a, b) in enumerate(zip(first, second)) if a != b]
if len(differences) == 1:
return True
return len(differences) == 2 and differences[1] == differences[0] + 1 and first[differences[0]] == second[differences[1]] and first[differences[1]] == second[differences[0]]
shorter, longer = sorted((first, second), key=len)
return any(longer[:index] + longer[index + 1 :] == shorter for index in range(len(longer)))
def strip_punctuation(token: str) -> str:
return token.strip("\"'()[]{}<>.,;:!?*_`#-=/\\")
def real_mistakes_for_training(vocabulary: set[str]) -> Counter:
"""Counts real typed-for-expected word swaps, using only the half of the dev split that eval doesn't tune on."""
counts = Counter()
for row in load_dataset(TYPO_PAIRS_DATASET, split="train"):
typed_tokens, fixed_tokens = (row["text"] or "").split(), (row["correction"] or "").split()
if len(typed_tokens) != len(fixed_tokens):
continue
different = [index for index, (first, second) in enumerate(zip(typed_tokens, fixed_tokens)) if first != second]
if len(different) != 1:
continue
typed, expected = strip_punctuation(typed_tokens[different[0]]), strip_punctuation(fixed_tokens[different[0]])
digest = int(hashlib.sha1(f"{typed}|{expected}".lower().encode()).hexdigest(), 16)
if digest % 2 != 0 or (digest >> 1) % 2 != 0:
continue
typed, expected = typed.lower(), expected.lower()
if typed != expected and len(typed) >= MIN_TYPED_LENGTH and typed in vocabulary and expected in vocabulary and is_target_word(typed) and is_target_word(expected):
counts[(typed, expected)] += 1
return Counter({pair: count for pair, count in counts.items() if count >= MIN_REAL_MISTAKE_COUNT})
PAIRS_FILE: list[tuple[str, str]] = []
def load_pairs_file(api: HfApi, repo: str, path: str) -> list[tuple[str, str]]:
"""Reads a mix-up list ("typed meant1 meant2 ...") from the lab repo, so training and the engine use the same pairs."""
text = Path(api.hf_hub_download(repo, path)).read_text()
pairs = [(parts[0], meant) for parts in (line.split() for line in text.splitlines()) if parts for meant in parts[1:]]
return list(dict.fromkeys(pairs))
def group_pairs(groups: str) -> list[tuple[str, str]]:
if PAIRS_FILE:
return PAIRS_FILE
pairs = [(typed, expected) for group in GROUP_SETS[groups] for typed in group for expected in group if typed != expected]
if groups != "shipped":
pairs += ONE_WAY
return list(dict.fromkeys(pairs))
def build_confusions(vocabulary: list[str], use_real_mistakes: bool, groups: str, min_typed: int) -> dict[str, list[tuple[str, int]]]:
"""Maps a typed word to words people really meant by it, most likely first."""
known = set(vocabulary)
counts = real_mistakes_for_training(known) if use_real_mistakes else Counter()
print(f"real mistake pairs={len(counts):,}", flush=True)
for typed, expected in group_pairs(groups):
if typed in known and expected in known and len(typed) >= min_typed:
counts[(typed, expected)] += HOMOPHONE_WEIGHT
confusions = defaultdict(list)
for (typed, expected), count in sorted(counts.items(), key=lambda item: (-item[1], item[0])):
confusions[typed].append((expected, count))
return {typed: options[:MAX_NEIGHBORS] for typed, options in confusions.items()}
def build_real_mistake_tables(vocabulary: list[str], confusions: dict[str, list[tuple[str, int]]]) -> tuple[np.ndarray, np.ndarray]:
"""For each intended word, lists the words typed by mistake and how often, for sampling."""
token_to_id = {token: index for index, token in enumerate(vocabulary)}
mistakes = defaultdict(list)
for typed, options in confusions.items():
for expected, count in options:
mistakes[token_to_id[expected]].append((token_to_id[typed], count))
typed_table = np.zeros((len(vocabulary), MAX_NEIGHBORS), dtype=np.int32)
weights = np.zeros((len(vocabulary), MAX_NEIGHBORS), dtype=np.float32)
for expected, options in mistakes.items():
options = sorted(options, key=lambda option: -option[1])[:MAX_NEIGHBORS]
typed_table[expected, : len(options)] = [typed for typed, _ in options]
weights[expected, : len(options)] = [count**0.5 for _, count in options]
return typed_table, weights
def build_neighbors(vocabulary: list[str], confusions: dict[str, list[tuple[str, int]]]) -> np.ndarray:
"""Lists known mix-ups first, then the most common words one edit away, the same order the browser uses."""
token_to_id = {token: index for index, token in enumerate(vocabulary)}
buckets = defaultdict(set)
for index, token in enumerate(vocabulary):
if not is_target_word(token):
continue
buckets[token].add(index)
for position in range(len(token)):
buckets[token[:position] + token[position + 1 :]].add(index)
table = np.zeros((len(vocabulary), MAX_NEIGHBORS), dtype=np.int32)
for index, token in enumerate(vocabulary):
if not is_target_word(token):
continue
nearby = set(buckets[token])
for position in range(len(token)):
nearby |= buckets[token[:position] + token[position + 1 :]]
one_edit = sorted(other for other in nearby if within_one_edit(token, vocabulary[other]))
known_mixups = [token_to_id[expected] for expected, _ in confusions.get(token, [])]
found = list(dict.fromkeys(known_mixups + one_edit))
table[index, : min(len(found), MAX_NEIGHBORS)] = found[:MAX_NEIGHBORS]
return table
class MatMulLinear(nn.Module):
"""Linear layer that exports as MatMul so ONNX Runtime dynamic quantization applies."""
def __init__(self, inputs: int, outputs: int):
super().__init__()
self.weight = nn.Parameter(nn.init.xavier_uniform_(torch.empty(inputs, outputs)))
self.bias = nn.Parameter(torch.zeros(outputs))
def forward(self, values):
return values.matmul(self.weight) + self.bias
class Gru(nn.Module):
"""GRU built from MatMul layers because ONNX Runtime cannot quantize its GRU operator."""
def __init__(self, inputs: int, hidden: int):
super().__init__()
self.hidden = hidden
self.reset_gate = MatMulLinear(inputs + hidden, hidden)
self.update_gate = MatMulLinear(inputs + hidden, hidden)
self.proposal_input = MatMulLinear(inputs, hidden)
self.proposal_state = MatMulLinear(hidden, hidden)
def forward(self, sequence):
state = sequence[:, 0, :].sum(-1, keepdim=True).mul(0).repeat(1, self.hidden)
for step in range(sequence.shape[1]):
current = sequence[:, step, :]
joined = torch.cat([current, state], dim=-1)
reset = torch.sigmoid(self.reset_gate(joined))
update = torch.sigmoid(self.update_gate(joined))
proposal = torch.tanh(self.proposal_input(current) + reset * self.proposal_state(state))
state = (1 - update) * proposal + update * state
return state
class WrongWordModel(nn.Module):
"""Scores the typed word and its neighbors against the words on both sides."""
def __init__(self, vocabulary_size: int, embedding: int, hidden: int):
super().__init__()
self.embedding = nn.Embedding(vocabulary_size, embedding, padding_idx=PAD)
self.word_bias = nn.Embedding(vocabulary_size, 1, padding_idx=PAD)
self.left = Gru(embedding, hidden)
self.right = Gru(embedding, hidden)
self.mix = MatMulLinear(hidden * 2, hidden)
self.project = MatMulLinear(hidden, embedding)
self.typed_bonus = nn.Parameter(torch.tensor(1.0))
self.register_buffer("typed_position", F.one_hot(torch.tensor(0), CANDIDATES).float())
def forward(self, left, right, candidates):
context = torch.cat([self.left(self.embedding(left)), self.right(self.embedding(right))], dim=-1)
summary = self.project(torch.tanh(self.mix(context)))
scores = (self.embedding(candidates) * summary.unsqueeze(1)).sum(-1) + self.word_bias(candidates).squeeze(-1)
return scores + self.typed_bonus * self.typed_position
INFLECTION_ENDINGS = {"", "s", "es", "d", "ed", "ing", "r", "er", "ly"}
def is_inflection(first: str, second: str) -> bool:
"""True for pairs like friendship/friendships and continued/continues that only differ in an ending."""
shared = 0
while shared < min(len(first), len(second)) and first[shared] == second[shared]:
shared += 1
return shared >= 3 and first[shared:] in INFLECTION_ENDINGS and second[shared:] in INFLECTION_ENDINGS
class Sampler:
"""Draws training cases by swapping real words for nearby real words."""
def __init__(self, tokens: torch.Tensor, vocabulary: list[str], neighbors: np.ndarray, real_typed: np.ndarray, real_weights: np.ndarray, device: str, options: dict):
self.init_args = (vocabulary, neighbors, real_typed, real_weights, options)
self.tokens = tokens
self.device = device
self.vocabulary = vocabulary
self.options = options
self.real_typed = torch.from_numpy(real_typed).to(device).long()
self.real_weights = torch.from_numpy(real_weights).to(device)
self.has_real = self.real_weights.sum(1) > 0
self.neighbors = torch.from_numpy(neighbors).to(device).long()
self.neighbor_counts = (self.neighbors != PAD).sum(1)
neighbor_weights = (neighbors != PAD).astype(np.float32)
if options["inflection_weight"] != 1:
for row in np.nonzero(neighbors[:, 0] != PAD)[0]:
for column in range(MAX_NEIGHBORS):
other = neighbors[row, column]
if other != PAD and is_inflection(vocabulary[row], vocabulary[other]):
neighbor_weights[row, column] = options["inflection_weight"]
neighbor_weights[neighbor_weights.sum(1) == 0] = 1
self.neighbor_weights = torch.from_numpy(neighbor_weights).to(device)
lengths = torch.tensor([len(token) for token in vocabulary], device=device)
typed_ok = lengths >= options["min_typed"]
if options["short_listed"]:
typed_ok |= (lengths >= 2) & self.has_listed(real_typed, len(vocabulary), device)
self.typed_ok = typed_ok
self.is_target = torch.tensor([is_target_word(token) for token in vocabulary], device=device)
self.group_share = options["group_share"]
if self.group_share > 0:
group_ids = torch.tensor(sorted(options["group_ids"]), device=device)
inside = torch.zeros(len(tokens), dtype=torch.bool, device=device)
inside[LEFT_WINDOW + 1 : len(tokens) - RIGHT_WINDOW - 1] = True
where = torch.nonzero(torch.isin(tokens, group_ids) & inside).squeeze(1)
order = torch.argsort(tokens[where], stable=True)
self.group_positions = where[order]
type_counts = torch.bincount(tokens[self.group_positions], minlength=len(vocabulary))
self.group_starts = torch.cumsum(type_counts, 0) - type_counts
self.group_counts = type_counts
self.group_type_weights = type_counts.float().pow(options["group_power"])
print(f"group positions={len(where):,} types={(type_counts > 0).sum().item()}", flush=True)
@staticmethod
def has_listed(real_typed: np.ndarray, size: int, device: str) -> torch.Tensor:
listed = torch.zeros(size, dtype=torch.bool, device=device)
typed = torch.from_numpy(real_typed[real_typed != PAD]).to(device).long()
listed[typed] = True
return listed
def to(self, device: str) -> "Sampler":
return Sampler(self.tokens.to(device), *self.init_args[:4], device, self.init_args[4])
def positions(self, size: int) -> torch.Tensor:
positions = torch.randint(LEFT_WINDOW + 1, len(self.tokens) - RIGHT_WINDOW - 1, (size,), device=self.device)
if self.group_share <= 0:
return positions
from_groups = torch.rand(size, device=self.device) < self.group_share
types = torch.multinomial(self.group_type_weights, size, replacement=True)
offsets = (torch.rand(size, device=self.device) * self.group_counts[types]).long()
chosen = self.group_positions[(self.group_starts[types] + offsets).clamp(max=len(self.group_positions) - 1)]
return torch.where(from_groups, chosen, positions)
def batch(self, size: int, corrupt_rate: float):
positions = self.positions(size)
target = self.tokens[positions]
counts = self.neighbor_counts[target]
choice = torch.multinomial(self.neighbor_weights[target], 1).squeeze(1)
corrupted = self.neighbors[target, choice]
weights = self.real_weights[target]
weights = weights.where(self.has_real[target].unsqueeze(1), torch.ones_like(weights))
real_choice = torch.multinomial(weights, 1).squeeze(1)
uses_real = self.has_real[target] & (torch.rand(size, device=self.device) < self.options["real_share"])
corrupted = torch.where(uses_real, self.real_typed[target, real_choice], corrupted)
counts = counts.where(~uses_real, torch.ones_like(counts))
is_corrupted = (torch.rand(size, device=self.device) < corrupt_rate) & (counts > 0)
typed = torch.where(is_corrupted, corrupted, target)
candidates = torch.cat([typed.unsqueeze(1), self.neighbors[typed]], dim=1)
matches = candidates == target.unsqueeze(1)
valid = self.is_target[target] & self.typed_ok[typed] & (self.neighbor_counts[typed] > 0) & matches.any(1)
positions, candidates, matches, is_corrupted = positions[valid], candidates[valid], matches[valid], is_corrupted[valid]
count = len(positions)
left_offsets = torch.arange(-LEFT_WINDOW, 0, device=self.device)
left = self.tokens[positions.unsqueeze(1) + left_offsets]
right = self.tokens[positions.unsqueeze(1) + torch.arange(1, RIGHT_WINDOW + 1, device=self.device)]
visible_right = torch.randint(0, RIGHT_WINDOW + 1, (count,), device=self.device).unsqueeze(1)
right = right.where(torch.arange(RIGHT_WINDOW, device=self.device).unsqueeze(0) < visible_right, torch.tensor(PAD, device=self.device)).flip(1)
is_short = torch.rand(count, device=self.device) < self.options["short_share"]
kept = torch.randint(0, LEFT_WINDOW, (count,), device=self.device).unsqueeze(1)
distance_from_end = (-left_offsets).unsqueeze(0)
hidden_left = is_short.unsqueeze(1) & (distance_from_end > kept + 1)
marks_start = is_short.unsqueeze(1) & (distance_from_end == kept + 1)
left = left.where(~hidden_left, torch.tensor(PAD, device=self.device)).where(~marks_start, torch.tensor(START, device=self.device))
return left, right, candidates, matches.long().argmax(1), is_corrupted
def mask_empty(scores, candidates):
return scores.where(candidates != PAD, torch.tensor(-1e4, device=scores.device))
@torch.no_grad()
def evaluate(model, sampler, batches: int, batch_size: int) -> dict:
model.eval()
clean_margins, corrupted_margins, corrupted_right = [], [], []
for _ in range(batches):
left, right, candidates, label, is_corrupted = sampler.batch(batch_size, 0.5)
scores = mask_empty(model(left, right, candidates), candidates)
best = scores[:, 1:].max(1)
margin = best.values - scores[:, 0]
clean_margins.append(margin[~is_corrupted])
corrupted_margins.append(margin[is_corrupted])
corrupted_right.append(best.indices[is_corrupted] + 1 == label[is_corrupted])
model.train()
clean = torch.cat(clean_margins)
margin = torch.cat(corrupted_margins)
right = torch.cat(corrupted_right)
results = {"clean_cases": len(clean), "corrupted_cases": len(margin)}
for false_rate in (0.005, 0.001, 0.0005):
threshold = torch.quantile(clean[:200000].float(), 1 - false_rate).item()
results[f"caught_at_false_{false_rate}"] = round(((margin >= threshold) & right).float().mean().item(), 4)
results[f"threshold_at_false_{false_rate}"] = round(threshold, 2)
return results
def export_onnx(model: WrongWordModel, output: Path, sampler: Sampler) -> dict:
model = model.cpu().eval()
float_path = output / "model.onnx"
example = (torch.ones(2, LEFT_WINDOW, dtype=torch.long), torch.ones(2, RIGHT_WINDOW, dtype=torch.long), torch.ones(2, CANDIDATES, dtype=torch.long))
torch.onnx.export(
model,
example,
float_path,
input_names=["left", "right", "candidates"],
output_names=["scores"],
dynamic_axes={"left": {0: "batch"}, "right": {0: "batch"}, "candidates": {0: "batch"}, "scores": {0: "batch"}},
opset_version=17,
dynamo=False,
)
quantized = {
"model_quint8.onnx": ["MatMul", "Gather"],
"model_matmul_quint8.onnx": ["MatMul"],
}
for name, operators in quantized.items():
quantize_dynamic(float_path, output / name, weight_type=QuantType.QUInt8, op_types_to_quantize=operators)
sampler_cpu = sampler.to("cpu")
left, right, candidates, label, _ = sampler_cpu.batch(4096, 0.5)
inputs = {"left": left.numpy().astype(np.int64), "right": right.numpy().astype(np.int64), "candidates": candidates.numpy().astype(np.int64)}
with torch.no_grad():
reference = model(left, right, candidates).numpy()
def run(path: Path) -> np.ndarray:
return ort.InferenceSession(str(path), providers=["CPUExecutionProvider"]).run(None, inputs)[0]
def margins(scores: np.ndarray) -> np.ndarray:
scores = np.where(inputs["candidates"] != PAD, scores, -1e4)
return scores[:, 1:].max(1) - scores[:, 0]
report = {"float_vs_torch_max_difference": float(np.abs(run(float_path) - reference).max())}
report["float_mb"] = round(float_path.stat().st_size / 1e6, 2)
for name in quantized:
scores = run(output / name)
report[name] = {
"mb": round((output / name).stat().st_size / 1e6, 2),
"pick_agreement": round(float((scores.argmax(1) == reference.argmax(1)).mean()), 4),
"mean_margin_difference": round(float(np.abs(margins(scores) - margins(reference)).mean()), 3),
}
return report
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--repo", default="cfahlgren1/tinyfix-lab")
parser.add_argument("--run-name", required=True)
parser.add_argument("--groups", choices=sorted(GROUP_SETS), default="shipped")
parser.add_argument("--min-typed", type=int, default=MIN_TYPED_LENGTH)
parser.add_argument("--short-listed", action="store_true", help="Also train two-letter typed words that have listed mix-ups.")
parser.add_argument("--real-share", type=float, default=REAL_MISTAKE_SHARE)
parser.add_argument("--group-share", type=float, default=0.0, help="Share of cases centered on a word from a mix-up group.")
parser.add_argument("--group-power", type=float, default=0.5)
parser.add_argument("--inflection-weight", type=float, default=1.0)
parser.add_argument("--short-share", type=float, default=0.2)
parser.add_argument("--corrupt-rate", type=float, default=0.5)
parser.add_argument("--force-group-words", action="store_true")
parser.add_argument("--slip-words", type=int, default=0)
parser.add_argument("--real-mistakes", action="store_true", help="Also train on GitHub word swaps. Scored worse than homophones alone in v3.")
parser.add_argument("--fineweb-tokens", type=int, default=70_000_000)
parser.add_argument("--smoltalk-tokens", type=int, default=50_000_000)
parser.add_argument("--vocabulary", type=int, default=20_000)
parser.add_argument("--embedding", type=int, default=64)
parser.add_argument("--hidden", type=int, default=256)
parser.add_argument("--steps", type=int, default=60_000)
parser.add_argument("--batch", type=int, default=4096)
parser.add_argument("--learning-rate", type=float, default=2e-3)
parser.add_argument("--output", default="/tmp/tinyfix-realword")
parser.add_argument("--no-upload", action="store_true")
parser.add_argument("--pairs-file", default="", help="Path in --repo of a mix-up list to train on instead of --groups.")
parser.add_argument("--seed", type=int, default=0)
args = parser.parse_args()
torch.manual_seed(args.seed)
device = "cuda" if torch.cuda.is_available() else "cpu"
output = Path(args.output)
output.mkdir(parents=True, exist_ok=True)
print(f"device={device} args={vars(args)}", flush=True)
api = HfApi(token=os.environ["HF_TOKEN"])
if args.pairs_file:
PAIRS_FILE.extend(load_pairs_file(api, args.repo, args.pairs_file))
print(f"pairs from {args.pairs_file}: {len(PAIRS_FILE)}", flush=True)
documents, raw_ids = load_corpus_cached({"fineweb": args.fineweb_tokens, "smoltalk": args.smoltalk_tokens, "oasst": 10**12}, api)
pairs = group_pairs(args.groups)
forced = list(dict.fromkeys(word for pair in pairs for word in pair)) if args.force_group_words else []
if args.slip_words:
forced += [word for word in SLIP_WORDS if word not in forced]
vocabulary, lookup = build_vocabulary(documents, raw_ids, args.vocabulary, forced, args.slip_words)
print(f"documents={len(documents):,} raw_types={len(raw_ids):,} vocabulary={len(vocabulary):,}", flush=True)
order = np.random.default_rng(7).permutation(len(documents))
held_out = max(1, len(documents) // 20)
train_tokens = torch.from_numpy(lookup[np.concatenate([documents[index] for index in order[held_out:]])]).to(device).long()
validation_tokens = torch.from_numpy(lookup[np.concatenate([documents[index] for index in order[:held_out]])]).to(device).long()
print(f"train={len(train_tokens):,} validation={len(validation_tokens):,} unknown={(train_tokens == UNK).float().mean().item():.3f}", flush=True)
del documents
min_typed = 2 if args.short_listed else args.min_typed
confusions = build_confusions(vocabulary, args.real_mistakes, args.groups, min_typed)
(output / "confusions.txt").write_text("".join(f"{typed} {' '.join(expected for expected, _ in options)}\n" for typed, options in confusions.items()))
neighbors = build_neighbors(vocabulary, confusions)
real_typed, real_weights = build_real_mistake_tables(vocabulary, confusions)
print(f"words with neighbors={(neighbors[:, 0] != PAD).sum():,}", flush=True)
(output / "vocab.txt").write_text("\n".join(vocabulary) + "\n")
token_to_id = {token: index for index, token in enumerate(vocabulary)}
options = {
"min_typed": args.min_typed, "short_listed": args.short_listed, "inflection_weight": args.inflection_weight,
"group_share": args.group_share, "group_power": args.group_power, "real_share": args.real_share, "short_share": args.short_share,
"group_ids": {token_to_id[word] for pair in pairs for word in pair if word in token_to_id},
}
train_sampler = Sampler(train_tokens, vocabulary, neighbors, real_typed, real_weights, device, options)
validation_sampler = Sampler(validation_tokens, vocabulary, neighbors, real_typed, real_weights, device, {**options, "group_share": 0.0})
model = WrongWordModel(len(vocabulary), args.embedding, args.hidden).to(device)
parameters = sum(parameter.numel() for parameter in model.parameters())
print(f"parameters={parameters:,}", flush=True)
optimizer = torch.optim.AdamW(model.parameters(), lr=args.learning_rate, weight_decay=1e-6)
schedule = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr=args.learning_rate, total_steps=args.steps, pct_start=0.1)
started = time.time()
running_loss = 0.0
report_every = max(args.steps // 40, 1)
for step in range(1, args.steps + 1):
left, right, candidates, label, _ = train_sampler.batch(args.batch, args.corrupt_rate)
loss = F.cross_entropy(mask_empty(model(left, right, candidates), candidates), label)
optimizer.zero_grad(set_to_none=True)
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
optimizer.step()
schedule.step()
running_loss += loss.item()
if step % report_every == 0:
print(f"step {step}/{args.steps} loss={running_loss / report_every:.4f} elapsed={time.time() - started:.0f}s", flush=True)
running_loss = 0.0
if step % (args.steps // 5 or 1) == 0:
print(f"validation step {step}: {evaluate(model, validation_sampler, 60, args.batch)}", flush=True)
validation = evaluate(model, validation_sampler, 200, args.batch)
print(f"final validation: {validation}", flush=True)
torch.save(model.state_dict(), output / "model.pt")
export_report = export_onnx(model, output, validation_sampler)
print(f"export: {export_report}", flush=True)
metrics = {
"args": vars(args),
"parameters": parameters,
"validation": validation,
"export": export_report,
"train_tokens": len(train_tokens),
"train_minutes": round((time.time() - started) / 60, 1),
}
(output / "metrics.json").write_text(json.dumps(metrics, indent=2) + "\n")
print(json.dumps(metrics, indent=2), flush=True)
if args.no_upload:
print(f"Skipping upload; artifacts in {output}", flush=True)
return
api.create_repo(args.repo, repo_type="model", private=True, exist_ok=True)
api.upload_folder(folder_path=str(output), repo_id=args.repo, path_in_repo=f"combine/{args.run_name}", repo_type="model")
print(f"Uploaded to {args.repo}/combine/{args.run_name}", flush=True)
if __name__ == "__main__":
main()