cfahlgren1 HF Staff
retrain on more mix-up groups: fixes about 40% of wrong words on the new test bench, up from 24%
f4cab9b verified Download train_realword.py from paperweight/tinyfix: direct link, hf CLI and curl.
- Browser
- Download file 37.1 kB
-
https://huggingface.co/paperweight/tinyfix/resolve/main/train_realword.py
- Command line
-
hf download hf://paperweight/tinyfix/train_realword.py
-
curl -L -o train_realword.py https://huggingface.co/paperweight/tinyfix/resolve/main/train_realword.py
37.1 kB
| # /// script | |
| # requires-python = ">=3.11,<3.14" | |
| # dependencies = [ | |
| # "torch>=2.6,<2.8", | |
| # "datasets>=3.0", | |
| # "numpy<3", | |
| # "onnx>=1.17", | |
| # "onnxruntime>=1.20", | |
| # "huggingface_hub>=0.30", | |
| # ] | |
| # /// | |
| """Train a small model that spots correctly spelled wrong words, like "don" typed for "do".""" | |
| import argparse | |
| import hashlib | |
| import json | |
| import os | |
| import re | |
| import time | |
| import urllib.request | |
| from collections import Counter, defaultdict | |
| from pathlib import Path | |
| import numpy as np | |
| import onnxruntime as ort | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| from datasets import load_dataset | |
| from huggingface_hub import HfApi | |
| from onnxruntime.quantization import QuantType, quantize_dynamic | |
| LEFT_WINDOW = 16 | |
| RIGHT_WINDOW = 4 | |
| MAX_NEIGHBORS = 8 | |
| CANDIDATES = MAX_NEIGHBORS + 1 | |
| MIN_TYPED_LENGTH = 3 | |
| PAD, UNK, NUMBER, START, NAME = 0, 1, 2, 3, 4 | |
| SPECIAL_TOKENS = ["<pad>", "<unk>", "<num>", "<s>", "<name>"] | |
| REAL_MISTAKE_SHARE = 0.6 | |
| HOMOPHONE_WEIGHT = 20 | |
| MIN_REAL_MISTAKE_COUNT = 2 | |
| HOMOPHONES = [ | |
| ("their", "there", "they're"), ("your", "you're"), ("its", "it's"), ("too", "two"), ("then", "than"), | |
| ("were", "where", "we're"), ("whose", "who's"), ("hear", "here"), ("buy", "bye"), ("weather", "whether"), | |
| ("peace", "piece"), ("affect", "effect"), ("lose", "loose"), ("accept", "except"), ("advice", "advise"), | |
| ("quiet", "quite"), ("through", "though", "thorough"), ("passed", "past"), ("principal", "principle"), | |
| ("complement", "compliment"), ("brake", "break"), ("waist", "waste"), ("right", "write"), ("sight", "site", "cite"), | |
| ("wear", "where"), ("led", "lead"), ("meet", "meat"), ("week", "weak"), ("wood", "would"), ("hole", "whole"), | |
| ("male", "mail"), ("sale", "sell"), ("role", "roll"), ("aloud", "allowed"), ("board", "bored"), ("fair", "fare"), | |
| ] | |
| TYPO_PAIRS_DATASET = "chirunder/github_typo_corrections" | |
| # Sound-alikes and classic mix-ups. Every member can be typed for every other member. | |
| EXTENDED_GROUPS = HOMOPHONES + [ | |
| ("to", "too", "two"), ("of", "off"), ("no", "know", "now"), ("by", "buy", "bye"), ("for", "four"), ("one", "won"), | |
| ("new", "knew"), ("wait", "weight"), ("way", "weigh"), ("plane", "plain"), ("made", "maid"), ("pair", "pear"), | |
| ("tale", "tail"), ("flour", "flower"), ("threw", "through"), ("son", "sun"), ("bought", "brought"), ("wear", "were"), | |
| ("bare", "bear"), ("dear", "deer"), ("hour", "our"), ("knight", "night"), ("knot", "not"), ("sea", "see"), | |
| ("steal", "steel"), ("stair", "stare"), ("tide", "tied"), ("which", "witch"), ("course", "coarse"), ("sent", "scent", "cent"), | |
| ("die", "dye"), ("flew", "flu"), ("great", "grate"), ("heal", "heel"), ("heard", "herd"), ("lessen", "lesson"), | |
| ("main", "mane"), ("medal", "meddle"), ("missed", "mist"), ("morning", "mourning"), ("pain", "pane"), ("pause", "paws"), | |
| ("peak", "peek"), ("pole", "poll"), ("poor", "pour", "pore"), ("pray", "prey"), ("rain", "reign", "rein"), ("read", "red"), | |
| ("real", "reel"), ("road", "rode"), ("root", "route"), ("rose", "rows"), ("scene", "seen"), ("seam", "seem"), | |
| ("sole", "soul"), ("some", "sum"), ("stationary", "stationery"), ("throne", "thrown"), ("vain", "vein"), ("waive", "wave"), | |
| ("ate", "eight"), ("blew", "blue"), ("cell", "sell"), ("cereal", "serial"), ("council", "counsel"), ("desert", "dessert"), | |
| ("dual", "duel"), ("fourth", "forth"), ("guessed", "guest"), ("hall", "haul"), ("higher", "hire"), ("loan", "lone"), | |
| ("hours", "ours"), ("presence", "presents"), ("sail", "sale"), ("stake", "steak"), ("breath", "breathe"), ("chose", "choose"), | |
| ("clothes", "cloths"), ("dose", "does"), ("later", "latter"), ("personal", "personnel"), ("precede", "proceed"), | |
| ("quite", "quit"), ("woman", "women"), ("form", "from"), ("lose", "loss"), ("whit", "with"), | |
| ("grown", "groan"), ("idle", "idol"), ("sweet", "suite"), | |
| ("ensure", "insure"), ("emigrate", "immigrate"), | |
| ("we", "wee"), ("so", "sew"), | |
| ] | |
| # Mix-ups that only make sense one way ("could of" for "could have"). | |
| ONE_WAY = [("of", "have"), ("then", "than"), ("loose", "lose"), ("defiantly", "definitely")] | |
| GROUP_SETS = {"shipped": HOMOPHONES, "extended": EXTENDED_GROUPS} | |
| SLIP_WORDS = ["fro", "nit", "grate", "manger", "bye", "whit", "wee", "thee", "lest", "sew"] | |
| WORD_LIST_URL = "https://raw.githubusercontent.com/wolfgarbe/SymSpell/master/SymSpell.FrequencyDictionary/en-80k.txt" | |
| TOKEN = re.compile(r"[A-Za-z]+(?:'[A-Za-z]+)?|\d+(?:[.,]\d+)*|[^\s\w]") | |
| SENTENCE_OPENERS = {".", "!", "?", ":", '"'} | |
| WORD = re.compile(r"^[a-z]+(?:'[a-z]+)?$") | |
| DIGITS = re.compile(r"^\d") | |
| def tokenize(text: str) -> list[str]: | |
| """Lowercases words, except capitalized words mid-sentence, which become a single name token.""" | |
| tokens = [] | |
| for raw in TOKEN.findall(text.replace("’", "'")): | |
| at_sentence_start = not tokens or tokens[-1] in SENTENCE_OPENERS | |
| is_pronoun = raw == "I" or raw.startswith("I'") | |
| tokens.append("<name>" if raw[0].isupper() and not at_sentence_start and not is_pronoun else raw.lower()) | |
| return tokens | |
| def fineweb_documents(): | |
| stream = load_dataset("HuggingFaceFW/fineweb-edu", split="train", streaming=True).shuffle(seed=7, buffer_size=20000) | |
| for row in stream: | |
| yield row["text"] | |
| def smoltalk_documents(): | |
| stream = load_dataset("HuggingFaceTB/smoltalk", "all", split="train", streaming=True).shuffle(seed=7, buffer_size=20000) | |
| for row in stream: | |
| yield "\n".join(message["content"] for message in row["messages"] if message["role"] != "system") | |
| def open_assistant_documents(): | |
| for row in load_dataset("OpenAssistant/oasst1", split="train"): | |
| if row["lang"] == "en": | |
| yield row["text"] | |
| def load_corpus(token_budgets: dict[str, int]) -> tuple[list[np.ndarray], dict[str, int]]: | |
| sources = {"fineweb": fineweb_documents, "smoltalk": smoltalk_documents, "oasst": open_assistant_documents} | |
| raw_ids = {"<s>": 0} | |
| documents = [] | |
| for name, budget in token_budgets.items(): | |
| started, used = time.time(), 0 | |
| for text in sources[name](): | |
| tokens = tokenize(text) | |
| if len(tokens) < 8: | |
| continue | |
| documents.append(np.array([0] + [raw_ids.setdefault(token, len(raw_ids)) for token in tokens], dtype=np.int32)) | |
| used += len(tokens) | |
| if used >= budget: | |
| break | |
| print(f"{name}: {used:,} tokens in {time.time() - started:.0f}s", flush=True) | |
| return documents, raw_ids | |
| CACHE_REPO = "cfahlgren1/tinyfix-lab" | |
| def load_corpus_cached(token_budgets: dict[str, int], api: HfApi) -> tuple[list[np.ndarray], dict[str, int]]: | |
| """Tokenizing 120M tokens takes a while, so the first run saves the result to the private lab repo.""" | |
| folder = f"data/corpus-{token_budgets['fineweb']}-{token_budgets['smoltalk']}" | |
| try: | |
| paths = {name: api.hf_hub_download(CACHE_REPO, f"{folder}/{name}") for name in ("tokens.npy", "lengths.npy", "raw_tokens.json")} | |
| tokens, lengths = np.load(paths["tokens.npy"]), np.load(paths["lengths.npy"]) | |
| raw_tokens = json.loads(Path(paths["raw_tokens.json"]).read_text()) | |
| print(f"loaded cached corpus {folder}: {len(tokens):,} tokens", flush=True) | |
| return np.split(tokens, np.cumsum(lengths)[:-1]), {token: index for index, token in enumerate(raw_tokens)} | |
| except Exception as error: | |
| print(f"no cached corpus ({type(error).__name__}); building it", flush=True) | |
| documents, raw_ids = load_corpus(token_budgets) | |
| cache = Path("/tmp/corpus-cache") | |
| cache.mkdir(exist_ok=True) | |
| np.save(cache / "tokens.npy", np.concatenate(documents)) | |
| np.save(cache / "lengths.npy", np.array([len(document) for document in documents], dtype=np.int64)) | |
| (cache / "raw_tokens.json").write_text(json.dumps(list(raw_ids))) | |
| api.create_repo(CACHE_REPO, repo_type="model", private=True, exist_ok=True) | |
| api.upload_folder(folder_path=str(cache), repo_id=CACHE_REPO, path_in_repo=folder, repo_type="model") | |
| print(f"cached corpus to {CACHE_REPO}/{folder}", flush=True) | |
| return documents, raw_ids | |
| def slip_words(ranked: list[str], counts_by_token: dict[str, int], start: int, how_many: int, min_count: int) -> list[str]: | |
| """Words just outside the vocabulary that are one edit from a common word, so a slip that lands on them can be judged.""" | |
| common = ranked[:3000] | |
| buckets = defaultdict(list) | |
| for word in common: | |
| if len(word) < 3: | |
| continue | |
| for position in range(len(word)): | |
| buckets[word[:position] + word[position + 1 :]].append(word) | |
| buckets[word].append(word) | |
| found = [] | |
| for word in ranked[start:]: | |
| if len(found) >= how_many or counts_by_token[word] < min_count: | |
| break | |
| if len(word) < 3 or "'" in word: | |
| continue | |
| keys = {word} | {word[:position] + word[position + 1 :] for position in range(len(word))} | |
| if any(within_one_edit(word, other) for key in keys for other in buckets.get(key, [])): | |
| found.append(word) | |
| return found | |
| def build_vocabulary(documents: list[np.ndarray], raw_ids: dict[str, int], size: int, forced: list[str], slips: int) -> tuple[list[str], np.ndarray]: | |
| with urllib.request.urlopen(WORD_LIST_URL) as response: | |
| dictionary = {line.split()[0] for line in response.read().decode().splitlines() if line.strip()} | |
| counts = np.bincount(np.concatenate(documents), minlength=len(raw_ids)) | |
| allowed = [(token, identifier) for token, identifier in raw_ids.items() if (WORD.match(token) and token in dictionary) or (len(token) == 1 and not token.isalnum())] | |
| allowed = [pair for pair in allowed if pair[0] != "<s>"] | |
| allowed.sort(key=lambda pair: (-counts[pair[1]], pair[0])) | |
| ranked = [token for token, _ in allowed] | |
| room = size - len(SPECIAL_TOKENS) | |
| extra = [word for word in forced if word in raw_ids and word not in ranked[:room]] | |
| if slips: | |
| counts_by_token = {token: int(counts[identifier]) for token, identifier in allowed} | |
| extra += [word for word in slip_words(ranked, counts_by_token, room, slips, 20) if word not in extra] | |
| missing = [word for word in forced if word not in raw_ids] | |
| print(f"vocabulary: forcing {len(extra)} extra words, e.g. {extra[:40]}; not in corpus: {missing}", flush=True) | |
| vocabulary = SPECIAL_TOKENS + ranked[: room - len(extra)] + extra | |
| token_to_id = {token: index for index, token in enumerate(vocabulary)} | |
| lookup = np.full(len(raw_ids), UNK, dtype=np.int32) | |
| for token, identifier in raw_ids.items(): | |
| if token == "<s>": | |
| lookup[identifier] = START | |
| elif token == "<name>": | |
| lookup[identifier] = NAME | |
| elif DIGITS.match(token): | |
| lookup[identifier] = NUMBER | |
| elif token in token_to_id: | |
| lookup[identifier] = token_to_id[token] | |
| return vocabulary, lookup | |
| def is_target_word(token: str) -> bool: | |
| return len(token) >= 2 and token not in SPECIAL_TOKENS and WORD.match(token) is not None | |
| def within_one_edit(first: str, second: str) -> bool: | |
| if first == second or abs(len(first) - len(second)) > 1: | |
| return False | |
| if len(first) == len(second): | |
| differences = [index for index, (a, b) in enumerate(zip(first, second)) if a != b] | |
| if len(differences) == 1: | |
| return True | |
| return len(differences) == 2 and differences[1] == differences[0] + 1 and first[differences[0]] == second[differences[1]] and first[differences[1]] == second[differences[0]] | |
| shorter, longer = sorted((first, second), key=len) | |
| return any(longer[:index] + longer[index + 1 :] == shorter for index in range(len(longer))) | |
| def strip_punctuation(token: str) -> str: | |
| return token.strip("\"'()[]{}<>.,;:!?*_`#-=/\\") | |
| def real_mistakes_for_training(vocabulary: set[str]) -> Counter: | |
| """Counts real typed-for-expected word swaps, using only the half of the dev split that eval doesn't tune on.""" | |
| counts = Counter() | |
| for row in load_dataset(TYPO_PAIRS_DATASET, split="train"): | |
| typed_tokens, fixed_tokens = (row["text"] or "").split(), (row["correction"] or "").split() | |
| if len(typed_tokens) != len(fixed_tokens): | |
| continue | |
| different = [index for index, (first, second) in enumerate(zip(typed_tokens, fixed_tokens)) if first != second] | |
| if len(different) != 1: | |
| continue | |
| typed, expected = strip_punctuation(typed_tokens[different[0]]), strip_punctuation(fixed_tokens[different[0]]) | |
| digest = int(hashlib.sha1(f"{typed}|{expected}".lower().encode()).hexdigest(), 16) | |
| if digest % 2 != 0 or (digest >> 1) % 2 != 0: | |
| continue | |
| typed, expected = typed.lower(), expected.lower() | |
| if typed != expected and len(typed) >= MIN_TYPED_LENGTH and typed in vocabulary and expected in vocabulary and is_target_word(typed) and is_target_word(expected): | |
| counts[(typed, expected)] += 1 | |
| return Counter({pair: count for pair, count in counts.items() if count >= MIN_REAL_MISTAKE_COUNT}) | |
| PAIRS_FILE: list[tuple[str, str]] = [] | |
| def load_pairs_file(api: HfApi, repo: str, path: str) -> list[tuple[str, str]]: | |
| """Reads a mix-up list ("typed meant1 meant2 ...") from the lab repo, so training and the engine use the same pairs.""" | |
| text = Path(api.hf_hub_download(repo, path)).read_text() | |
| pairs = [(parts[0], meant) for parts in (line.split() for line in text.splitlines()) if parts for meant in parts[1:]] | |
| return list(dict.fromkeys(pairs)) | |
| def group_pairs(groups: str) -> list[tuple[str, str]]: | |
| if PAIRS_FILE: | |
| return PAIRS_FILE | |
| pairs = [(typed, expected) for group in GROUP_SETS[groups] for typed in group for expected in group if typed != expected] | |
| if groups != "shipped": | |
| pairs += ONE_WAY | |
| return list(dict.fromkeys(pairs)) | |
| def build_confusions(vocabulary: list[str], use_real_mistakes: bool, groups: str, min_typed: int) -> dict[str, list[tuple[str, int]]]: | |
| """Maps a typed word to words people really meant by it, most likely first.""" | |
| known = set(vocabulary) | |
| counts = real_mistakes_for_training(known) if use_real_mistakes else Counter() | |
| print(f"real mistake pairs={len(counts):,}", flush=True) | |
| for typed, expected in group_pairs(groups): | |
| if typed in known and expected in known and len(typed) >= min_typed: | |
| counts[(typed, expected)] += HOMOPHONE_WEIGHT | |
| confusions = defaultdict(list) | |
| for (typed, expected), count in sorted(counts.items(), key=lambda item: (-item[1], item[0])): | |
| confusions[typed].append((expected, count)) | |
| return {typed: options[:MAX_NEIGHBORS] for typed, options in confusions.items()} | |
| def build_real_mistake_tables(vocabulary: list[str], confusions: dict[str, list[tuple[str, int]]]) -> tuple[np.ndarray, np.ndarray]: | |
| """For each intended word, lists the words typed by mistake and how often, for sampling.""" | |
| token_to_id = {token: index for index, token in enumerate(vocabulary)} | |
| mistakes = defaultdict(list) | |
| for typed, options in confusions.items(): | |
| for expected, count in options: | |
| mistakes[token_to_id[expected]].append((token_to_id[typed], count)) | |
| typed_table = np.zeros((len(vocabulary), MAX_NEIGHBORS), dtype=np.int32) | |
| weights = np.zeros((len(vocabulary), MAX_NEIGHBORS), dtype=np.float32) | |
| for expected, options in mistakes.items(): | |
| options = sorted(options, key=lambda option: -option[1])[:MAX_NEIGHBORS] | |
| typed_table[expected, : len(options)] = [typed for typed, _ in options] | |
| weights[expected, : len(options)] = [count**0.5 for _, count in options] | |
| return typed_table, weights | |
| def build_neighbors(vocabulary: list[str], confusions: dict[str, list[tuple[str, int]]]) -> np.ndarray: | |
| """Lists known mix-ups first, then the most common words one edit away, the same order the browser uses.""" | |
| token_to_id = {token: index for index, token in enumerate(vocabulary)} | |
| buckets = defaultdict(set) | |
| for index, token in enumerate(vocabulary): | |
| if not is_target_word(token): | |
| continue | |
| buckets[token].add(index) | |
| for position in range(len(token)): | |
| buckets[token[:position] + token[position + 1 :]].add(index) | |
| table = np.zeros((len(vocabulary), MAX_NEIGHBORS), dtype=np.int32) | |
| for index, token in enumerate(vocabulary): | |
| if not is_target_word(token): | |
| continue | |
| nearby = set(buckets[token]) | |
| for position in range(len(token)): | |
| nearby |= buckets[token[:position] + token[position + 1 :]] | |
| one_edit = sorted(other for other in nearby if within_one_edit(token, vocabulary[other])) | |
| known_mixups = [token_to_id[expected] for expected, _ in confusions.get(token, [])] | |
| found = list(dict.fromkeys(known_mixups + one_edit)) | |
| table[index, : min(len(found), MAX_NEIGHBORS)] = found[:MAX_NEIGHBORS] | |
| return table | |
| class MatMulLinear(nn.Module): | |
| """Linear layer that exports as MatMul so ONNX Runtime dynamic quantization applies.""" | |
| def __init__(self, inputs: int, outputs: int): | |
| super().__init__() | |
| self.weight = nn.Parameter(nn.init.xavier_uniform_(torch.empty(inputs, outputs))) | |
| self.bias = nn.Parameter(torch.zeros(outputs)) | |
| def forward(self, values): | |
| return values.matmul(self.weight) + self.bias | |
| class Gru(nn.Module): | |
| """GRU built from MatMul layers because ONNX Runtime cannot quantize its GRU operator.""" | |
| def __init__(self, inputs: int, hidden: int): | |
| super().__init__() | |
| self.hidden = hidden | |
| self.reset_gate = MatMulLinear(inputs + hidden, hidden) | |
| self.update_gate = MatMulLinear(inputs + hidden, hidden) | |
| self.proposal_input = MatMulLinear(inputs, hidden) | |
| self.proposal_state = MatMulLinear(hidden, hidden) | |
| def forward(self, sequence): | |
| state = sequence[:, 0, :].sum(-1, keepdim=True).mul(0).repeat(1, self.hidden) | |
| for step in range(sequence.shape[1]): | |
| current = sequence[:, step, :] | |
| joined = torch.cat([current, state], dim=-1) | |
| reset = torch.sigmoid(self.reset_gate(joined)) | |
| update = torch.sigmoid(self.update_gate(joined)) | |
| proposal = torch.tanh(self.proposal_input(current) + reset * self.proposal_state(state)) | |
| state = (1 - update) * proposal + update * state | |
| return state | |
| class WrongWordModel(nn.Module): | |
| """Scores the typed word and its neighbors against the words on both sides.""" | |
| def __init__(self, vocabulary_size: int, embedding: int, hidden: int): | |
| super().__init__() | |
| self.embedding = nn.Embedding(vocabulary_size, embedding, padding_idx=PAD) | |
| self.word_bias = nn.Embedding(vocabulary_size, 1, padding_idx=PAD) | |
| self.left = Gru(embedding, hidden) | |
| self.right = Gru(embedding, hidden) | |
| self.mix = MatMulLinear(hidden * 2, hidden) | |
| self.project = MatMulLinear(hidden, embedding) | |
| self.typed_bonus = nn.Parameter(torch.tensor(1.0)) | |
| self.register_buffer("typed_position", F.one_hot(torch.tensor(0), CANDIDATES).float()) | |
| def forward(self, left, right, candidates): | |
| context = torch.cat([self.left(self.embedding(left)), self.right(self.embedding(right))], dim=-1) | |
| summary = self.project(torch.tanh(self.mix(context))) | |
| scores = (self.embedding(candidates) * summary.unsqueeze(1)).sum(-1) + self.word_bias(candidates).squeeze(-1) | |
| return scores + self.typed_bonus * self.typed_position | |
| INFLECTION_ENDINGS = {"", "s", "es", "d", "ed", "ing", "r", "er", "ly"} | |
| def is_inflection(first: str, second: str) -> bool: | |
| """True for pairs like friendship/friendships and continued/continues that only differ in an ending.""" | |
| shared = 0 | |
| while shared < min(len(first), len(second)) and first[shared] == second[shared]: | |
| shared += 1 | |
| return shared >= 3 and first[shared:] in INFLECTION_ENDINGS and second[shared:] in INFLECTION_ENDINGS | |
| class Sampler: | |
| """Draws training cases by swapping real words for nearby real words.""" | |
| def __init__(self, tokens: torch.Tensor, vocabulary: list[str], neighbors: np.ndarray, real_typed: np.ndarray, real_weights: np.ndarray, device: str, options: dict): | |
| self.init_args = (vocabulary, neighbors, real_typed, real_weights, options) | |
| self.tokens = tokens | |
| self.device = device | |
| self.vocabulary = vocabulary | |
| self.options = options | |
| self.real_typed = torch.from_numpy(real_typed).to(device).long() | |
| self.real_weights = torch.from_numpy(real_weights).to(device) | |
| self.has_real = self.real_weights.sum(1) > 0 | |
| self.neighbors = torch.from_numpy(neighbors).to(device).long() | |
| self.neighbor_counts = (self.neighbors != PAD).sum(1) | |
| neighbor_weights = (neighbors != PAD).astype(np.float32) | |
| if options["inflection_weight"] != 1: | |
| for row in np.nonzero(neighbors[:, 0] != PAD)[0]: | |
| for column in range(MAX_NEIGHBORS): | |
| other = neighbors[row, column] | |
| if other != PAD and is_inflection(vocabulary[row], vocabulary[other]): | |
| neighbor_weights[row, column] = options["inflection_weight"] | |
| neighbor_weights[neighbor_weights.sum(1) == 0] = 1 | |
| self.neighbor_weights = torch.from_numpy(neighbor_weights).to(device) | |
| lengths = torch.tensor([len(token) for token in vocabulary], device=device) | |
| typed_ok = lengths >= options["min_typed"] | |
| if options["short_listed"]: | |
| typed_ok |= (lengths >= 2) & self.has_listed(real_typed, len(vocabulary), device) | |
| self.typed_ok = typed_ok | |
| self.is_target = torch.tensor([is_target_word(token) for token in vocabulary], device=device) | |
| self.group_share = options["group_share"] | |
| if self.group_share > 0: | |
| group_ids = torch.tensor(sorted(options["group_ids"]), device=device) | |
| inside = torch.zeros(len(tokens), dtype=torch.bool, device=device) | |
| inside[LEFT_WINDOW + 1 : len(tokens) - RIGHT_WINDOW - 1] = True | |
| where = torch.nonzero(torch.isin(tokens, group_ids) & inside).squeeze(1) | |
| order = torch.argsort(tokens[where], stable=True) | |
| self.group_positions = where[order] | |
| type_counts = torch.bincount(tokens[self.group_positions], minlength=len(vocabulary)) | |
| self.group_starts = torch.cumsum(type_counts, 0) - type_counts | |
| self.group_counts = type_counts | |
| self.group_type_weights = type_counts.float().pow(options["group_power"]) | |
| print(f"group positions={len(where):,} types={(type_counts > 0).sum().item()}", flush=True) | |
| def has_listed(real_typed: np.ndarray, size: int, device: str) -> torch.Tensor: | |
| listed = torch.zeros(size, dtype=torch.bool, device=device) | |
| typed = torch.from_numpy(real_typed[real_typed != PAD]).to(device).long() | |
| listed[typed] = True | |
| return listed | |
| def to(self, device: str) -> "Sampler": | |
| return Sampler(self.tokens.to(device), *self.init_args[:4], device, self.init_args[4]) | |
| def positions(self, size: int) -> torch.Tensor: | |
| positions = torch.randint(LEFT_WINDOW + 1, len(self.tokens) - RIGHT_WINDOW - 1, (size,), device=self.device) | |
| if self.group_share <= 0: | |
| return positions | |
| from_groups = torch.rand(size, device=self.device) < self.group_share | |
| types = torch.multinomial(self.group_type_weights, size, replacement=True) | |
| offsets = (torch.rand(size, device=self.device) * self.group_counts[types]).long() | |
| chosen = self.group_positions[(self.group_starts[types] + offsets).clamp(max=len(self.group_positions) - 1)] | |
| return torch.where(from_groups, chosen, positions) | |
| def batch(self, size: int, corrupt_rate: float): | |
| positions = self.positions(size) | |
| target = self.tokens[positions] | |
| counts = self.neighbor_counts[target] | |
| choice = torch.multinomial(self.neighbor_weights[target], 1).squeeze(1) | |
| corrupted = self.neighbors[target, choice] | |
| weights = self.real_weights[target] | |
| weights = weights.where(self.has_real[target].unsqueeze(1), torch.ones_like(weights)) | |
| real_choice = torch.multinomial(weights, 1).squeeze(1) | |
| uses_real = self.has_real[target] & (torch.rand(size, device=self.device) < self.options["real_share"]) | |
| corrupted = torch.where(uses_real, self.real_typed[target, real_choice], corrupted) | |
| counts = counts.where(~uses_real, torch.ones_like(counts)) | |
| is_corrupted = (torch.rand(size, device=self.device) < corrupt_rate) & (counts > 0) | |
| typed = torch.where(is_corrupted, corrupted, target) | |
| candidates = torch.cat([typed.unsqueeze(1), self.neighbors[typed]], dim=1) | |
| matches = candidates == target.unsqueeze(1) | |
| valid = self.is_target[target] & self.typed_ok[typed] & (self.neighbor_counts[typed] > 0) & matches.any(1) | |
| positions, candidates, matches, is_corrupted = positions[valid], candidates[valid], matches[valid], is_corrupted[valid] | |
| count = len(positions) | |
| left_offsets = torch.arange(-LEFT_WINDOW, 0, device=self.device) | |
| left = self.tokens[positions.unsqueeze(1) + left_offsets] | |
| right = self.tokens[positions.unsqueeze(1) + torch.arange(1, RIGHT_WINDOW + 1, device=self.device)] | |
| visible_right = torch.randint(0, RIGHT_WINDOW + 1, (count,), device=self.device).unsqueeze(1) | |
| right = right.where(torch.arange(RIGHT_WINDOW, device=self.device).unsqueeze(0) < visible_right, torch.tensor(PAD, device=self.device)).flip(1) | |
| is_short = torch.rand(count, device=self.device) < self.options["short_share"] | |
| kept = torch.randint(0, LEFT_WINDOW, (count,), device=self.device).unsqueeze(1) | |
| distance_from_end = (-left_offsets).unsqueeze(0) | |
| hidden_left = is_short.unsqueeze(1) & (distance_from_end > kept + 1) | |
| marks_start = is_short.unsqueeze(1) & (distance_from_end == kept + 1) | |
| left = left.where(~hidden_left, torch.tensor(PAD, device=self.device)).where(~marks_start, torch.tensor(START, device=self.device)) | |
| return left, right, candidates, matches.long().argmax(1), is_corrupted | |
| def mask_empty(scores, candidates): | |
| return scores.where(candidates != PAD, torch.tensor(-1e4, device=scores.device)) | |
| def evaluate(model, sampler, batches: int, batch_size: int) -> dict: | |
| model.eval() | |
| clean_margins, corrupted_margins, corrupted_right = [], [], [] | |
| for _ in range(batches): | |
| left, right, candidates, label, is_corrupted = sampler.batch(batch_size, 0.5) | |
| scores = mask_empty(model(left, right, candidates), candidates) | |
| best = scores[:, 1:].max(1) | |
| margin = best.values - scores[:, 0] | |
| clean_margins.append(margin[~is_corrupted]) | |
| corrupted_margins.append(margin[is_corrupted]) | |
| corrupted_right.append(best.indices[is_corrupted] + 1 == label[is_corrupted]) | |
| model.train() | |
| clean = torch.cat(clean_margins) | |
| margin = torch.cat(corrupted_margins) | |
| right = torch.cat(corrupted_right) | |
| results = {"clean_cases": len(clean), "corrupted_cases": len(margin)} | |
| for false_rate in (0.005, 0.001, 0.0005): | |
| threshold = torch.quantile(clean[:200000].float(), 1 - false_rate).item() | |
| results[f"caught_at_false_{false_rate}"] = round(((margin >= threshold) & right).float().mean().item(), 4) | |
| results[f"threshold_at_false_{false_rate}"] = round(threshold, 2) | |
| return results | |
| def export_onnx(model: WrongWordModel, output: Path, sampler: Sampler) -> dict: | |
| model = model.cpu().eval() | |
| float_path = output / "model.onnx" | |
| example = (torch.ones(2, LEFT_WINDOW, dtype=torch.long), torch.ones(2, RIGHT_WINDOW, dtype=torch.long), torch.ones(2, CANDIDATES, dtype=torch.long)) | |
| torch.onnx.export( | |
| model, | |
| example, | |
| float_path, | |
| input_names=["left", "right", "candidates"], | |
| output_names=["scores"], | |
| dynamic_axes={"left": {0: "batch"}, "right": {0: "batch"}, "candidates": {0: "batch"}, "scores": {0: "batch"}}, | |
| opset_version=17, | |
| dynamo=False, | |
| ) | |
| quantized = { | |
| "model_quint8.onnx": ["MatMul", "Gather"], | |
| "model_matmul_quint8.onnx": ["MatMul"], | |
| } | |
| for name, operators in quantized.items(): | |
| quantize_dynamic(float_path, output / name, weight_type=QuantType.QUInt8, op_types_to_quantize=operators) | |
| sampler_cpu = sampler.to("cpu") | |
| left, right, candidates, label, _ = sampler_cpu.batch(4096, 0.5) | |
| inputs = {"left": left.numpy().astype(np.int64), "right": right.numpy().astype(np.int64), "candidates": candidates.numpy().astype(np.int64)} | |
| with torch.no_grad(): | |
| reference = model(left, right, candidates).numpy() | |
| def run(path: Path) -> np.ndarray: | |
| return ort.InferenceSession(str(path), providers=["CPUExecutionProvider"]).run(None, inputs)[0] | |
| def margins(scores: np.ndarray) -> np.ndarray: | |
| scores = np.where(inputs["candidates"] != PAD, scores, -1e4) | |
| return scores[:, 1:].max(1) - scores[:, 0] | |
| report = {"float_vs_torch_max_difference": float(np.abs(run(float_path) - reference).max())} | |
| report["float_mb"] = round(float_path.stat().st_size / 1e6, 2) | |
| for name in quantized: | |
| scores = run(output / name) | |
| report[name] = { | |
| "mb": round((output / name).stat().st_size / 1e6, 2), | |
| "pick_agreement": round(float((scores.argmax(1) == reference.argmax(1)).mean()), 4), | |
| "mean_margin_difference": round(float(np.abs(margins(scores) - margins(reference)).mean()), 3), | |
| } | |
| return report | |
| def main(): | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("--repo", default="cfahlgren1/tinyfix-lab") | |
| parser.add_argument("--run-name", required=True) | |
| parser.add_argument("--groups", choices=sorted(GROUP_SETS), default="shipped") | |
| parser.add_argument("--min-typed", type=int, default=MIN_TYPED_LENGTH) | |
| parser.add_argument("--short-listed", action="store_true", help="Also train two-letter typed words that have listed mix-ups.") | |
| parser.add_argument("--real-share", type=float, default=REAL_MISTAKE_SHARE) | |
| parser.add_argument("--group-share", type=float, default=0.0, help="Share of cases centered on a word from a mix-up group.") | |
| parser.add_argument("--group-power", type=float, default=0.5) | |
| parser.add_argument("--inflection-weight", type=float, default=1.0) | |
| parser.add_argument("--short-share", type=float, default=0.2) | |
| parser.add_argument("--corrupt-rate", type=float, default=0.5) | |
| parser.add_argument("--force-group-words", action="store_true") | |
| parser.add_argument("--slip-words", type=int, default=0) | |
| parser.add_argument("--real-mistakes", action="store_true", help="Also train on GitHub word swaps. Scored worse than homophones alone in v3.") | |
| parser.add_argument("--fineweb-tokens", type=int, default=70_000_000) | |
| parser.add_argument("--smoltalk-tokens", type=int, default=50_000_000) | |
| parser.add_argument("--vocabulary", type=int, default=20_000) | |
| parser.add_argument("--embedding", type=int, default=64) | |
| parser.add_argument("--hidden", type=int, default=256) | |
| parser.add_argument("--steps", type=int, default=60_000) | |
| parser.add_argument("--batch", type=int, default=4096) | |
| parser.add_argument("--learning-rate", type=float, default=2e-3) | |
| parser.add_argument("--output", default="/tmp/tinyfix-realword") | |
| parser.add_argument("--no-upload", action="store_true") | |
| parser.add_argument("--pairs-file", default="", help="Path in --repo of a mix-up list to train on instead of --groups.") | |
| parser.add_argument("--seed", type=int, default=0) | |
| args = parser.parse_args() | |
| torch.manual_seed(args.seed) | |
| device = "cuda" if torch.cuda.is_available() else "cpu" | |
| output = Path(args.output) | |
| output.mkdir(parents=True, exist_ok=True) | |
| print(f"device={device} args={vars(args)}", flush=True) | |
| api = HfApi(token=os.environ["HF_TOKEN"]) | |
| if args.pairs_file: | |
| PAIRS_FILE.extend(load_pairs_file(api, args.repo, args.pairs_file)) | |
| print(f"pairs from {args.pairs_file}: {len(PAIRS_FILE)}", flush=True) | |
| documents, raw_ids = load_corpus_cached({"fineweb": args.fineweb_tokens, "smoltalk": args.smoltalk_tokens, "oasst": 10**12}, api) | |
| pairs = group_pairs(args.groups) | |
| forced = list(dict.fromkeys(word for pair in pairs for word in pair)) if args.force_group_words else [] | |
| if args.slip_words: | |
| forced += [word for word in SLIP_WORDS if word not in forced] | |
| vocabulary, lookup = build_vocabulary(documents, raw_ids, args.vocabulary, forced, args.slip_words) | |
| print(f"documents={len(documents):,} raw_types={len(raw_ids):,} vocabulary={len(vocabulary):,}", flush=True) | |
| order = np.random.default_rng(7).permutation(len(documents)) | |
| held_out = max(1, len(documents) // 20) | |
| train_tokens = torch.from_numpy(lookup[np.concatenate([documents[index] for index in order[held_out:]])]).to(device).long() | |
| validation_tokens = torch.from_numpy(lookup[np.concatenate([documents[index] for index in order[:held_out]])]).to(device).long() | |
| print(f"train={len(train_tokens):,} validation={len(validation_tokens):,} unknown={(train_tokens == UNK).float().mean().item():.3f}", flush=True) | |
| del documents | |
| min_typed = 2 if args.short_listed else args.min_typed | |
| confusions = build_confusions(vocabulary, args.real_mistakes, args.groups, min_typed) | |
| (output / "confusions.txt").write_text("".join(f"{typed} {' '.join(expected for expected, _ in options)}\n" for typed, options in confusions.items())) | |
| neighbors = build_neighbors(vocabulary, confusions) | |
| real_typed, real_weights = build_real_mistake_tables(vocabulary, confusions) | |
| print(f"words with neighbors={(neighbors[:, 0] != PAD).sum():,}", flush=True) | |
| (output / "vocab.txt").write_text("\n".join(vocabulary) + "\n") | |
| token_to_id = {token: index for index, token in enumerate(vocabulary)} | |
| options = { | |
| "min_typed": args.min_typed, "short_listed": args.short_listed, "inflection_weight": args.inflection_weight, | |
| "group_share": args.group_share, "group_power": args.group_power, "real_share": args.real_share, "short_share": args.short_share, | |
| "group_ids": {token_to_id[word] for pair in pairs for word in pair if word in token_to_id}, | |
| } | |
| train_sampler = Sampler(train_tokens, vocabulary, neighbors, real_typed, real_weights, device, options) | |
| validation_sampler = Sampler(validation_tokens, vocabulary, neighbors, real_typed, real_weights, device, {**options, "group_share": 0.0}) | |
| model = WrongWordModel(len(vocabulary), args.embedding, args.hidden).to(device) | |
| parameters = sum(parameter.numel() for parameter in model.parameters()) | |
| print(f"parameters={parameters:,}", flush=True) | |
| optimizer = torch.optim.AdamW(model.parameters(), lr=args.learning_rate, weight_decay=1e-6) | |
| schedule = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr=args.learning_rate, total_steps=args.steps, pct_start=0.1) | |
| started = time.time() | |
| running_loss = 0.0 | |
| report_every = max(args.steps // 40, 1) | |
| for step in range(1, args.steps + 1): | |
| left, right, candidates, label, _ = train_sampler.batch(args.batch, args.corrupt_rate) | |
| loss = F.cross_entropy(mask_empty(model(left, right, candidates), candidates), label) | |
| optimizer.zero_grad(set_to_none=True) | |
| loss.backward() | |
| torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0) | |
| optimizer.step() | |
| schedule.step() | |
| running_loss += loss.item() | |
| if step % report_every == 0: | |
| print(f"step {step}/{args.steps} loss={running_loss / report_every:.4f} elapsed={time.time() - started:.0f}s", flush=True) | |
| running_loss = 0.0 | |
| if step % (args.steps // 5 or 1) == 0: | |
| print(f"validation step {step}: {evaluate(model, validation_sampler, 60, args.batch)}", flush=True) | |
| validation = evaluate(model, validation_sampler, 200, args.batch) | |
| print(f"final validation: {validation}", flush=True) | |
| torch.save(model.state_dict(), output / "model.pt") | |
| export_report = export_onnx(model, output, validation_sampler) | |
| print(f"export: {export_report}", flush=True) | |
| metrics = { | |
| "args": vars(args), | |
| "parameters": parameters, | |
| "validation": validation, | |
| "export": export_report, | |
| "train_tokens": len(train_tokens), | |
| "train_minutes": round((time.time() - started) / 60, 1), | |
| } | |
| (output / "metrics.json").write_text(json.dumps(metrics, indent=2) + "\n") | |
| print(json.dumps(metrics, indent=2), flush=True) | |
| if args.no_upload: | |
| print(f"Skipping upload; artifacts in {output}", flush=True) | |
| return | |
| api.create_repo(args.repo, repo_type="model", private=True, exist_ok=True) | |
| api.upload_folder(folder_path=str(output), repo_id=args.repo, path_in_repo=f"combine/{args.run_name}", repo_type="model") | |
| print(f"Uploaded to {args.repo}/combine/{args.run_name}", flush=True) | |
| if __name__ == "__main__": | |
| main() | |