| | """
|
| | Generate realistic typo-based misspellings from words.txt → misspellings.txt
|
| |
|
| | Colab version
|
| | Place words.txt in /content/ before running
|
| | """
|
| |
|
| | import os
|
| | import time
|
| |
|
| |
|
| |
|
| |
|
| |
|
| |
|
| | words_path = '/content/words.txt'
|
| | output_path = '/content/misspellings.txt'
|
| |
|
| | KEYBOARD_NEIGHBORS = {
|
| | 'q': 'wa', 'w': 'qeas', 'e': 'wrds', 'r': 'etfs', 't': 'rygs',
|
| | 'y': 'tuhs', 'u': 'yijs', 'i': 'uoks', 'o': 'ipls', 'p': 'o',
|
| | 'a': 'qwsz', 's': 'awedxz', 'd': 'serfcx', 'f': 'drtgvc',
|
| | 'g': 'ftyhbv', 'h': 'gyujnb', 'j': 'huikmn', 'k': 'jiolm',
|
| | 'l': 'kop', 'z': 'asx', 'x': 'zsdc', 'c': 'xdfv', 'v': 'cfgb',
|
| | 'b': 'vghn', 'n': 'bhjm', 'm': 'njk',
|
| | }
|
| |
|
| | def generate_adjacent_swaps(word):
|
| | typos = []
|
| | for i in range(len(word) - 1):
|
| | chars = list(word)
|
| | chars[i], chars[i + 1] = chars[i + 1], chars[i]
|
| | typo = ''.join(chars)
|
| | if typo != word:
|
| | typos.append(typo)
|
| | return typos
|
| |
|
| | def generate_deletions(word):
|
| | typos = []
|
| | for i in range(len(word)):
|
| | typo = word[:i] + word[i + 1:]
|
| | if len(typo) >= 2:
|
| | typos.append(typo)
|
| | return typos
|
| |
|
| | def generate_duplications(word):
|
| | typos = []
|
| | for i in range(len(word)):
|
| | typo = word[:i] + word[i] + word[i:]
|
| | if typo != word:
|
| | typos.append(typo)
|
| | return typos
|
| |
|
| | def generate_nearby_key_subs(word):
|
| | typos = []
|
| | lower = word.lower()
|
| | for i in range(len(word)):
|
| | ch = lower[i]
|
| | if ch in KEYBOARD_NEIGHBORS:
|
| | for neighbor in KEYBOARD_NEIGHBORS[ch]:
|
| | typo = lower[:i] + neighbor + lower[i + 1:]
|
| | if typo != lower:
|
| | typos.append(typo)
|
| | return typos
|
| |
|
| | def generate_all_typos(word):
|
| | typos = set()
|
| | typos.update(generate_adjacent_swaps(word))
|
| | typos.update(generate_deletions(word))
|
| | typos.update(generate_duplications(word))
|
| | typos.update(generate_nearby_key_subs(word))
|
| | typos.discard(word)
|
| | typos.discard(word.lower())
|
| | return typos
|
| |
|
| | def is_pure_alpha(word):
|
| | return word.isalpha()
|
| |
|
| |
|
| | if not os.path.exists(words_path):
|
| | raise FileNotFoundError(f"{words_path} not found. Upload it to /content/ first.")
|
| |
|
| | print(f"Reading words from: {words_path}")
|
| |
|
| | with open(words_path, 'r', encoding='utf-8', errors='replace') as f:
|
| | raw_words = [line.strip() for line in f if line.strip()]
|
| |
|
| | print(f"Total raw entries: {len(raw_words):,}")
|
| |
|
| | words = [w for w in raw_words if is_pure_alpha(w) and len(w) >= 3]
|
| | print(f"Filtered to {len(words):,} alphabetical words (len >= 3)")
|
| |
|
| | start = time.time()
|
| | total_typos = 0
|
| | batch_size = 10_000
|
| |
|
| | print(f"Generating typos → {output_path}")
|
| |
|
| | with open(output_path, 'w', encoding='utf-8', newline='\n') as out:
|
| | out.write("# Auto-generated misspellings database\n")
|
| | out.write("# Format: misspelling=correction\n\n")
|
| |
|
| | for idx, word in enumerate(words):
|
| | correction = word
|
| | typos = generate_all_typos(word.lower())
|
| |
|
| | for typo in sorted(typos):
|
| | out.write(f"{typo}={correction}\n")
|
| | total_typos += 1
|
| |
|
| | if (idx + 1) % batch_size == 0:
|
| | elapsed = time.time() - start
|
| | pct = (idx + 1) / len(words) * 100
|
| | rate = (idx + 1) / elapsed if elapsed > 0 else 0
|
| | print(f"[{pct:5.1f}%] {idx + 1:,}/{len(words):,} words | "
|
| | f"{total_typos:,} typos | {rate:.0f} words/sec")
|
| |
|
| | elapsed = time.time() - start
|
| | file_size_mb = os.path.getsize(output_path) / (1024 * 1024)
|
| |
|
| | print("\n" + "=" * 60)
|
| | print(f"Done in {elapsed:.1f}s")
|
| | print(f"Words processed : {len(words):,}")
|
| | print(f"Typos generated : {total_typos:,}")
|
| | print(f"Output file : {output_path}")
|
| | print(f"File size : {file_size_mb:.1f} MB")
|
| | print("=" * 60) |
