File size: 4,362 Bytes
55e5ae5 17e84da 55e5ae5 9583edf 55e5ae5 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 | # Import necessary libraries
import string # For lowercase English letters
import re # For regular expression operations
from collections import Counter # For counting word occurrences
import editdistance # For calculating edit distance
import numpy as np # Not used in code (can be removed)
class Autocorrection(object):
def __init__(self, filename):
# Read the file and extract all lowercase words
with open(filename, "r", encoding="UTF-8") as file:
word = []
lines = file.readlines()
for line in lines:
# Extract words using regex and convert to lowercase
word += re.findall(r'\w+', line.lower())
# Create a vocabulary from the unique words
self.vocabulary = set(word)
# Count occurrences of each word
self.counts_of_word = Counter(word)
# Total number of words for probability calculation
self.total_words = float(sum(self.counts_of_word.values()))
# Compute the probability of each word
self.prob_of_word = {
w: self.counts_of_word[w] / self.total_words
for w in self.counts_of_word.keys()
}
def edit1(self, word):
# All letters from a-z
# letter = string.ascii_lowercase
letter = 'abcdefghijklmnopqrstuvwxyzàâçéèêëîïôùûüÿœæ'
# All possible split positions in the word
splits = [(word[:i], word[i:]) for i in range(len(word) + 1)]
# Generate all one-edit-distance words:
# Insertion: insert a letter at any position
insert = [l + c + r for l, r in splits for c in letter]
# Deletion: remove one letter
delete = [l + r[1:] for l, r in splits if r]
# Replacement: change one letter
replace = [l + c + r[1:] for l, r in splits if r for c in letter]
# Transposition (swap): swap adjacent letters
swap = [l + r[1] + r[0] + r[2:] for l, r in splits if len(r) > 1]
# Return all variations as a set to avoid duplicates
return set(replace + insert + delete + swap)
def edit2(self, word):
# Generate all words that are two edits away
return [e2 for e1 in self.edit1(word) for e2 in self.edit1(e1)]
def common_prefix_length(self, word1, word2):
# Find the length of the common prefix between two words
common_len = 0
for i, (c1, c2) in enumerate(zip(word1, word2)):
if c1 == c2:
common_len += 1
else:
break
return common_len
def custom_score(self, suggestion, original_word):
# Weights for different operations
replace_weight = 1
insert_weight = 2
delete_weight = 3
swap_weight = 4
# Get the edit distance between suggestion and original word
distance = editdistance.eval(suggestion, original_word)
# Compute a custom score based on the type of edit
if len(suggestion) == len(original_word): # Likely replace
common_prefix_len = self.common_prefix_length(suggestion, original_word)
score = distance * replace_weight + (len(original_word) - common_prefix_len)
elif len(suggestion) == len(original_word) + 1: # Likely insertion
common_prefix_len = self.common_prefix_length(suggestion, original_word)
score = distance * insert_weight + (len(original_word) - common_prefix_len)
elif len(suggestion) == len(original_word) - 1: # Likely deletion
score = distance * delete_weight
else: # Likely swap or other
score = distance * swap_weight
return score
def correct_spelling(self, word):
# Return if word is already in the vocabulary
if word in self.vocabulary:
print(f"{word} is already correctly spelt")
return
# Get edit distance candidates
suggestions = self.edit1(word) or self.edit2(word) or [word]
# Filter candidates that are real words
best_guesses = [w for w in suggestions if w in self.vocabulary]
# Sort suggestions by custom score
best_guesses.sort(key=lambda w: self.custom_score(w, word))
# Return suggestions with their probabilities
return [(w, self.prob_of_word[w]) for w in best_guesses] |