NRF_LLM / modules /dictionary.py
casamN's picture
Update modules/dictionary.py
d1cef83 verified
Raw
History Blame Contribute Delete
4.76 kB
import os
import pandas as pd
from rapidfuzz import process, fuzz
# ============================================================
# DATA LOCATION
# ============================================================
BASE_DIR = os.path.dirname(
os.path.dirname(
os.path.abspath(__file__)
)
)
DICTIONARY_FILE = os.path.join(
BASE_DIR,
"data",
"kiembu_dictionary.csv"
)
# ============================================================
# LOAD DICTIONARY
# ============================================================
def load_dictionary():
try:
# First attempt: tab-separated
df = pd.read_csv(
DICTIONARY_FILE,
sep="\t",
encoding="utf-8"
)
# If only one column was loaded,
# try comma-separated instead
if len(df.columns) < 2:
df = pd.read_csv(
DICTIONARY_FILE,
sep=",",
encoding="utf-8"
)
df.columns = (
df.columns
.str.strip()
)
df.fillna("", inplace=True)
print("Dictionary loaded successfully")
print("Columns:", df.columns.tolist())
print("Records:", len(df))
return df
except Exception as e:
print(
f"Dictionary loading error: {e}"
)
return pd.DataFrame()
# ============================================================
# GLOBAL DATAFRAME
# ============================================================
dictionary_df = load_dictionary()
# ============================================================
# TRANSLATION FUNCTION
# ============================================================
def translate_word(
word,
direction="English → Kiembu",
df=None
):
if df is None:
df = dictionary_df
if df.empty:
return "Dictionary data unavailable."
if not word:
return "Please enter a word."
query = word.strip()
if not query:
return "Please enter a word."
query_lower = query.lower()
# ========================================================
# DETERMINE DIRECTION
# ========================================================
if direction == "English → Kiembu":
source_col = "English"
target_col = "Kiembu"
else:
source_col = "Kiembu"
target_col = "English"
# ========================================================
# CHECK COLUMNS
# ========================================================
if source_col not in df.columns:
return (
f"Column '{source_col}' "
f"not found in dictionary."
)
if target_col not in df.columns:
return (
f"Column '{target_col}' "
f"not found in dictionary."
)
# ========================================================
# EXACT MATCH
# ========================================================
exact_match = df[
df[source_col]
.astype(str)
.str.strip()
.str.lower()
== query_lower
]
if not exact_match.empty:
source_word = str(
exact_match.iloc[0][source_col]
).strip()
translated_word = str(
exact_match.iloc[0][target_col]
).strip()
return (
f"{source_word}\n\n"
f"➜ {translated_word}"
)
# ========================================================
# FUZZY MATCH
# ========================================================
choices = (
df[source_col]
.astype(str)
.str.strip()
.tolist()
)
best_match = process.extractOne(
query,
choices,
scorer=fuzz.WRatio
)
if not best_match:
return "No translation found."
matched_word = best_match[0]
score = best_match[1]
if score < 75:
return (
"No close translation found.\n\n"
f"Best score: {score}"
)
row = df[
df[source_col]
.astype(str)
.str.strip()
== matched_word
]
if row.empty:
return "No translation found."
translated_word = str(
row.iloc[0][target_col]
).strip()
return (
f"Suggested Match: {matched_word}\n\n"
f"➜ {translated_word}"
)
# ============================================================
# OPTIONAL TEST
# ============================================================
if __name__ == "__main__":
print(
translate_word(
"house",
"English → Kiembu"
)
)
print(
translate_word(
"Nyomba",
"Kiembu → English"
)
)