data_preparation/chatbot_data_prep.py

# data preparation for chatbot
from pathlib import Path
import pandas as pd
import re
from tqdm import tqdm
from nltk.tokenize import word_tokenize
import nltk
from wordfreq import word_frequency, zipf_frequency
try:
    nltk.data.find("tokenizers/punkt")
except LookupError:
    nltk.download("punkt")
from nltk.corpus import words

nltk.download('words', quiet=True)
from lingua import Language
from lingua import LanguageDetectorBuilder
from gibberish_detector import detector
tqdm.pandas()
from sklearn.model_selection import train_test_split

# load datasets
data_dir = Path("../datasets")

faq_df = pd.read_csv(data_dir / "raw" / "BankFAQs.csv")
support_df = pd.read_csv(data_dir / "raw" / "customer_support_data.csv")

print("FAQ dataset")
print(faq_df.head())
print("Customer Support dataset")
print(support_df.head())

eng_words = set(word.lower() for word in words.words())

# load the gibberish detection model
Detector = detector.create_from_model('gibberish-detector.model')

stopwords = {
    "ho", "rahi", "hai", "ke", "mein", "raha", "hoon", "kar"
}

# cleaning customer support dataset
def clean_text(text):
    # handle missing values
    if pd.isna(text):
        return ""

    text = text.replace("’", "'").replace("’", "'")
    # text = re.sub(r"[^a-zA-Z0-9'\s-]", ' ', text)
    # text = re.sub(r'\s+', ' ', text).strip()

    # split into phrases by punctuation
    phrases = re.split(r'[.!?,;:/]', text)
    # print(phrases)
    
    clean_phrases = []
    total_removed = 0
    
    for phrase in phrases:
        phrase = phrase.strip()
        if not phrase:
            continue

        tokens = re.findall(r"[a-zA-Z0-9]*[0-9][a-zA-Z][a-zA-Z0-9]*|[a-zA-Z]+(?:'[a-zA-Z]+)*|[0-9]+(?:-[0-9]+)*", phrase)
        
        real_words = []
        removed_words = []
    
        for token in tokens:
            if re.fullmatch(r"[a-zA-Z0-9]*[0-9][a-zA-Z]+", token):
                real_words.append(token)
                continue

            if re.fullmatch(r"[0-9]+(?:-[0-9]+)*", token):
                real_words.append(token)
                continue

            token_lower = token.lower()

            if token_lower in stopwords:
                removed_words.append(token)
                continue

            if Detector.is_gibberish(token_lower):
                # print(token)
                removed_words.append(token)
                continue

            if token_lower in {"a", "i"}:
                real_words.append(token)
                continue
            
            # contractions - keep directly without English check
            if "'" in token_lower:
                real_words.append(token)
                continue

            if token_lower in eng_words or zipf_frequency(token_lower, "en") >= 2.0:
                real_words.append(token)
                continue

            removed_words.append(token)

        # print("Real Words: ", real_words)
        # print(f"  Removed words : {removed_words} ({len(removed_words)} removed)")

        if not real_words:
            continue

        # keep phrases when at least a reasonable portion of tokens are English-like
        # if len(real_words) / len(tokens) < 0.25:
        #     continue

        if len(removed_words) >= len(real_words):
            continue
        
        total_removed += len(removed_words)
        clean_phrases.append(" ".join(real_words))

    # print(f"\nTotal words removed: {total_removed}")
    return " ".join(clean_phrases)

# cleaned_support_df = support_df[support_df["language"] == "en"][["conv_id", "turn_index", "role", "text", "industry", "product", "outcome", "issue_type", "overall_urgency"]].copy()
# cleaned_support_df["text"] = cleaned_support_df["text"].progress_apply(clean_text)

# cleaned_support_df = cleaned_support_df[cleaned_support_df["text"].str.strip() != ""]
# cleaned_support_df = cleaned_support_df.dropna(subset=["text"])
# cleaned_support_df.to_csv(data_dir / "processed" / "customer_support" / "cleaned_support_text.csv", index=False)



# row_text = support_df.loc[support_df["language"] == "en", "text"].iloc[4]

# cleaned_text = clean_text(row_text)
# print(row_text)
# print(cleaned_text)


# cleaning bank faqs
faq_df = faq_df.drop_duplicates(subset=["Question", "Answer"])

def clean_text_faq(text):
    if not isinstance(text, str):
        return text
    text = re.sub(r"[\r\n\t]+", " ", text)             # normalise line breaks
    text = re.sub(r"[^\x00-\x7F]+", "", text)
    text = re.sub(r"<<\s*>>", "", text)  
    text = re.sub(r" {2,}", " ", text)                 # collapse spaces
    return text.strip()

for col in ["Question", "Answer"]:
    faq_df[col] = faq_df[col].progress_apply(clean_text_faq)
faq_df.to_csv(data_dir / "processed" / "bank_faq" / "cleaned_faq.csv", index=False)

# split dataset for train/test
train_faq_df, test_faq_df = train_test_split(
    faq_df,
    test_size=0.2,
    stratify=faq_df["Class"],
    random_state=42
)

print(len(train_faq_df))
print(len(test_faq_df))

train_faq_df.to_csv(data_dir / "processed" / "bank_faq" / "train_faq.csv", index=False)
test_faq_df.to_csv(data_dir / "processed" / "bank_faq" / "test_faq.csv", index=False)