File size: 5,179 Bytes
b22c324 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 | # data preparation for chatbot
from pathlib import Path
import pandas as pd
import re
from tqdm import tqdm
from nltk.tokenize import word_tokenize
import nltk
from wordfreq import word_frequency, zipf_frequency
try:
nltk.data.find("tokenizers/punkt")
except LookupError:
nltk.download("punkt")
from nltk.corpus import words
nltk.download('words', quiet=True)
from lingua import Language
from lingua import LanguageDetectorBuilder
from gibberish_detector import detector
tqdm.pandas()
from sklearn.model_selection import train_test_split
# load datasets
data_dir = Path("../datasets")
faq_df = pd.read_csv(data_dir / "raw" / "BankFAQs.csv")
support_df = pd.read_csv(data_dir / "raw" / "customer_support_data.csv")
print("FAQ dataset")
print(faq_df.head())
print("Customer Support dataset")
print(support_df.head())
eng_words = set(word.lower() for word in words.words())
# load the gibberish detection model
Detector = detector.create_from_model('gibberish-detector.model')
stopwords = {
"ho", "rahi", "hai", "ke", "mein", "raha", "hoon", "kar"
}
# cleaning customer support dataset
def clean_text(text):
# handle missing values
if pd.isna(text):
return ""
text = text.replace("’", "'").replace("’", "'")
# text = re.sub(r"[^a-zA-Z0-9'\s-]", ' ', text)
# text = re.sub(r'\s+', ' ', text).strip()
# split into phrases by punctuation
phrases = re.split(r'[.!?,;:/]', text)
# print(phrases)
clean_phrases = []
total_removed = 0
for phrase in phrases:
phrase = phrase.strip()
if not phrase:
continue
tokens = re.findall(r"[a-zA-Z0-9]*[0-9][a-zA-Z][a-zA-Z0-9]*|[a-zA-Z]+(?:'[a-zA-Z]+)*|[0-9]+(?:-[0-9]+)*", phrase)
real_words = []
removed_words = []
for token in tokens:
if re.fullmatch(r"[a-zA-Z0-9]*[0-9][a-zA-Z]+", token):
real_words.append(token)
continue
if re.fullmatch(r"[0-9]+(?:-[0-9]+)*", token):
real_words.append(token)
continue
token_lower = token.lower()
if token_lower in stopwords:
removed_words.append(token)
continue
if Detector.is_gibberish(token_lower):
# print(token)
removed_words.append(token)
continue
if token_lower in {"a", "i"}:
real_words.append(token)
continue
# contractions - keep directly without English check
if "'" in token_lower:
real_words.append(token)
continue
if token_lower in eng_words or zipf_frequency(token_lower, "en") >= 2.0:
real_words.append(token)
continue
removed_words.append(token)
# print("Real Words: ", real_words)
# print(f" Removed words : {removed_words} ({len(removed_words)} removed)")
if not real_words:
continue
# keep phrases when at least a reasonable portion of tokens are English-like
# if len(real_words) / len(tokens) < 0.25:
# continue
if len(removed_words) >= len(real_words):
continue
total_removed += len(removed_words)
clean_phrases.append(" ".join(real_words))
# print(f"\nTotal words removed: {total_removed}")
return " ".join(clean_phrases)
# cleaned_support_df = support_df[support_df["language"] == "en"][["conv_id", "turn_index", "role", "text", "industry", "product", "outcome", "issue_type", "overall_urgency"]].copy()
# cleaned_support_df["text"] = cleaned_support_df["text"].progress_apply(clean_text)
# cleaned_support_df = cleaned_support_df[cleaned_support_df["text"].str.strip() != ""]
# cleaned_support_df = cleaned_support_df.dropna(subset=["text"])
# cleaned_support_df.to_csv(data_dir / "processed" / "customer_support" / "cleaned_support_text.csv", index=False)
# row_text = support_df.loc[support_df["language"] == "en", "text"].iloc[4]
# cleaned_text = clean_text(row_text)
# print(row_text)
# print(cleaned_text)
# cleaning bank faqs
faq_df = faq_df.drop_duplicates(subset=["Question", "Answer"])
def clean_text_faq(text):
if not isinstance(text, str):
return text
text = re.sub(r"[\r\n\t]+", " ", text) # normalise line breaks
text = re.sub(r"[^\x00-\x7F]+", "", text)
text = re.sub(r"<<\s*>>", "", text)
text = re.sub(r" {2,}", " ", text) # collapse spaces
return text.strip()
for col in ["Question", "Answer"]:
faq_df[col] = faq_df[col].progress_apply(clean_text_faq)
faq_df.to_csv(data_dir / "processed" / "bank_faq" / "cleaned_faq.csv", index=False)
# split dataset for train/test
train_faq_df, test_faq_df = train_test_split(
faq_df,
test_size=0.2,
stratify=faq_df["Class"],
random_state=42
)
print(len(train_faq_df))
print(len(test_faq_df))
train_faq_df.to_csv(data_dir / "processed" / "bank_faq" / "train_faq.csv", index=False)
test_faq_df.to_csv(data_dir / "processed" / "bank_faq" / "test_faq.csv", index=False)
|