ogx786
/

chatbot

Model card Files Files and versions

xet

Community

ogx786 commited on Jul 29

Commit

efe510a

verified ·

1 Parent(s): 3670588

Update make_data.py

Browse files

Files changed (1) hide show

make_data.py +14 -562

make_data.py CHANGED Viewed

@@ -1,562 +1,14 @@
-import re
-import json
-import csv
-import pandas as pd
-from tqdm import tqdm
-# ============================================================
-# CONFIGURATION
-# ============================================================
-INPUT_FILE = "hbl_mongodb_export.txt"
-OUTPUT_FILE = "hbl_nadra_clean.csv"
-# Use for testing
-# Example:
-# LIMIT_ROWS = 2000
-#
-# Full dataset:
-LIMIT_ROWS = 2000
-FIELDS = [
-    "name",
-    "fatherName",
-    "motherName",
-    "address1",
-    "address2",
-    "placeOfBirth"
-]
-# ============================================================
-# MONGODB SYNTAX CLEANING
-# ============================================================
-def clean_mongo_syntax(text):
-    # ObjectId("xxx")
-    text = re.sub(
-        r'ObjectId\("([^"]+)"\)',
-        r'"\1"',
-        text
-    )
-    # ISODate("xxx")
-    text = re.sub(
-        r'ISODate\("([^"]+)"\)',
-        r'"\1"',
-        text
-    )
-    # NumberInt(200)
-    text = re.sub(
-        r'NumberInt\((\d+)\)',
-        r'\1',
-        text
-    )
-    return text
-# ============================================================
-# STREAM MONGODB DOCUMENTS
-# ============================================================
-def stream_mongodb_documents(path, limit=None):
-    print("Reading MongoDB export...")
-    count = 0
-    buffer = ""
-    inside_document = False
-    brace_count = 0
-    with open(
-        path,
-        "r",
-        encoding="utf-8"
-    ) as file:
-        for line in file:
-            line = line.rstrip()
-            if not line:
-                continue
-            # document starts
-            if line.startswith("{"):
-                inside_document = True
-                brace_count = 0
-                buffer = ""
-            if inside_document:
-                buffer += line + "\n"
-                brace_count += line.count("{")
-                brace_count -= line.count("}")
-                # complete document
-                if brace_count == 0:
-                    cleaned = clean_mongo_syntax(
-                        buffer
-                    )
-                    try:
-                        obj = json.loads(
-                            cleaned
-                        )
-                        yield obj
-                        count += 1
-                        if limit and count >= limit:
-                            return
-                    except Exception as e:
-                        print(
-                            "\nSkipping invalid document:",
-                            e
-                        )
-                    inside_document = False
-                    buffer = ""
-# ============================================================
-# TEXT CLEANING
-# ============================================================
-def remove_unicode_artifacts(text):
-    if not text:
-        return ""
-    text = str(text)
-    # RTL/LTR marks
-    text = re.sub(
-        r"[\u202A-\u202E]",
-        "",
-        text
-    )
-    # zero width chars
-    text = re.sub(
-        r"[\u200B-\u200D\uFEFF]",
-        "",
-        text
-    )
-    return text
-def clean_urdu(text):
-    if not text:
-        return ""
-    text = remove_unicode_artifacts(
-        text
-    )
-    # normalize spaces only
-    text = re.sub(
-        r"\s+",
-        " ",
-        text
-    )
-    return text.strip()
-def clean_roman(text):
-    if not text:
-        return ""
-    text = remove_unicode_artifacts(
-        text
-    )
-    # uppercase
-    text = text.upper()
-    # remove punctuation
-    text = re.sub(
-        r"[.,،؛:()\[\]{}]",
-        " ",
-        text
-    )
-    # remove symbols
-    text = re.sub(
-        r"[^A-Z0-9\s]",
-        "",
-        text
-    )
-    # normalize spaces
-    text = re.sub(
-        r"\s+",
-        " ",
-        text
-    )
-    return text.strip()
-# ============================================================
-# PIPE FORMAT
-# ============================================================
-def create_pipe_format(data, cleaner):
-    values = []
-    for field in FIELDS:
-        value = data.get(
-            field,
-            ""
-        )
-        value = cleaner(
-            value
-        )
-        values.append(
-            value
-        )
-    return "|".join(values)
-# ============================================================
-# PROCESS DOCUMENTS
-# ============================================================
-def process_documents(documents):
-    rows = []
-    stats = {
-        "total":0,
-        "success":0,
-        "missing":0,
-        "pipe_error":0
-    }
-    print(
-        "Processing documents..."
-    )
-    for doc in tqdm(documents):
-        stats["total"] += 1
-        try:
-            # keep only successful calls
-            if doc.get("message") != "Success":
-                continue
-            if doc.get("statusCode") != 200:
-                continue
-            urdu_data = doc.get(
-                "REQUEST_BODY",
-                {}
-            )
-            roman_data = doc.get(
-                "data",
-                {}
-            )
-            if not urdu_data or not roman_data:
-                stats["missing"] += 1
-                continue
-            urdu_text = create_pipe_format(
-                urdu_data,
-                clean_urdu
-            )
-            roman_text = create_pipe_format(
-                roman_data,
-                clean_roman
-            )
-            # check empty fields
-            if (
-                "" in urdu_text.split("|")
-                or
-                "" in roman_text.split("|")
-            ):
-                stats["missing"] += 1
-                continue
-            # pipe count validation
-            if (
-                urdu_text.count("|")
-                !=
-                roman_text.count("|")
-            ):
-                stats["pipe_error"] += 1
-                continue
-            rows.append(
-                {
-                    "urdu":urdu_text,
-                    "roman":roman_text
-                }
-            )
-            stats["success"] += 1
-        except Exception:
-            continue
-    print("\nStatistics")
-    print("----------------")
-    for k,v in stats.items():
-        print(
-            f"{k}: {v}"
-        )
-    return rows
-# ============================================================
-# MAIN
-# ============================================================
-documents = stream_mongodb_documents(
-    INPUT_FILE,
-    LIMIT_ROWS
-)
-rows = process_documents(
-    documents
-)
-df = pd.DataFrame(
-    rows
-)
-print(
-    "\nRemoving duplicates..."
-)
-before = len(df)
-df = df.drop_duplicates(
-    subset=[
-        "urdu",
-        "roman"
-    ]
-)
-print(
-    "Duplicates removed:",
-    before - len(df)
-)
-# shuffle
-df = df.sample(
-    frac=1,
-    random_state=42
-).reset_index(
-    drop=True
-)
-print(
-    "Final dataset size:",
-    len(df)
-)
-df.to_csv(
-    OUTPUT_FILE,
-    index=False,
-    encoding="utf-8-sig",
-    quoting=csv.QUOTE_ALL
-)
-print(
-    "\nSaved:",
-    OUTPUT_FILE
-)

+(venv) PS D:\AI\chatbot> python clean_data.py
+                                                urdu                                              roman
+0  احسن احمد|سکندر احمد|سکندر احمد|مکان نمبر B-44...  AHSAN AHMED|SIKANDAR AHMAD|SIKANDAR AHMAD|HOUS...
+1  محمد شان|غلام دراز خان|غلام دراز خان|آڑہ روڈ ح...  MUHAMMAD SHAN|GHULAM DARAZ KHAN|GHULAM DARAZ K...
+2  اکبر پرویز|پرویز خالد شیخ|پرویز خالد شیخ|مکان ...  AKBAR PERVEZ|PERVEZ KHALID SHEIKH|PERVEZ KHALI...
+3  رحمان حسن|رفاقت علی|رفاقت علی|ڈاک خانہ ونیکے ت...  REHMAN HASSAN|RAFAQAT ALI|RAFAQAT ALI|POST OFF...
+4  ناصر شفیع|شفیع مسیح|شفیع مسیح|مکان نمبر 271،نع...  NASIR SHAFI|SHAFI MASIH|SHAFI MASIH|HOUSE NUMB...
+Urdu:
+احسن احمد|سکندر احمد|سکندر احمد|مکان نمبر B-447،محلہ میمن،ٹنڈو آدم،ضلع سانگھڑ|مکان نمبر B-447،محلہ میمن،ٹنڈو آدم،ضلع سانگھڑ|ٹنڈو آدم,سانگھڑ
+Roman:
+AHSAN AHMED|SIKANDAR AHMAD|SIKANDAR AHMAD|HOUSE NUMBER B447 MOHALA MEMON TANDO ADAM DISTRICT SANGHAR|HOUSE NUMBER B447 MOHALA MEMON TANDO ADAM DISTRICT SANGHAR|TANDO ADAM SANGHAR
+(venv) PS D:\AI\chatbot>