Create make_data.py

make_data.py

@@ -0,0 +1,503 @@
+import re
+import json
+import csv
+import pandas as pd
+from tqdm import tqdm
+
+
+# ============================================================
+# CONFIGURATION
+# ============================================================
+
+INPUT_FILE = "hbl_mongodb_export.txt"
+
+OUTPUT_FILE = "hbl_nadra_clean.csv"
+
+
+# For testing:
+# Example:
+# LIMIT_ROWS = 10000
+#
+# Full dataset:
+LIMIT_ROWS = None
+
+
+# Required API fields
+FIELDS = [
+    "name",
+    "fatherName",
+    "motherName",
+    "address1",
+    "address2",
+    "placeOfBirth"
+]
+
+
+# ============================================================
+# MONGODB TEXT PARSER
+# ============================================================
+
+
+def clean_mongo_syntax(text):
+
+    """
+    Convert Mongo shell syntax into JSON compatible syntax
+    """
+
+    # ObjectId("xxx") -> "xxx"
+    text = re.sub(
+        r'ObjectId\("([^"]+)"\)',
+        r'"\1"',
+        text
+    )
+
+
+    # ISODate("xxx") -> "xxx"
+    text = re.sub(
+        r'ISODate\("([^"]+)"\)',
+        r'"\1"',
+        text
+    )
+
+
+    # NumberInt(200) -> 200
+    text = re.sub(
+        r'NumberInt\((\d+)\)',
+        r'\1',
+        text
+    )
+
+
+    return text
+
+
+
+def extract_documents(path):
+
+    """
+    Extract individual Mongo documents
+    """
+
+    print("Reading MongoDB export...")
+
+
+    with open(
+        path,
+        "r",
+        encoding="utf-8"
+    ) as f:
+
+        content = f.read()
+
+
+
+    content = clean_mongo_syntax(
+        content
+    )
+
+
+    documents = []
+
+
+    decoder = json.JSONDecoder()
+
+
+    index = 0
+
+
+    while index < len(content):
+
+        # skip whitespace
+        while (
+            index < len(content)
+            and content[index].isspace()
+        ):
+            index += 1
+
+
+        if index >= len(content):
+            break
+
+
+        # Find next object
+        if content[index] != "{":
+
+            index += 1
+            continue
+
+
+        try:
+
+            obj, end = decoder.raw_decode(
+                content,
+                index
+            )
+
+            documents.append(obj)
+
+            index = end
+
+
+        except Exception:
+
+            index += 1
+
+
+
+    return documents
+
+
+
+# ============================================================
+# TEXT NORMALIZATION
+# ============================================================
+
+
+def remove_unicode_artifacts(text):
+
+    if not text:
+        return ""
+
+
+    text = str(text)
+
+
+    # Directional marks
+    text = re.sub(
+        r"[\u202A-\u202E]",
+        "",
+        text
+    )
+
+
+    # Zero width characters
+    text = re.sub(
+        r"[\u200B-\u200D\uFEFF]",
+        "",
+        text
+    )
+
+
+    return text
+
+
+
+def clean_urdu(text):
+
+    if not text:
+        return ""
+
+
+    text = remove_unicode_artifacts(
+        text
+    )
+
+
+    # Normalize spaces only
+    text = re.sub(
+        r"\s+",
+        " ",
+        text
+    )
+
+
+    return text.strip()
+
+
+
+def clean_roman(text):
+
+    if not text:
+        return ""
+
+
+    text = remove_unicode_artifacts(
+        text
+    )
+
+
+    # Uppercase
+    text = text.upper()
+
+
+    # Remove punctuation
+    text = re.sub(
+        r"[.,،؛:()\[\]{}]",
+        " ",
+        text
+    )
+
+
+    # Remove unwanted symbols
+    text = re.sub(
+        r"[^A-Z0-9\s]",
+        "",
+        text
+    )
+
+
+    # Normalize spaces
+    text = re.sub(
+        r"\s+",
+        " ",
+        text
+    )
+
+
+    return text.strip()
+
+
+
+# ============================================================
+# PIPE FORMAT CREATION
+# ============================================================
+
+
+def create_pipe_string(data, cleaner):
+
+    values = []
+
+
+    for field in FIELDS:
+
+        value = data.get(
+            field,
+            ""
+        )
+
+
+        value = cleaner(
+            value
+        )
+
+
+        values.append(
+            value
+        )
+
+
+    return "|".join(values)
+
+
+
+# ============================================================
+# PROCESS DATA
+# ============================================================
+
+
+def process_documents(documents):
+
+    rows = []
+
+
+    stats = {
+
+        "total": 0,
+        "success": 0,
+        "missing": 0,
+        "pipe_error": 0
+
+    }
+
+
+    print("Processing records...")
+
+
+    for doc in tqdm(documents):
+
+        stats["total"] += 1
+
+
+        try:
+
+
+            # Only successful API responses
+
+            if doc.get("message") != "Success":
+                continue
+
+
+            if doc.get("statusCode") != 200:
+                continue
+
+
+
+            urdu_data = doc.get(
+                "REQUEST_BODY",
+                {}
+            )
+
+
+            roman_data = doc.get(
+                "data",
+                {}
+            )
+
+
+
+            if not urdu_data or not roman_data:
+
+                stats["missing"] += 1
+                continue
+
+
+
+            urdu_text = create_pipe_string(
+                urdu_data,
+                clean_urdu
+            )
+
+
+            roman_text = create_pipe_string(
+                roman_data,
+                clean_roman
+            )
+
+
+
+            # Check empty fields
+
+            if (
+                "" in urdu_text.split("|")
+                or
+                "" in roman_text.split("|")
+            ):
+
+                stats["missing"] += 1
+                continue
+
+
+
+            # Pipe validation
+
+            if (
+                urdu_text.count("|")
+                !=
+                roman_text.count("|")
+            ):
+
+                stats["pipe_error"] += 1
+                continue
+
+
+
+            rows.append(
+                {
+                    "urdu": urdu_text,
+                    "roman": roman_text
+                }
+            )
+
+
+            stats["success"] += 1
+
+
+
+        except Exception:
+
+            continue
+
+
+
+    print("\nStatistics")
+    print("----------------")
+    for k,v in stats.items():
+        print(k, ":", v)
+
+
+
+    return rows
+
+
+
+# ============================================================
+# MAIN
+# ============================================================
+
+
+documents = extract_documents(
+    INPUT_FILE
+)
+
+
+print(
+    "Documents loaded:",
+    len(documents)
+)
+
+
+
+if LIMIT_ROWS:
+
+    documents = documents[:LIMIT_ROWS]
+
+
+
+rows = process_documents(
+    documents
+)
+
+
+
+df = pd.DataFrame(
+    rows
+)
+
+
+
+print(
+    "\nRemoving duplicates..."
+)
+
+
+before = len(df)
+
+
+df = df.drop_duplicates(
+    subset=[
+        "urdu",
+        "roman"
+    ]
+)
+
+
+print(
+    "Duplicates removed:",
+    before - len(df)
+)
+
+
+
+# Shuffle
+
+df = df.sample(
+    frac=1,
+    random_state=42
+).reset_index(
+    drop=True
+)
+
+
+
+print(
+    "Final rows:",
+    len(df)
+)
+
+
+
+# Save
+
+df.to_csv(
+    OUTPUT_FILE,
+    index=False,
+    encoding="utf-8-sig",
+    quoting=csv.QUOTE_ALL
+)
+
+
+
+print(
+    "\nSaved:",
+    OUTPUT_FILE
+)