Update example

example

@@ -7,11 +7,13 @@ import unicodedata
 # CONFIGURATION
 # ==========================
 
-INPUT_FILE = "hbl_dump.txt"          # Your MongoDB export file
+INPUT_FILE = "hbl_dump.txt"       # MongoDB dump file
 OUTPUT_FILE = "hbl_pipe_dataset.csv"
 
 
-# Fields to keep (CNIC removed)
+# Fields to use
+# CNIC removed because it is numeric
+
 FIELDS = [
     "name",
     "fatherName",
@@ -26,46 +28,61 @@ FIELDS = [
 # TEXT CLEANING
 # ==========================
 
-def clean_text(text):
+def clean_text(text, uppercase=False):
     """
-    Remove Unicode formatting characters
-    and normalize spaces.
+    Cleans text:
+    - Removes invisible Unicode formatting characters
+    - Removes extra spaces
+    - Optional uppercase conversion
     """
 
     if not text:
         return ""
 
-    # Remove invisible Unicode control characters
-    # Includes:
-    # U+202A LRE
-    # U+202B RLE
-    # U+202C PDF
-    # U+202D LRO
-    # U+202E RLO
+    # Remove Unicode control formatting characters
+    # Example:
+    # U+202A LEFT-TO-RIGHT EMBEDDING
+    # U+202B RIGHT-TO-LEFT EMBEDDING
+    # U+202C POP DIRECTIONAL FORMATTING
+    # U+202D LEFT-TO-RIGHT OVERRIDE
+    # U+202E RIGHT-TO-LEFT OVERRIDE
 
     text = "".join(
         ch for ch in text
         if unicodedata.category(ch) != "Cf"
     )
 
-    # Normalize spaces
+
+    # Normalize whitespace
+
     text = re.sub(
         r"\s+",
         " ",
         text
     )
 
-    return text.strip()
+
+    text = text.strip()
+
+
+    # Roman Urdu normalization
+
+    if uppercase:
+        text = text.upper()
+
+
+    return text
 
 
 
 # ==========================
-# EXTRACTION FUNCTIONS
+# MONGODB PARSING
 # ==========================
 
 def extract_block(text, key):
     """
-    Extract dictionary block after a key.
+    Extract dictionary content after a key.
+
     Example:
     "data" : {
         ...
@@ -82,61 +99,70 @@ def extract_block(text, key):
         re.DOTALL
     )
 
+
     if match:
         return match.group(1)
 
+
     return None
 
 
 
-def extract_fields(block):
+def extract_fields(block, uppercase=False):
     """
-    Extract required fields from block.
+    Extract required fields from MongoDB block.
     """
 
     result = {}
 
+
     for field in FIELDS:
 
         pattern = (
             rf'"{field}"\s*:\s*"([^"]*)"'
         )
 
+
         match = re.search(
             pattern,
             block
         )
 
+
         if match:
+
             result[field] = clean_text(
-                match.group(1)
+                match.group(1),
+                uppercase=uppercase
             )
+
         else:
+
+            # Keep empty field
             result[field] = ""
 
+
     return result
 
 
 
 def join_fields(data):
-    """
-    Create pipe separated format.
-    """
 
     return "|".join(
-        clean_text(data[field])
+        data[field]
         for field in FIELDS
     )
 
 
 
 # ==========================
-# MAIN PROCESS
+# MAIN
 # ==========================
 
 def process():
 
-    print("Reading file...")
+    print("Reading MongoDB dump...")
+
 
     with open(
         INPUT_FILE,
@@ -147,7 +173,8 @@ def process():
         content = f.read()
 
 
-    # Split MongoDB documents
+
+    # Split individual MongoDB documents
 
     documents = re.split(
         r'\n}\s*\n{',
@@ -156,7 +183,7 @@ def process():
 
 
     print(
-        "Documents detected:",
+        "Documents found:",
         len(documents)
     )
 
@@ -166,6 +193,7 @@ def process():
     skipped = 0
 
 
+
     for doc in documents:
 
 
@@ -174,6 +202,7 @@ def process():
             "REQUEST_BODY"
         )
 
+
         roman_block = extract_block(
             doc,
             "data"
@@ -181,28 +210,40 @@ def process():
 
 
         if not urdu_block or not roman_block:
+
             skipped += 1
             continue
 
 
+
+        # Urdu stays original
+
         urdu_fields = extract_fields(
-            urdu_block
+            urdu_block,
+            uppercase=False
         )
 
+
+        # Roman Urdu converted to uppercase
+
         roman_fields = extract_fields(
-            roman_block
+            roman_block,
+            uppercase=True
         )
 
 
+
         urdu_text = join_fields(
             urdu_fields
         )
 
+
         roman_text = join_fields(
             roman_fields
         )
 
 
+
         rows.append(
             {
                 "urdu": urdu_text,
@@ -211,17 +252,20 @@ def process():
         )
 
 
+
     print(
-        "Generated pairs:",
+        "Pairs generated:",
         len(rows)
     )
 
+
     print(
         "Skipped:",
         skipped
     )
 
 
+
     # Save CSV
 
     with open(
@@ -247,6 +291,7 @@ def process():
         writer.writerows(rows)
 
 
+
     print(
         "Saved:",
         OUTPUT_FILE