# ============================================================================ # HINDI TEXT PREPROCESSOR (OPTIMIZED FOR PYTHON 3.11) # ============================================================================ # This module contains regex-based preprocessing functions for Hindi text # Handles Devanagari script patterns, punctuation, and normalization # Optimized for large datasets with compiled regex patterns import re from typing import Iterator, Optional # Devanagari Unicode ranges # Devanagari: U+0900 to U+097F # Devanagari Extended: U+A8E0 to U+A8FF DEVANAGARI_PATTERN = r'[\u0900-\u097F\uA8E0-\uA8FF]+' # Common Hindi punctuation marks HINDI_PUNCTUATION = r'[।॥,;:!?\-—–()\[\]{}"\'\'""]' # Numbers (both Devanagari and Arabic) DEVANAGARI_NUMBERS = r'[\u0966-\u096F]' # ०-९ ARABIC_NUMBERS = r'[0-9]' # Pre-compile regex patterns for better performance (Python 3.11 optimization) # Compiling once and reusing is much faster than compiling on each call _WHITESPACE_PATTERN = re.compile(r'\s+') _SPACE_BEFORE_PUNCT = re.compile(r'\s+([,;:!?।॥])') _PUNCT_AFTER_SPACE = re.compile(r'([,;:!?।॥])([^\s])') _OPEN_QUOTE_SPACE = re.compile(r'([""\'\([{])\s+') _CLOSE_QUOTE_SPACE = re.compile(r'\s+([""\'\]}])') _CLOSE_QUOTE_WORD = re.compile(r'([""\'\]}])([^\s])') _DASH_NORMALIZE = re.compile(r'[—–]') _QUOTE_NORMALIZE_DOUBLE = re.compile(r'[""]') _QUOTE_NORMALIZE_SINGLE = re.compile(r'[\'\']') _INVISIBLE_CHARS = re.compile(r'[\u200B-\u200D\uFEFF]') def normalize_hindi_text(text): """ Normalize Hindi text using regex patterns for proper formatting. This function: 1. Normalizes whitespace 2. Handles punctuation properly (no space before, one space after) 3. Normalizes common characters (dashes, quotes) 4. Removes invisible characters Args: text (str): Raw Hindi text Returns: str: Normalized Hindi text """ if not text: return text # --- Step 1: Normalize multiple whitespaces to single space --- text = re.sub(r'\s+', ' ', text).strip() # --- Step 2: Handle punctuation properly --- # A. Remove any existing space *before* standard Hindi punctuation marks # (,, ;, :, !, ?, ।, ॥) text = re.sub(r'\s+([,;:!?।॥])', r'\1', text) # B. Ensure a single space *after* these punctuation marks # This correctly formats sentences like "है।अगला" to "है। अगला" text = re.sub(r'([,;:!?।॥])([^\s])', r'\1 \2', text) # C. Handle quotation marks and parentheses correctly # Hindi follows Western rules for these: no space inside the enclosure. # Remove space after opening quotes/parentheses text = re.sub(r'([“"‘\([{])\s+', r'\1', text) # Remove space before closing quotes/parentheses text = re.sub(r'\s+([”"’\])}])', r'\1', text) # Ensure a space *after* closing quotes/parentheses if a word follows text = re.sub(r'([”"’\])}])([^\s])', r'\1 \2', text) # --- Step 3: Normalize common characters --- # Normalize different types of dashes text = re.sub(r'[—–]', '-', text) # Normalize quotes (if you want to force straight quotes everywhere) text = re.sub(r'[“”]', '"', text) text = re.sub(r'[‘’]', "'", text) # --- Step 4: Remove zero-width characters and other invisible characters --- text = re.sub(r'[\u200B-\u200D\uFEFF]', '', text) # Zero-width spaces etc. # Strip leading/trailing spaces again after final operations text = text.strip() return text # Example Usage: # raw_text = """ # यह एक उदाहरण पाठ है,जो दिखाताहै कि स्पेसिंग कितनी ज़रूरीहै! # "यह एक उद्धरण है।" क्या यह काम करता है? # यह।सही है॥हाँ # """ # normalized_text = normalize_hindi_text(raw_text) # print("Original Text:") # print(raw_text) # print("\nNormalized Text:") # print(normalized_text) # Expected Output: # Normalized Text: # यह एक उदाहरण पाठ है, जो दिखाता है कि स्पेसिंग कितनी ज़रूरी है! "यह एक उद्धरण है।" क्या यह काम करता है? यह। सही है॥ हाँ def separate_punctuation(text): """ Separate punctuation marks from Hindi words using regex. This helps the tokenizer handle punctuation better by treating it as separate tokens. Args: text (str): Hindi text Returns: str: Text with punctuation separated """ if not text: return text # Separate punctuation from Devanagari words # Pattern: word followed by punctuation or punctuation followed by word text = re.sub(r'([\u0900-\u097F\uA8E0-\uA8FF]+)([।॥,;:!?\-()\[\]{}"\'])', r'\1 \2', text) text = re.sub(r'([।॥,;:!?\-()\[\]{}"\'])([\u0900-\u097F\uA8E0-\uA8FF]+)', r'\1 \2', text) return text def clean_hindi_text(text: str) -> str: """ Comprehensive cleaning of Hindi text using regex. Combines normalization and punctuation separation. Optimized for large text processing. Args: text (str): Raw Hindi text Returns: str: Cleaned and normalized Hindi text """ if not text: return text # Apply normalization (uses pre-compiled patterns) text = normalize_hindi_text(text) # Separate punctuation text = separate_punctuation(text) # Final whitespace normalization (use pre-compiled pattern) text = _WHITESPACE_PATTERN.sub(' ', text).strip() return text def clean_hindi_text_streaming(file_path: str, chunk_size: int = 1024 * 1024) -> Iterator[str]: """ Stream and clean large Hindi text files in chunks to avoid memory issues. This is optimized for very large files (>100MB) that don't fit in memory. Args: file_path (str): Path to the text file chunk_size (int): Size of chunks to read (default: 1MB) Yields: str: Cleaned text chunks """ with open(file_path, 'r', encoding='utf-8') as f: buffer = "" while True: chunk = f.read(chunk_size) if not chunk: if buffer: yield clean_hindi_text(buffer) break buffer += chunk # Process complete lines to avoid breaking in the middle of text while '\n' in buffer: line, buffer = buffer.split('\n', 1) if line.strip(): cleaned = clean_hindi_text(line) if cleaned: yield cleaned def extract_hindi_words(text): """ Extract only Hindi words (Devanagari script) from text using regex. Useful for filtering out non-Hindi content. Args: text (str): Mixed text Returns: list: List of Hindi words found """ if not text: return [] # Find all Devanagari words hindi_words = re.findall(DEVANAGARI_PATTERN, text) return hindi_words def is_hindi_text(text): """ Check if text contains Hindi (Devanagari script) using regex. Args: text (str): Text to check Returns: bool: True if text contains Devanagari characters """ if not text: return False return bool(re.search(DEVANAGARI_PATTERN, text)) def filter_hindi_only(text: str, min_hindi_ratio: float = 0.7) -> str: """ Filter text to keep only lines/sentences with significant Hindi content. This ensures the tokenizer only learns from Hindi text, not mixed content. Removes non-Hindi characters and keeps only Devanagari script with allowed punctuation. Args: text (str): Input text (may contain mixed Hindi/English/other) min_hindi_ratio (float): Minimum ratio of Devanagari chars to keep a line (0.0-1.0) Default 0.7 means at least 70% Hindi characters Returns: str: Filtered text containing only Hindi-dominant lines """ if not text: return text lines = text.split('\n') filtered_lines = [] for line in lines: line = line.strip() if not line: continue # Count Devanagari characters devanagari_chars = len(re.findall(DEVANAGARI_PATTERN, line)) total_chars = len(re.sub(r'\s', '', line)) # Exclude whitespace if total_chars == 0: continue # Calculate ratio of Hindi characters hindi_ratio = devanagari_chars / total_chars if total_chars > 0 else 0 # Keep line if it has sufficient Hindi content if hindi_ratio >= min_hindi_ratio: # Extract only Devanagari words and allowed punctuation # Keep Hindi words, Hindi punctuation (।॥), and basic punctuation hindi_line = re.sub( r'[^\u0900-\u097F\uA8E0-\uA8FF\s।॥,;:!?\-()\[\]{}"\']+', ' ', line ) # Normalize whitespace hindi_line = _WHITESPACE_PATTERN.sub(' ', hindi_line).strip() if hindi_line: filtered_lines.append(hindi_line) return '\n'.join(filtered_lines)