Update documents_prep.py

documents_prep.py

@@ -22,61 +22,50 @@ def normalize_text(text):
     
     return text
 
+import re
+
 def normalize_steel_designations(text):
     """
-    Convert Latin letters to Cyrillic in steel designations.
-    Only applies to specific patterns to avoid changing legitimate Latin text.
+    Normalize steel designations by converting Latin letters to Cyrillic.
+    Handles patterns like 08X18H10T, 12X18H9, 10H17N13M2T, etc.
     Returns: (normalized_text, changes_count, changes_list)
     """
     if not text:
         return text, 0, []
-    
-    import re
-    
+
     changes_count = 0
-    changes_list = []  # NEW: Track what changed
-    
+    changes_list = []
+
+    # Mapping of Latin to Cyrillic for steel designations
+    replacements = {
+        'X': 'Х',
+        'H': 'Н',
+        'T': 'Т',
+        'C': 'С',
+        'B': 'В',
+        'K': 'К',
+        'M': 'М',
+        'A': 'А',
+        'P': 'Р',
+    }
+
+    # Regex to match steel designations like 08X18H10T, 10H17N13M2T, etc.
+    # \b\d{1,3} — starts with 1–3 digits
+    # (?:[A-ZА-Я]\d*)+ — then one or more groups of a letter + optional digits
+    pattern = r'\b\d{1,3}(?:[A-ZА-Я]\d*)+\b'
+
     def replace_in_steel_grade(match):
-        """Replace Latin with Cyrillic only in steel grade context"""
         nonlocal changes_count, changes_list
-        grade = match.group(0)
-        original_grade = grade
-        
-        # Mapping of Latin to Cyrillic for steel designations
-        replacements = {
-            'X': 'Х',  # Latin X -> Cyrillic Х (Kha)
-            'H': 'Н',  # Latin H -> Cyrillic Н (En)
-            'T': 'Т',  # Latin T -> Cyrillic Т (Te)
-            'C': 'С',  # Latin C -> Cyrillic С (Es)
-            'B': 'В',  # Latin B -> Cyrillic В (Ve)
-            'K': 'К',  # Latin K -> Cyrillic К (Ka)
-            'M': 'М',  # Latin M -> Cyrillic М (Em)
-            'A': 'А',  # Latin A -> Cyrillic А (A)
-            'P': 'Р',  # Latin P -> Cyrillic Р (Er)
-        }
-        for latin, cyrillic in replacements.items():
-            grade = grade.replace(latin, cyrillic)
-        
-        if grade != original_grade:
+        original = match.group(0)
+        converted = ''.join(replacements.get(ch, ch) for ch in original)
+        if converted != original:
             changes_count += 1
-            changes_list.append(f"{original_grade} → {grade}")  # NEW: Record change
-        
-        return grade
-    
-    # Pattern for steel grades
-    text = re.sub(r'\b\d{1,3}[XHTCBKMAP]{1,4}\d{0,2}[XHTCBKMAP]{0,4}\b', 
-                  replace_in_steel_grade, text)
-    
-    # Pattern 2: Welding wire designations
-    text = re.sub(r'\b[CS]B-\d{1,3}[XHTCBKMAP]{1,4}\d{0,2}[XHTCBKMAP]{0,4}\b',
-                  replace_in_steel_grade, text)
-    
-    # Pattern 3: Welding consumables
-    text = re.sub(r'\b[C]-\d{1,2}\b', 
-                  lambda m: (changes_list.append(f"{m.group(0)} → {m.group(0).replace('C', 'С')}") or changes_count.__add__(1), m.group(0).replace('C', 'С'))[1] if m.group(0) != m.group(0).replace('C', 'С') else m.group(0), 
-                  text)
-    
-    return text, changes_count, changes_list
+            changes_list.append(f"{original} → {converted}")
+        return converted
+
+    normalized_text = re.sub(pattern, replace_in_steel_grade, text)
+
+    return normalized_text, changes_count, changes_list