Update app.py

app.py

@@ -17,47 +17,43 @@ from PyPDF2 import PdfReader  # plain text extraction
 import gradio as gr  # UI
 from dotenv import load_dotenv  # optional .env support
 
+
 # ─────────────────────────────────────────────────────────────────────────────
-# 1. PDF & TEXT PROCESSING  (LOGIC MODIFIED HERE)
+# 1. PDF & TEXT PROCESSING
 # ─────────────────────────────────────────────────────────────────────────────
 
 def extract_pdf_text(pdf_file) -> str:
     """Extracts text from a PDF file using PyPDF2."""
     reader = PdfReader(pdf_file)
-    # MODIFICATION: Skips the first 4 pages (ToC/List of Rules)
-    return "\n".join(p.extract_text() or "" for i, p in enumerate(reader.pages) if i >= 4)
+    return "\n".join(p.extract_text() or "" for p in reader.pages)
 
 
 def extract_pdf_word(pdf_file) -> str:
     """Extracts text from PDF using PyMuPDF (fitz) for better layout preservation."""
     doc = fitz.open(pdf_file)
-    # MODIFICATION: Skips the first 4 pages (ToC)
-    text_blocks = [page.get_text("text") for i, page in enumerate(doc) if i >= 4]
+    text_blocks = [page.get_text("text") for page in doc]
     return "\n".join(filter(None, text_blocks))
 
 
 def merge_pdf_wrapped_lines(raw_text: str) -> list[str]:
-    """Re-join hard-wrapped lines from PDF extraction based on grammatical context."""
+    """Re-join hard-wrapped lines from PDF extraction."""
     merged = []
     for ln in raw_text.splitlines():
         ln_stripped = ln.strip()
-        if not ln_stripped:
-            continue
-        
+        if not ln_stripped: continue
         if merged:
             prev = merged[-1]
-            # Merge if previous line ends with '—' or lacks closing punctuation,
-            # and the next line appears to be a continuation.
-            if prev.endswith('—') or \
-               (not re.search(r'[.:;)]\s*$', prev) and re.match(r'^[a-z\(]', ln_stripped)):
-                merged[-1] = prev + ' ' + ln_stripped
+            if (re.search(r'[a-z]$', prev) and re.match(r'^[\(a-z]', ln_stripped)) or \
+                    (re.search(r'\b(?:rule|may|and|or)$', prev, re.I) and re.match(r'^\d+\.\d+', ln_stripped)) or \
+                    (re.search(r'\brule\s+\d+\.$', prev, re.I) and re.match(r'^\d', ln_stripped)):
+                merged[-1] = prev + (' ' if re.search(r'[a-z]$', prev) else '') + ln_stripped
                 continue
         merged.append(ln_stripped)
     return merged
 
 
 # ─────────────────────────────────────────────────────────────────────────────
-# 2. RULE PARSING & CLEANING (LOGIC MODIFIED HERE)
+# 2. RULE PARSING & CLEANING (Initial Automated Pass)
 # ─────────────────────────────────────────────────────────────────────────────
 
 # --- Regex for rule structure ---
@@ -73,9 +69,6 @@ subpart_pat = re.compile(
     r'^\s*\d+\.\s*Subpart\s+([A-Z]{1,2})\s*[—-]\s*(.+)$',
     re.IGNORECASE
 )
-# NEW: Regex to specifically identify sub-rule paragraphs like (a), (1), (i)
-sub_rule_pat = re.compile(r'^\s*(\((?:[a-z]{1,2}|[ivx]+|\d+)\))\s*(.*)', re.IGNORECASE)
-
 
 # --- Regex for cleaning ---
 page_pat = re.compile(r'Page\s+\d+\s*/\s*\d+', re.IGNORECASE)
@@ -104,89 +97,53 @@ def clean_line(line: str, source: str) -> str:
     line = re.sub(r'\s{2,}', ' ', line)
     return line.strip()
 
-def get_rule_level(paren_str):
-    """Determines nesting level of a sub-rule, e.g., (1) is 1, (a) is 2, (i) is 3."""
-    content = paren_str.strip('()').lower()
-    if not content: return 99
-    if content.isdigit(): return 1
-    if all(c in 'ivxlmc' for c in content): return 3 # roman numerals
-    if content.isalpha(): return 2 # alphabetical
-    return 4 # Unknown level, treat as deeply nested
 
 def parse_rules(text: str, source: str) -> dict[str, str]:
-    """
-    Parses raw text into a dictionary of {rule_id: rule_text}.
-    This version is stateful and context-aware to handle hierarchies correctly.
-    """
-    rules = {}
-    parent_parts = []  # Tracks the current rule hierarchy, e.g., ['108.51', '(3)']
-    lines_buffer = []
-
-    def commit_buffer():
-        """Saves the buffered lines to the current rule ID."""
-        if parent_parts and lines_buffer:
-            rule_id = "".join(parent_parts)
-            existing_text = rules.get(rule_id, "")
-            new_text = " ".join(lines_buffer)
-            rules[rule_id] = (existing_text + " " + new_text).strip()
-        lines_buffer.clear()
+    """Parses raw text into a dictionary of {rule_id: rule_text}."""
+    rules, current, title = {}, None, ""
 
     lines = merge_pdf_wrapped_lines(text)
 
-    for line in lines:
-        cleaned = clean_line(line, source)
-        if not cleaned: continue
-
-        m_main = rule_pat.match(cleaned)
-        m_sub = sub_rule_pat.match(cleaned)
-        m_sp = subpart_pat.match(cleaned)
-        
-        if m_sp:
-            commit_buffer()
-            parent_parts = [f"subpart-{m_sp.group(1).upper()}"]
-            rules["".join(parent_parts)] = f"Subpart {m_sp.group(1).upper()} — {m_sp.group(2).strip()}"
-        
-        elif m_main:
-            new_base_id = m_main.group('base_rule')
-            current_base_id = parent_parts[0] if parent_parts and not parent_parts[0].startswith("subpart") else None
-            
-            if new_base_id == current_base_id:
-                lines_buffer.append(cleaned)
-                continue
-
-            commit_buffer()
-            parent_parts = [new_base_id]
-            title = m_main.group('title').strip()
+    for raw_line in lines:
+        line = clean_line(raw_line, source)
+        if not line: continue
+
+        m_ap_item = appendix_item_pat.match(line)
+        m_sp = subpart_pat.match(line)
+        m_rule = rule_pat.match(line)
+
+        new_key = None
+        new_title = ""
+
+        if m_ap_item:
+            key_parts = [m_ap_item.group(1).upper(), m_ap_item.group(2)]
+            if m_ap_item.group(3): key_parts.append(f"({m_ap_item.group(3).strip()})")
+            new_key = ".".join(key_parts)
+            new_title = m_ap_item.group('title').strip()
+        elif m_sp:
+            new_key = f"subpart-{m_sp.group(1).upper()}"
+            new_title = f"Subpart {m_sp.group(1).upper()} — {m_sp.group(2).strip()}"
+        elif m_rule:
+            base = m_rule.group('base_rule')
+            parens_str = m_rule.group('parens') or ""
+            new_key = base + "".join(re.findall(r'\([^)]+\)', parens_str))
+            new_title = m_rule.group('title').strip()
+
+        if new_key:
+            current = new_key
+            title = new_title
+            rules.setdefault(current, [])
             if title:
-                rules["".join(parent_parts)] = title
-        
-        elif m_sub and parent_parts and not parent_parts[0].startswith("subpart"):
-            commit_buffer()
-            paren_part = m_sub.group(1)
-            text_part = m_sub.group(2).strip()
-            new_level = get_rule_level(paren_part)
-
-            while len(parent_parts) > 1:
-                last_part = parent_parts[-1]
-                last_level = get_rule_level(last_part)
-                if last_level >= new_level:
-                    parent_parts.pop()
-                else:
-                    break
-            
-            parent_parts.append(paren_part)
-            if text_part:
-                lines_buffer.append(text_part)
+                rules[current].append(title)
+        elif current:
+            if not title or line.lower() != title.lower():
+                rules[current].append(line)
 
-        else:
-            lines_buffer.append(cleaned)
-    
-    commit_buffer()
-    return {k: v for k, v in rules.items() if v}
+    return {k: " ".join(v).strip() for k, v in rules.items()}
 
 
 # ─────────────────────────────────────────────────────────────────────────────
-# 3. COMPARISON & UI LOGIC (LOGIC MODIFIED HERE)
+# 3. COMPARISON & UI LOGIC
 # ─────────────────────────────────────────────────────────────────────────────
 
 def diff_unified(one: str, caa: str) -> str:
@@ -223,16 +180,14 @@ def combined_sort_key(key: str):
     else:
         return (4, key)
 
-    # MODIFICATION: More robust splitting for hierarchical keys like "108.51(3)(i)"
-    parts = re.split(r'(\d+\.\d+)|(\([a-zA-Z0-9]+\))', key)
+    parts = re.split(r'[.()]', key)
     parts = [p for p in parts if p]
 
     for part in parts:
-        num_match = re.match(r'^\d+\.\d+$', part)
-        if num_match:
-             sortable_tuple += tuple( (1, int(x)) for x in part.split('.'))
+        if part.isdigit():
+            sortable_tuple += ((1, int(part)),)
         else:
-             sortable_tuple += ((2, part.lower()),)
+            sortable_tuple += ((2, part.lower()),)
     return sortable_tuple
 
 
@@ -263,6 +218,7 @@ def save_clean_and_dirty_versions(dirty_one, dirty_caa, clean_one, clean_caa, fi
     return filename
 
 
+# --- STAGE 1: Process PDFs and prepare for user review ---
 def stage1_process_and_review(part, onereg_pdf, caa_pdf):
     if not (onereg_pdf and caa_pdf):
         raise gr.Error("Please upload both PDF files.")
@@ -306,6 +262,7 @@ def stage1_process_and_review(part, onereg_pdf, caa_pdf):
         raise gr.Error(f"Failed during initial processing: {e}")
 
 
+# --- STAGE 2: Take user-cleaned text and perform the final comparison ---
 def stage2_finalize_and_compare(review_df, original_one, original_caa):
     if review_df is None or review_df.empty:
         raise gr.Error("No data to compare. Please process the files first.")
@@ -360,7 +317,7 @@ def stage2_finalize_and_compare(review_df, original_one, original_caa):
 
 
 # ─────────────────────────────────────────────────────────────────────────────
-# 4. GRADIO UI LAYOUT (UI IS IDENTICAL TO YOUR ORIGINAL SCRIPT)
+# 4. GRADIO UI LAYOUT
 # ─────────────────────────────────────────────────────────────────────────────
 
 with gr.Blocks(theme=gr.themes.Soft(), title="Dual Rule Cleaning Tool") as demo: