Update app.py

Changed the rule expression logic

app.py

@@ -35,18 +35,22 @@ def extract_pdf_word(pdf_file) -> str:
     return "\n".join(filter(None, text_blocks))
 
 
+# MODIFIED FUNCTION: Improved logic for re-joining wrapped lines
 def merge_pdf_wrapped_lines(raw_text: str) -> list[str]:
-    """Re-join hard-wrapped lines from PDF extraction."""
+    """Re-join hard-wrapped lines from PDF extraction based on grammatical context."""
     merged = []
     for ln in raw_text.splitlines():
         ln_stripped = ln.strip()
-        if not ln_stripped: continue
+        if not ln_stripped:
+            continue
+        
         if merged:
             prev = merged[-1]
-            if (re.search(r'[a-z]$', prev) and re.match(r'^[\(a-z]', ln_stripped)) or \
-                    (re.search(r'\b(?:rule|may|and|or)$', prev, re.I) and re.match(r'^\d+\.\d+', ln_stripped)) or \
-                    (re.search(r'\brule\s+\d+\.$', prev, re.I) and re.match(r'^\d', ln_stripped)):
-                merged[-1] = prev + (' ' if re.search(r'[a-z]$', prev) else '') + ln_stripped
+            # YOUR INSIGHT: Merge if previous line ends with '—' or lacks closing punctuation,
+            # and the next line appears to be a continuation (starts with lowercase or parenthesis).
+            if prev.endswith('—') or \
+               (not re.search(r'[.:;)]\s*$', prev) and re.match(r'^[a-z\(]', ln_stripped)):
+                merged[-1] = prev + ' ' + ln_stripped
                 continue
         merged.append(ln_stripped)
     return merged
@@ -69,6 +73,9 @@ subpart_pat = re.compile(
     r'^\s*\d+\.\s*Subpart\s+([A-Z]{1,2})\s*[—-]\s*(.+)$',
     re.IGNORECASE
 )
+# NEW: Regex to specifically identify sub-rule paragraphs like (a), (1), (i)
+sub_rule_pat = re.compile(r'^\s*(\((?:[a-z]{1,2}|[ivx]+|\d+)\))\s*(.*)', re.IGNORECASE)
+
 
 # --- Regex for cleaning ---
 page_pat = re.compile(r'Page\s+\d+\s*/\s*\d+', re.IGNORECASE)
@@ -97,49 +104,88 @@ def clean_line(line: str, source: str) -> str:
     line = re.sub(r'\s{2,}', ' ', line)
     return line.strip()
 
+# NEW HELPER: Determines nesting level of a sub-rule, e.g., (1) is level 1, (a) is 2, (i) is 3
+def get_rule_level(paren_str):
+    content = paren_str.strip('()').lower()
+    if content.isdigit(): return 1
+    if all(c in 'ivxl' for c in content): return 3 # roman numerals
+    if content.isalpha(): return 2 # alphabetical
+    return 4 # Unknown level, treat as deeply nested
 
+# REWRITTEN FUNCTION: Stateful, hierarchical parser to correctly handle sub-rules.
 def parse_rules(text: str, source: str) -> dict[str, str]:
-    """Parses raw text into a dictionary of {rule_id: rule_text}."""
-    rules, current, title = {}, None, ""
+    """
+    Parses raw text into a dictionary of {rule_id: rule_text}.
+    This version is stateful and context-aware to handle hierarchies correctly.
+    """
+    rules = {}
+    parent_parts = []  # Tracks the current rule hierarchy, e.g., ['108.51', '(3)']
+    lines_buffer = []
+
+    def commit_buffer():
+        """Saves the buffered lines to the current rule ID."""
+        if parent_parts and lines_buffer:
+            rule_id = "".join(parent_parts)
+            # Append to existing text if rule already has a title, otherwise create it
+            existing_text = rules.get(rule_id, "")
+            new_text = " ".join(lines_buffer)
+            rules[rule_id] = (existing_text + " " + new_text).strip()
+        lines_buffer.clear()
 
     lines = merge_pdf_wrapped_lines(text)
 
-    for raw_line in lines:
-        line = clean_line(raw_line, source)
-        if not line: continue
-
-        m_ap_item = appendix_item_pat.match(line)
-        m_sp = subpart_pat.match(line)
-        m_rule = rule_pat.match(line)
-
-        new_key = None
-        new_title = ""
-
-        if m_ap_item:
-            key_parts = [m_ap_item.group(1).upper(), m_ap_item.group(2)]
-            if m_ap_item.group(3): key_parts.append(f"({m_ap_item.group(3).strip()})")
-            new_key = ".".join(key_parts)
-            new_title = m_ap_item.group('title').strip()
-        elif m_sp:
-            new_key = f"subpart-{m_sp.group(1).upper()}"
-            new_title = f"Subpart {m_sp.group(1).upper()} — {m_sp.group(2).strip()}"
-        elif m_rule:
-            base = m_rule.group('base_rule')
-            parens_str = m_rule.group('parens') or ""
-            new_key = base + "".join(re.findall(r'\([^)]+\)', parens_str))
-            new_title = m_rule.group('title').strip()
-
-        if new_key:
-            current = new_key
-            title = new_title
-            rules.setdefault(current, [])
-            if title:
-                rules[current].append(title)
-        elif current:
-            if not title or line.lower() != title.lower():
-                rules[current].append(line)
+    for line in lines:
+        cleaned = clean_line(line, source)
+        if not cleaned: continue
+
+        m_main = rule_pat.match(cleaned)
+        m_sub = sub_rule_pat.match(cleaned)
+        m_sp = subpart_pat.match(cleaned)
+
+        if m_sp:
+            commit_buffer()
+            parent_parts = [f"subpart-{m_sp.group(1).upper()}"]
+            rules["".join(parent_parts)] = f"Subpart {m_sp.group(1).upper()} — {m_sp.group(2).strip()}"
+
+        elif m_main:
+            new_base_id = m_main.group('base_rule')
+            current_base_id = parent_parts[0] if parent_parts and not parent_parts[0].startswith("subpart") else None
+            
+            # YOUR INSIGHT: A rule never refers to itself. If it's the same base ID, it's content.
+            if new_base_id == current_base_id:
+                lines_buffer.append(cleaned)
+                continue
 
-    return {k: " ".join(v).strip() for k, v in rules.items()}
+            commit_buffer()
+            parent_parts = [new_base_id]
+            title = m_main.group('title').strip()
+            if title:
+                rules["".join(parent_parts)] = title
+
+        elif m_sub and parent_parts:
+            commit_buffer()
+            paren_part = m_sub.group(1)
+            text_part = m_sub.group(2).strip()
+            new_level = get_rule_level(paren_part)
+
+            # Adjust hierarchy: pop parent parts until we are at the correct level
+            while len(parent_parts) > 1:
+                last_part = parent_parts[-1]
+                last_level = get_rule_level(last_part)
+                if last_level >= new_level:
+                    parent_parts.pop()
+                else:
+                    break
+            
+            parent_parts.append(paren_part)
+            if text_part:
+                lines_buffer.append(text_part)
+
+        else: # It's continuation text
+            lines_buffer.append(cleaned)
+    
+    commit_buffer()
+    return {k: v for k, v in rules.items() if v}
 
 
 # ─────────────────────────────────────────────────────────────────────────────
@@ -180,7 +226,7 @@ def combined_sort_key(key: str):
     else:
         return (4, key)
 
-    parts = re.split(r'[.()]', key)
+    parts = re.split(r'([()])', key) # Split but keep delimiters
     parts = [p for p in parts if p]
 
     for part in parts:
@@ -219,16 +265,23 @@ def save_clean_and_dirty_versions(dirty_one, dirty_caa, clean_one, clean_caa, fi
 
 
 # --- STAGE 1: Process PDFs and prepare for user review ---
+# MODIFIED FUNCTION: Skips ToC pages before parsing.
 def stage1_process_and_review(part, onereg_pdf, caa_pdf):
     if not (onereg_pdf and caa_pdf):
         raise gr.Error("Please upload both PDF files.")
     try:
-        # Process OneReg PDF
-        raw_one = extract_pdf_word(onereg_pdf.name)
+        # --- Process OneReg PDF (skipping ToC) ---
+        onereg_doc = fitz.open(onereg_pdf.name)
+        # OneReg ToC for Part 108 is pages 2-4 (index 1-3). Content starts on page 5 (index 4).
+        onereg_text_blocks = [page.get_text("text") for i, page in enumerate(onereg_doc) if i >= 4]
+        raw_one = "\n".join(filter(None, onereg_text_blocks))
         one_data = parse_rules(raw_one, "onereg")
 
-        # Process CAA PDF
-        raw_caa = extract_pdf_text(caa_pdf.name)
+        # --- Process CAA PDF (skipping ToC) ---
+        caa_doc = PdfReader(caa_pdf.name)
+        # CAA 'List of Rules' for Part 108 is page 4 (index 3). Content starts on page 5 (index 4).
+        caa_text_blocks = [p.extract_text() or "" for i, p in enumerate(caa_doc.pages) if i >= 4]
+        raw_caa = "\n".join(caa_text_blocks)
         caa_data = parse_rules(raw_caa, "caa")
 
         # Get all rule IDs and sort them
@@ -237,6 +290,7 @@ def stage1_process_and_review(part, onereg_pdf, caa_pdf):
             key=combined_sort_key
         )
 
+        # Filter for the relevant part, but always include subparts and appendices
         rules_to_review = [
             r for r in all_ids
             if r.startswith(f"{part}.") or r.startswith("subpart-") or re.match(r'^[A-Z]\.', r)
@@ -286,7 +340,7 @@ def stage2_finalize_and_compare(review_df, original_one, original_caa):
     sections = []
     for rule_id in all_ids:
         one_clean = clean_one_data.get(rule_id, "")
-        caa_clean = clean_caa_data.get(rule_id, "")
+        caa_clean = caa_data.get(rule_id, "")
 
         diff_html = diff_unified(one_clean, caa_clean)
 
@@ -329,7 +383,7 @@ with gr.Blocks(theme=gr.themes.Soft(), title="Dual Rule Cleaning Tool") as demo:
 
     # --- Stage 1: Inputs and Initial Processing ---
     with gr.Row():
-        part_num = gr.Textbox(label="Part Number", value="139")
+        part_num = gr.Textbox(label="Part Number", value="108")
         onereg_pdf = gr.File(label="Upload OneReg PDF")
         caa_pdf = gr.File(label="Upload CAA PDF")