Upload structural_patterns.py

structural_patterns.py

@@ -0,0 +1,388 @@
+"""
+MACI — Structural Pattern Layer (Phase 4)
+-------------------------------------------
+Built in direct response to SRB (Shariyah Review Bureau) review, 2026-07-31.
+Arabic coverage added 2026-08-07, ahead of SRB follow-up meeting.
+
+WHY THIS EXISTS:
+The core XLM-R classifier is a language-pattern signal layer — it flags
+text that reads like a known violation category (Riba, Gharar, etc).
+SRB's review showed it correctly catches overt language-level issues
+but misses violations that are STRUCTURAL: they only become apparent
+by recognizing a known prohibited contract *pattern*, not a single
+alarming phrase.
+
+This module is a separate, deterministic, rule-based layer that runs
+ALONGSIDE the ML classifier — not instead of it. It targets five
+specific structural patterns, each modeled directly on a real clause
+SRB identified as missed:
+
+  1. Bay' al-Inah / mandatory buy-back structure   (Schedule 4)
+  2. Disguised Riba via "late payment as income"     (Clause 10.6)
+  3. Unilateral post-execution price variation       (Clause 10.7)
+  4. Risk/ownership decoupling (absolute payment
+     obligation regardless of asset condition)       (Clause 10.5)
+  5. Total liability waiver regardless of defect     (Clause 7.3)
+
+LANGUAGE COVERAGE:
+Each pattern now matches both English and Arabic phrasing. Arabic
+regexes were built by translating the five real clauses, then
+validated against reworded/paraphrased Arabic (not just the direct
+translations) to reduce overfitting to exact wording.
+
+HONESTY NOTE — READ BEFORE PRESENTING:
+- This is pattern-matching on KNOWN structural red flags, not general
+  reasoning. It will NOT catch every possible structural violation.
+- The Arabic patterns were built and tested by a non-native-Arabic-
+  speaking developer (with AI assistance) using standard Islamic
+  finance terminology. They are NOT yet reviewed by a native Arabic
+  speaker or Shari'a-trained Arabic legal reviewer. Treat Arabic
+  coverage as a good-faith first pass, not certified/validated
+  coverage, until a native/scholar review has happened.
+- Arabic morphology (verb conjugation, attached prefixes/suffixes)
+  makes literal phrase matching more fragile than in English. Recall
+  on genuinely novel Arabic phrasing (beyond what's been tested here)
+  is unverified and likely lower than English recall.
+- Present this as: "targeted structural pattern detection, validated
+  against real SRB findings in English, with a first-pass Arabic
+  extension tested against paraphrase variants — not yet
+  scholar-reviewed for Arabic."
+"""
+
+import re
+from dataclasses import dataclass, field
+from typing import List
+
+
+@dataclass
+class StructuralFlag:
+    pattern_id: str
+    name: str
+    violation_type: str
+    aaoifi_standard: str
+    severity: str
+    matched_text: str
+    explanation: str
+
+
+@dataclass
+class StructuralPattern:
+    pattern_id: str
+    name: str
+    violation_type: str
+    aaoifi_standard: str
+    severity: str
+    explanation: str
+    # A match requires ALL regexes in `all_of` to hit within a proximity
+    # window (order-independent), which avoids false positives from any
+    # single generic phrase appearing anywhere in a long document.
+    all_of: List[str] = field(default_factory=list)
+    # If ANY of these regexes also matches within the same window, the
+    # match is suppressed. Used to rule out genuine exceptions/negations
+    # that would otherwise trip the all_of conditions (e.g. a clause that
+    # explicitly says a price CANNOT be changed, discussing that fact,
+    # will still contain the words "price"/"changed"/"discretion" nearby).
+    none_of: List[str] = field(default_factory=list)
+
+
+STRUCTURAL_PATTERNS: List[StructuralPattern] = [
+
+    StructuralPattern(
+        pattern_id="SP-01",
+        name="Bay' al-Inah / Mandatory Buy-Back Structure",
+        violation_type="Structural — disguised loan via buy-back",
+        aaoifi_standard="AAOIFI Shari'a Standard No. 8 (Murabaha), §3/1/2 — prohibition on Bay' al-Inah",
+        severity="CRITICAL",
+        explanation=(
+            "Clause requires the buyer to resell the same goods back to the "
+            "seller (or its affiliate) at a pre-agreed price. This is the "
+            "classic Bay' al-Inah structure — economically equivalent to an "
+            "interest-bearing loan disguised as two sales, and is prohibited "
+            "under mainstream AAOIFI-referenced Shari'a standards regardless "
+            "of the language used to describe it as a genuine sale."
+        ),
+        all_of=[
+            r"(resell|sell\s*back|re-?purchase|buy[-\s]?back|"
+            r"يعيد\s*بيع|إعادة\s*بيع|بيع\s*نفس|يشتري\s*مرة\s*أخرى|إعادة\s*الشراء|"
+            r"(?:بيع|شراء).{0,15}مرة\s*أخرى|مرة\s*أخرى.{0,15}(?:بيع|شراء)|"
+            r"دوبارہ\s*فروخت|واپس\s*فروخت|دوبارہ.{0,30}فروخت|فروخت.{0,30}دوبارہ)",
+            r"((the\s+)?(same\s+|identical\s+)?(goods|commodit(?:y|ies)|asset)s?|"
+            r"البضاعة|السلعة|الأصل|سامان|اثاثہ|اثاثے)",
+            r"(to\s+the\s+seller|to\s+the\s+bank|to\s+the\s+financier|"
+            r"its\s+designated\s+affiliate|to\s+the\s+original|"
+            r"إلى\s*البائع|للبائع|إلى\s*البنك|للبنك|الجهة\s*التابعة|"
+            r"بيچنے\s*والے\s*کو|بيچنے\s*والے\s*یا|بینک\s*کو)",
+            r"((pre[-\s]?agreed|fixed\s+in\s+advance|predetermined|agreed[-\s]?upon)\s+price|"
+            r"price\s+(?:is\s+|shall\s+be\s+|was\s+)?"
+            r"(pre[-\s]?agreed|fixed\s+in\s+advance|predetermined|agreed[-\s]?upon)|"
+            r"سعر\s*متفق\s*عليه\s*مسبقاً|بسعر\s*محدد\s*مسبقاً|سعر\s*ثابت\s*مسبقاً|"
+            r"پہلے\s*سے\s*طے\s*شدہ\s*قیمت|طے\s*شدہ\s*قیمت|مقررہ\s*قیمت)",
+        ],
+    ),
+
+    StructuralPattern(
+        pattern_id="SP-02",
+        name="Disguised Riba via Late-Payment-as-Income",
+        violation_type="Structural — interest disguised as penalty/fee income",
+        aaoifi_standard="AAOIFI Shari'a Standard No. 3 (Default in Payment by a Debtor), §2 — late payment charges must go to charity, not seller income",
+        severity="CRITICAL",
+        explanation=(
+            "Clause imposes a recurring percentage-based late payment charge "
+            "that explicitly becomes part of the seller's own income/profit. "
+            "Under AAOIFI SS3, a compliant late-payment clause can only "
+            "impose a charge that is donated to charity — retaining it as "
+            "seller income is functionally identical to compound interest "
+            "on a defaulted debt, i.e. disguised Riba."
+        ),
+        all_of=[
+            r"(late\s+payment|overdue|delay(?:ed)?\s+in\s+payment|"
+            r"تأخر|تأخير|تاخیر|تاخیری)",
+            r"(charge|fee|penalty|surcharge|رسوم|غرامة|چارج|جرمانہ)",
+            r"(\d+(\.\d+)?\s*%\s*per\s*(week|month|day|annum)|"
+            r"(?:نسبة\s*)?\d+(?:\.\d+)?\s*%\s*(?:أسبوعياً|شهرياً|يومياً|سنوياً)|"
+            r"\d+(?:\.\d+)?\s*%\s*(?:ہفتہ\s*وار|ماہانہ|یومیہ|سالانہ))",
+            r"((seller'?s?|bank'?s?|financier'?s?)\s+(income|profit|revenue)|"
+            r"(retain(?:ed)?|form\s+part\s+of).{0,25}(income|profit|revenue)|"
+            r"دخل\s*البائع|دخل\s*البنك|جزءاً\s*من\s*دخل|"
+            r"آمدنی\s*کا\s*حصہ|آمدنی)",
+        ],
+    ),
+
+    StructuralPattern(
+        pattern_id="SP-03",
+        name="Unilateral Post-Execution Price Variation",
+        violation_type="Structural — Gharar via undisclosed/variable price",
+        aaoifi_standard="AAOIFI Shari'a Standard No. 8 (Murabaha), §2/2 — Murabaha price must be fixed and disclosed at contract execution",
+        severity="HIGH",
+        explanation=(
+            "Clause allows the seller to adjust the Murabaha sale price "
+            "after the agreement has been executed, at its sole discretion. "
+            "A valid Murabaha requires the cost and profit margin to be "
+            "fixed and disclosed to the buyer at the point of sale — "
+            "allowing unilateral post-execution adjustment reintroduces "
+            "exactly the price uncertainty (Gharar) that Murabaha's "
+            "disclosure requirement exists to prevent."
+        ),
+        all_of=[
+            r"(adjust(?:ed)?|vary|varied|amend(?:ed)?|chang(?:ed|e)|revis(?:e|ed)|"
+            r"تعديل|يعدل|تغيير|يغير|تبدیل)",
+            r"((sole|unfettered|absolute|unilateral)\s+discretion|"
+            r"تقديره\s*المطلق|تقديره\s*المنفرد|وفقاً\s*لتقديره|"
+            r"مطلق\s*صوابدید|اپنی\s*صوابدید)",
+            r"(price|murabaha\s+sale\s+price|sale\s+price|السعر|سعر\s*بيع\s*المرابحة|"
+            r"قیمت|مرابحہ\s*(?:فروخت\s*کی\s*)?قیمت)",
+        ],
+        none_of=[
+            r"cannot\s+be\s+chang(?:ed)?",
+            r"(shall|may|will)\s+not\s+be\s+chang(?:ed)?",
+            r"discretion\s+may\s+or\s+may\s+not\s+give\s+a\s+rebate",
+            r"\brebate\b",
+            r"لا\s*يجوز\s*تعديله",
+            r"\bخصم\s*تعجيل\b|\bخصم\s*مبكر\b",  # early-settlement rebate carve-out
+            r"تبدیل\s*نہیں\s*کیا\s*جا\s*سکے\s*گا",
+        ],
+    ),
+
+    StructuralPattern(
+        pattern_id="SP-04",
+        name="Risk/Ownership Decoupling with Absolute Payment Obligation",
+        violation_type="Structural — Gharar/unjust risk allocation without ownership",
+        aaoifi_standard="AAOIFI Shari'a Standard No. 8 (Murabaha), §5/1 — seller must bear ownership risk before/at point of sale; buyer's payment obligation cannot be wholly absolute regardless of asset condition",
+        severity="HIGH",
+        explanation=(
+            "Clause shifts all risk (damage, loss, defects) to the buyer "
+            "from before/at the point of purchase while explicitly stating "
+            "the seller bears no risk, and separately makes the buyer's "
+            "payment obligation absolute and unconditional. A core Murabaha "
+            "requirement is that the seller carries genuine ownership risk "
+            "for some period before resale — structuring the deal so the "
+            "'seller' bears zero risk while the buyer's payment is "
+            "unconditional resembles a financing arrangement dressed as a "
+            "sale, undermining the risk-sharing basis that makes Murabaha "
+            "distinct from an interest-bearing loan."
+        ),
+        all_of=[
+            r"((seller|bank|financier)\s+shall\s+bear\s+(no|none\s+of\s+the)\s+risk|"
+            r"(bank|seller|financier)\s+shall\s+bear\s+none|"
+            r"لا\s*يتحمل\s*(?:البائع|البنك|الممول|الشركة)\s*(?:أي(?:ة)?)?\s*مخاطر|"
+            r"خطرہ.{0,20}(?:برداشت\s*نہیں\s*کرے|نہیں\s*اٹھائے)\s*گا)",
+            r"(absolute\s+and\s+unconditional|unconditional.{0,15}absolute|"
+            r"under\s+all\s+circumstances|"
+            r"مطلق(?:اً)?\s*وغير\s*مشروط|غير\s*مشروط.{0,15}مطلق|"
+            r"مطلق\s*اور\s*غیر\s*مشروط)",
+        ],
+    ),
+
+    StructuralPattern(
+        pattern_id="SP-05",
+        name="Total Liability Waiver Regardless of Defect/Title",
+        violation_type="Structural — waiver of Khiyar al-'Ayb (defect option) and title warranty",
+        aaoifi_standard="AAOIFI Shari'a Standard No. 8 (Murabaha), §4/6 — buyer retains recourse for defects the seller knew or should have known about",
+        severity="MEDIUM",
+        explanation=(
+            "Clause has the buyer waive all remedy against the seller for "
+            "defects in quality, condition, quantity, description, or even "
+            "title — deemed accepted 'unconditionally and without "
+            "reservation.' A total waiver of defect recourse, especially "
+            "including title defects, goes beyond normal 'as-is' sale terms "
+            "and can void the buyer's Shari'a-recognized right to remedy "
+            "for undisclosed defects (Khiyar al-'Ayb)."
+        ),
+        all_of=[
+            r"(no\s+(further\s+)?remedy|لا\s*يحق\s*له\s*أي\s*(?:مطالبة|تعويض)\s*آخر|"
+            r"لن\s*يكون\s*له\s*أي\s*حق\s*آخر|"
+            r"کوئی\s*(?:مزید\s*)?چارہ\s*جوئی|کوئی\s*مزید\s*دعو[یٰ])",
+            r"(unconditionally\s+and\s+without\s+reservation|"
+            r"غير\s*مشروط\s*ودون\s*(?:أي\s*)?تحفظ|بشكل\s*غير\s*مشروط.{0,15}تحفظ|"
+            r"غیر\s*مشروط\s*اور\s*بلا\s*کسی\s*تحفظ)",
+        ],
+    ),
+]
+
+
+def normalize_arabic_script(s: str) -> str:
+    """
+    Normalizes Perso-Arabic script variants that look nearly identical
+    but are different Unicode codepoints, so a regex written with one
+    variant still matches text written with the other. This matters
+    because Arabic text conventionally uses U+064A/U+0643 while
+    Urdu/Farsi text conventionally uses U+06CC/U+06A9 for the same
+    visual letters — mixing them (even accidentally, e.g. when a
+    pattern is typed by hand) causes silent match failures that are
+    very hard to spot by eye, since the characters render identically.
+    """
+    s = s.replace("\u064a", "\u06cc")  # Arabic yeh -> Urdu/Farsi yeh
+    s = s.replace("\u0649", "\u06cc")  # Arabic alef maksura -> yeh
+    s = s.replace("\u0643", "\u06a9")  # Arabic kaf -> Urdu/Farsi keheh
+    return s
+
+
+def scan_structural_patterns(text: str) -> List[StructuralFlag]:
+    """
+    Runs all structural patterns against the given text.
+
+    Uses WINDOWED proximity matching: all component regexes for a pattern
+    must match within a bounded character window of each other (not just
+    anywhere in the whole input), to avoid false positives when scanning
+    long documents where unrelated clauses can each contribute one
+    matching phrase. Window size is tuned to comfortably span one clause
+    (~2-4 sentences) without spanning into unrelated adjacent clauses.
+    """
+    WINDOW = 350  # chars — roughly one clause's worth of context
+
+    flags = []
+    normalized = normalize_arabic_script(re.sub(r"\s+", " ", text))
+
+    for pattern in STRUCTURAL_PATTERNS:
+        # Find all match positions for each component regex. Regexes are
+        # also script-normalized so a pattern typed with one Arabic/Urdu
+        # letter variant still matches text using the other variant.
+        match_lists = []
+        ok = True
+        for rx in pattern.all_of:
+            rx_norm = normalize_arabic_script(rx)
+            matches = list(re.finditer(rx_norm, normalized, re.IGNORECASE))
+            if not matches:
+                ok = False
+                break
+            match_lists.append(matches)
+        if not ok:
+            continue
+
+        # Check whether there exists a combination of one match per regex
+        # such that all matches fall within WINDOW chars of each other.
+        # Anchor on the first regex's matches and check the others nearby.
+        found_cluster = None
+        for anchor in match_lists[0]:
+            center = (anchor.start() + anchor.end()) // 2
+            lo, hi = center - WINDOW, center + WINDOW
+            others_ok = True
+            cluster_positions = [anchor]
+            for other_matches in match_lists[1:]:
+                nearby = [m for m in other_matches if lo <= m.start() <= hi]
+                if not nearby:
+                    others_ok = False
+                    break
+                cluster_positions.append(nearby[0])
+            if others_ok:
+                found_cluster = cluster_positions
+                break
+
+        if found_cluster is None:
+            continue
+
+        # Build snippet around the cluster
+        starts = [m.start() for m in found_cluster]
+        ends = [m.end() for m in found_cluster]
+        snip_start = max(0, min(starts) - 40)
+        snip_end = min(len(normalized), max(ends) + 60)
+        snippet = normalized[snip_start:snip_end].strip()
+
+        # Exclusion check: if any none_of regex matches within the same
+        # window, this is a known exception/negation — suppress the flag.
+        window_text = normalized[max(0, snip_start - 100):min(len(normalized), snip_end + 100)]
+        if pattern.none_of and any(
+            re.search(normalize_arabic_script(rx), window_text, re.IGNORECASE)
+            for rx in pattern.none_of
+        ):
+            continue
+
+        flags.append(StructuralFlag(
+            pattern_id=pattern.pattern_id,
+            name=pattern.name,
+            violation_type=pattern.violation_type,
+            aaoifi_standard=pattern.aaoifi_standard,
+            severity=pattern.severity,
+            matched_text=f"...{snippet}...",
+            explanation=pattern.explanation,
+        ))
+
+    return flags
+
+
+def structural_flags_to_dict(flags: List[StructuralFlag]) -> list:
+    return [
+        {
+            "pattern_id": f.pattern_id,
+            "name": f.name,
+            "violation_type": f.violation_type,
+            "aaoifi_standard": f.aaoifi_standard,
+            "severity": f.severity,
+            "matched_text": f.matched_text,
+            "explanation": f.explanation,
+        }
+        for f in flags
+    ]
+
+
+if __name__ == "__main__":
+    import json
+    import sys
+    sys.path.insert(0, ".")
+    from test_clauses import (
+        CLAUSE_10_5, CLAUSE_10_6, CLAUSE_10_7, SCHEDULE_4, CLAUSE_7_3_DOC2,
+        CLAUSE_135092_COMPLIANT, CLAUSE_135096_COMPLIANT, CLAUSE_135093_COMPLIANT,
+    )
+
+    test_cases = [
+        ("10.5 — Risk/Ownership decoupling (SHOULD FLAG)", CLAUSE_10_5),
+        ("10.6 — Disguised Riba late payment (SHOULD FLAG)", CLAUSE_10_6),
+        ("10.7 — Unilateral price variation (SHOULD FLAG)", CLAUSE_10_7),
+        ("Schedule 4 — Bay' al-Inah buyback (SHOULD FLAG)", SCHEDULE_4),
+        ("Clause 7.3 — Total liability waiver (SHOULD FLAG)", CLAUSE_7_3_DOC2),
+        ("135092 — Actual-loss indemnity (SHOULD NOT FLAG)", CLAUSE_135092_COMPLIANT),
+        ("135096 — Actual-loss indemnity (SHOULD NOT FLAG)", CLAUSE_135096_COMPLIANT),
+        ("135093 — Tax definition (SHOULD NOT FLAG)", CLAUSE_135093_COMPLIANT),
+    ]
+
+    print("=" * 80)
+    for label, text in test_cases:
+        flags = scan_structural_patterns(text)
+        print(f"\n{label}")
+        if flags:
+            for f in flags:
+                print(f"  ✅ FLAGGED [{f.pattern_id}] {f.name} — {f.severity}")
+                print(f"     Standard: {f.aaoifi_standard}")
+        else:
+            print("  — no structural pattern matched —")
+    print("\n" + "=" * 80)