Spaces:
Paused
Paused
Upload core.py
#8
by Basementup - opened
core.py
ADDED
|
@@ -0,0 +1,623 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import hashlib
|
| 4 |
+
import re
|
| 5 |
+
import uuid
|
| 6 |
+
from dataclasses import asdict, dataclass, field
|
| 7 |
+
from datetime import datetime, timezone
|
| 8 |
+
from typing import Any, Iterable
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
APP_TITLE = "Dr Drastic: Unified Evidence Engine"
|
| 12 |
+
APP_VERSION = "2.0.0"
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
@dataclass
|
| 16 |
+
class SourceDocument:
|
| 17 |
+
doc_id: str
|
| 18 |
+
name: str
|
| 19 |
+
doc_type: str
|
| 20 |
+
extracted_text: str = ""
|
| 21 |
+
role: str = "unknown"
|
| 22 |
+
extraction_status: str = "loaded"
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
@dataclass
|
| 26 |
+
class FactEvent:
|
| 27 |
+
event_id: str
|
| 28 |
+
date: str
|
| 29 |
+
sort_date: str
|
| 30 |
+
actor: str
|
| 31 |
+
fact: str
|
| 32 |
+
source_doc: str
|
| 33 |
+
source_role: str
|
| 34 |
+
tags: list[str] = field(default_factory=list)
|
| 35 |
+
confidence: float = 0.7
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
@dataclass
|
| 39 |
+
class Contradiction:
|
| 40 |
+
contradiction_id: str
|
| 41 |
+
kind: str
|
| 42 |
+
left_doc: str
|
| 43 |
+
right_doc: str
|
| 44 |
+
issue: str
|
| 45 |
+
left_excerpt: str
|
| 46 |
+
right_excerpt: str
|
| 47 |
+
severity: str
|
| 48 |
+
confidence: float
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
@dataclass
|
| 52 |
+
class OmissionFlag:
|
| 53 |
+
doc: str
|
| 54 |
+
category: str
|
| 55 |
+
reason: str
|
| 56 |
+
confidence: float
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
@dataclass
|
| 60 |
+
class EvidenceRow:
|
| 61 |
+
item_id: str
|
| 62 |
+
source_doc: str
|
| 63 |
+
source_role: str
|
| 64 |
+
date: str
|
| 65 |
+
actor: str
|
| 66 |
+
fact: str
|
| 67 |
+
tags: list[str]
|
| 68 |
+
legal_significance: str
|
| 69 |
+
contradiction_link: str = ""
|
| 70 |
+
omission_flag: str = ""
|
| 71 |
+
confidence: float = 0.7
|
| 72 |
+
next_action: str = "Verify against the source document"
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
@dataclass
|
| 76 |
+
class AnalysisResult:
|
| 77 |
+
release_id: str
|
| 78 |
+
owner: str = ""
|
| 79 |
+
documents: list[SourceDocument] = field(default_factory=list)
|
| 80 |
+
events: list[FactEvent] = field(default_factory=list)
|
| 81 |
+
contradictions: list[Contradiction] = field(default_factory=list)
|
| 82 |
+
omissions: list[OmissionFlag] = field(default_factory=list)
|
| 83 |
+
evidence_matrix: list[EvidenceRow] = field(default_factory=list)
|
| 84 |
+
routes: list[str] = field(default_factory=list)
|
| 85 |
+
score: dict[str, Any] = field(default_factory=dict)
|
| 86 |
+
warnings: list[str] = field(default_factory=list)
|
| 87 |
+
|
| 88 |
+
def to_dict(self) -> dict[str, Any]:
|
| 89 |
+
return asdict(self)
|
| 90 |
+
|
| 91 |
+
@classmethod
|
| 92 |
+
def from_dict(cls, data: dict[str, Any] | None) -> "AnalysisResult":
|
| 93 |
+
data = data or {}
|
| 94 |
+
return cls(
|
| 95 |
+
release_id=data.get("release_id") or make_release_id(),
|
| 96 |
+
owner=data.get("owner", ""),
|
| 97 |
+
documents=[SourceDocument(**x) for x in data.get("documents", [])],
|
| 98 |
+
events=[FactEvent(**x) for x in data.get("events", [])],
|
| 99 |
+
contradictions=[Contradiction(**x) for x in data.get("contradictions", [])],
|
| 100 |
+
omissions=[OmissionFlag(**x) for x in data.get("omissions", [])],
|
| 101 |
+
evidence_matrix=[EvidenceRow(**x) for x in data.get("evidence_matrix", [])],
|
| 102 |
+
routes=list(data.get("routes", [])),
|
| 103 |
+
score=dict(data.get("score", {})),
|
| 104 |
+
warnings=list(data.get("warnings", [])),
|
| 105 |
+
)
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
PARTY_DICTIONARY = {
|
| 109 |
+
"claimant": ["dwayne", "galloway", "claimant", "complainant", "applicant"],
|
| 110 |
+
"dbl-max": ["dbl-max", "dbl max", "db l max", "seller", "vendor"],
|
| 111 |
+
"halifax": ["halifax", "bank of scotland", "lloyds", "lloyds banking group"],
|
| 112 |
+
"eversheds": ["eversheds", "eversheds sutherland"],
|
| 113 |
+
"fos": ["financial ombudsman", "ombudsman", "adjudicator", "final decision"],
|
| 114 |
+
"ebay": ["ebay", "e-bay", "marketplace"],
|
| 115 |
+
}
|
| 116 |
+
|
| 117 |
+
ROLE_HINTS = {
|
| 118 |
+
"claimant": PARTY_DICTIONARY["claimant"],
|
| 119 |
+
"respondent_bank": PARTY_DICTIONARY["halifax"],
|
| 120 |
+
"respondent_solicitor": PARTY_DICTIONARY["eversheds"] + ["solicitor", "legal team"],
|
| 121 |
+
"adjudicator": PARTY_DICTIONARY["fos"],
|
| 122 |
+
"third_party_seller": PARTY_DICTIONARY["dbl-max"],
|
| 123 |
+
"third_party_platform": PARTY_DICTIONARY["ebay"],
|
| 124 |
+
}
|
| 125 |
+
|
| 126 |
+
TAG_RULES = {
|
| 127 |
+
"vulnerability": [
|
| 128 |
+
"vulnerab", "disability", "reasonable adjustment", "protected characteristic",
|
| 129 |
+
"hardship", "medical", "pip", "accessibility",
|
| 130 |
+
],
|
| 131 |
+
"admission": ["admit", "accepted", "confirmed", "acknowledged", "admission"],
|
| 132 |
+
"regulatory": ["fca", "sra", "ehrc", "ico", "ombudsman", "fos", "gdpr"],
|
| 133 |
+
"account_interference": [
|
| 134 |
+
"account closed", "closure", "blocked", "standing order", "froze", "frozen",
|
| 135 |
+
"freeze", "clawback",
|
| 136 |
+
],
|
| 137 |
+
"evidence_failure": [
|
| 138 |
+
"ignored", "misrecorded", "not raised", "failed to", "missing", "omitted",
|
| 139 |
+
"suppressed", "misrepresent",
|
| 140 |
+
],
|
| 141 |
+
"loss_or_harm": [
|
| 142 |
+
"distress", "loss", "eviction", "malnutrition", "injury", "health decline",
|
| 143 |
+
"business loss", "commercial loss",
|
| 144 |
+
],
|
| 145 |
+
"consumer_rights": [
|
| 146 |
+
"refund", "return", "reject", "defect", "faulty", "not as described", "brake",
|
| 147 |
+
"consumer rights",
|
| 148 |
+
],
|
| 149 |
+
}
|
| 150 |
+
|
| 151 |
+
LEGAL_ROUTES = {
|
| 152 |
+
"Equality Act / vulnerability": (
|
| 153 |
+
["disability", "reasonable adjustment", "equality act", "pip", "vulnerab"],
|
| 154 |
+
"Review Equality Act 2010 service-provider duties and evidence of knowledge, "
|
| 155 |
+
"disadvantage, and proposed adjustments.",
|
| 156 |
+
),
|
| 157 |
+
"Consumer rights": (
|
| 158 |
+
["consumer rights", "right to reject", "refund", "defect", "faulty", "not as described"],
|
| 159 |
+
"Review the Consumer Rights Act 2015 issues, remedy dates, trader identity, "
|
| 160 |
+
"product evidence, and any platform protections.",
|
| 161 |
+
),
|
| 162 |
+
"Banking complaint / FCA / FOS": (
|
| 163 |
+
["fca", "chargeback", "account", "bank", "standing order", "consumer duty", "fos"],
|
| 164 |
+
"Build a dated complaint trail and check the applicable FCA DISP/FOS route, "
|
| 165 |
+
"including limitation and final-response dates.",
|
| 166 |
+
),
|
| 167 |
+
"Data protection": (
|
| 168 |
+
["subject access", "sar", "gdpr", "personal data", "data protection", "ico"],
|
| 169 |
+
"Map each data request, response deadline, missing category, and ICO escalation.",
|
| 170 |
+
),
|
| 171 |
+
"Professional conduct": (
|
| 172 |
+
["solicitor", "eversheds", "sra", "professional conduct"],
|
| 173 |
+
"Separate litigation disagreement from evidence of a potential professional-"
|
| 174 |
+
"conduct issue and verify the applicable SRA rule.",
|
| 175 |
+
),
|
| 176 |
+
"Fraud / misrepresentation": (
|
| 177 |
+
["fraud", "false representation", "misrepresent", "deceiv"],
|
| 178 |
+
"Identify the exact representation, speaker, date, falsity, reliance, and loss. "
|
| 179 |
+
"Do not label conduct criminal without evidence supporting each element.",
|
| 180 |
+
),
|
| 181 |
+
"Loss and causation": (
|
| 182 |
+
["distress", "loss", "eviction", "arrears", "injury", "commercial"],
|
| 183 |
+
"Create a loss schedule with documents, dates, causation, mitigation, and a "
|
| 184 |
+
"clearly separated estimate for each head of loss.",
|
| 185 |
+
),
|
| 186 |
+
}
|
| 187 |
+
|
| 188 |
+
DATE_TOKEN = re.compile(
|
| 189 |
+
r"\b("
|
| 190 |
+
r"\d{1,2}[/-]\d{1,2}[/-]\d{2,4}|"
|
| 191 |
+
r"\d{4}-\d{2}-\d{2}|"
|
| 192 |
+
r"\d{1,2}\s+(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|"
|
| 193 |
+
r"Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:t(?:ember)?)?|Oct(?:ober)?|"
|
| 194 |
+
r"Nov(?:ember)?|Dec(?:ember)?)\s+\d{2,4}|"
|
| 195 |
+
r"(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|"
|
| 196 |
+
r"Jul(?:y)?|Aug(?:ust)?|Sep(?:t(?:ember)?)?|Oct(?:ober)?|"
|
| 197 |
+
r"Nov(?:ember)?|Dec(?:ember)?)\s+\d{4}"
|
| 198 |
+
r")\b",
|
| 199 |
+
re.IGNORECASE,
|
| 200 |
+
)
|
| 201 |
+
|
| 202 |
+
|
| 203 |
+
def make_release_id() -> str:
|
| 204 |
+
stamp = datetime.now(timezone.utc).strftime("%Y%m%d")
|
| 205 |
+
return f"DDU-{stamp}-{uuid.uuid4().hex[:8].upper()}"
|
| 206 |
+
|
| 207 |
+
|
| 208 |
+
def normalise_whitespace(text: str) -> str:
|
| 209 |
+
text = (text or "").replace("\r\n", "\n").replace("\r", "\n")
|
| 210 |
+
text = re.sub(r"[ \t]+", " ", text)
|
| 211 |
+
return re.sub(r"\n{3,}", "\n\n", text).strip()
|
| 212 |
+
|
| 213 |
+
|
| 214 |
+
def stable_doc_id(name: str, text: str = "") -> str:
|
| 215 |
+
seed = f"{name.lower()}:{len(text)}:{text[:200]}".encode("utf-8", errors="replace")
|
| 216 |
+
return "DOC-" + hashlib.sha256(seed).hexdigest()[:8].upper()
|
| 217 |
+
|
| 218 |
+
|
| 219 |
+
def guess_role(doc_name: str, text: str) -> str:
|
| 220 |
+
haystack = f"{doc_name}\n{text[:6000]}".lower()
|
| 221 |
+
scores = {
|
| 222 |
+
role: sum(haystack.count(hint) for hint in hints)
|
| 223 |
+
for role, hints in ROLE_HINTS.items()
|
| 224 |
+
}
|
| 225 |
+
role, score = max(scores.items(), key=lambda item: item[1])
|
| 226 |
+
return role if score else "unknown"
|
| 227 |
+
|
| 228 |
+
|
| 229 |
+
def resolve_actor(text: str) -> str:
|
| 230 |
+
lower = text.lower()
|
| 231 |
+
for party, aliases in PARTY_DICTIONARY.items():
|
| 232 |
+
if any(re.search(rf"\b{re.escape(alias)}\b", lower) for alias in aliases):
|
| 233 |
+
return party
|
| 234 |
+
sender = re.search(r"\b(?:from|by)\s*:\s*([^,;\n]{2,80})", text, re.IGNORECASE)
|
| 235 |
+
return sender.group(1).strip() if sender else "unknown"
|
| 236 |
+
|
| 237 |
+
|
| 238 |
+
def normalise_date(value: str) -> tuple[str, str]:
|
| 239 |
+
raw = value.strip()
|
| 240 |
+
formats = (
|
| 241 |
+
"%d/%m/%Y", "%d-%m-%Y", "%Y-%m-%d", "%d/%m/%y", "%d-%m-%y",
|
| 242 |
+
"%d %b %Y", "%d %B %Y", "%d %b %y", "%d %B %y", "%b %Y", "%B %Y",
|
| 243 |
+
)
|
| 244 |
+
for fmt in formats:
|
| 245 |
+
try:
|
| 246 |
+
parsed = datetime.strptime(raw, fmt)
|
| 247 |
+
display = parsed.strftime("%d %B %Y") if "%d" in fmt else parsed.strftime("%B %Y")
|
| 248 |
+
return display, parsed.strftime("%Y-%m-%d")
|
| 249 |
+
except ValueError:
|
| 250 |
+
continue
|
| 251 |
+
return raw, ""
|
| 252 |
+
|
| 253 |
+
|
| 254 |
+
def tag_text(text: str) -> list[str]:
|
| 255 |
+
lower = text.lower()
|
| 256 |
+
tags = [tag for tag, keywords in TAG_RULES.items() if any(k in lower for k in keywords)]
|
| 257 |
+
return tags or ["general"]
|
| 258 |
+
|
| 259 |
+
|
| 260 |
+
def _sentences(text: str) -> Iterable[str]:
|
| 261 |
+
clean = normalise_whitespace(text)
|
| 262 |
+
for paragraph in clean.splitlines():
|
| 263 |
+
for sentence in re.split(r"(?<=[.!?])\s+", paragraph):
|
| 264 |
+
sentence = sentence.strip(" -\t")
|
| 265 |
+
if len(sentence) >= 24:
|
| 266 |
+
yield sentence
|
| 267 |
+
|
| 268 |
+
|
| 269 |
+
def _event_candidate(line: str) -> tuple[str, str] | None:
|
| 270 |
+
line = line.strip()
|
| 271 |
+
if len(line) < 8:
|
| 272 |
+
return None
|
| 273 |
+
match = DATE_TOKEN.search(line)
|
| 274 |
+
if not match:
|
| 275 |
+
return None
|
| 276 |
+
date = match.group(1)
|
| 277 |
+
before = line[:match.start()].strip(" :-|>")
|
| 278 |
+
after = line[match.end():].strip(" :-|>")
|
| 279 |
+
fact = after or before
|
| 280 |
+
return (date, fact) if len(fact) >= 8 else None
|
| 281 |
+
|
| 282 |
+
|
| 283 |
+
def extract_events(documents: list[SourceDocument], chronology: str = "") -> list[FactEvent]:
|
| 284 |
+
raw_events: list[tuple[str, str, str, str]] = []
|
| 285 |
+
if chronology.strip():
|
| 286 |
+
for line in chronology.splitlines():
|
| 287 |
+
candidate = _event_candidate(line)
|
| 288 |
+
if candidate:
|
| 289 |
+
raw_events.append((candidate[0], candidate[1], "CHRONOLOGY", "claimant"))
|
| 290 |
+
|
| 291 |
+
for doc in documents:
|
| 292 |
+
candidates = list(doc.extracted_text.splitlines()) + list(_sentences(doc.extracted_text))
|
| 293 |
+
for text in candidates[:800]:
|
| 294 |
+
candidate = _event_candidate(text)
|
| 295 |
+
if candidate:
|
| 296 |
+
raw_events.append((candidate[0], candidate[1], doc.doc_id, doc.role))
|
| 297 |
+
|
| 298 |
+
seen: set[str] = set()
|
| 299 |
+
events: list[FactEvent] = []
|
| 300 |
+
for date_raw, fact, source_doc, source_role in raw_events:
|
| 301 |
+
key = re.sub(r"\W+", " ", fact.lower()).strip()[:180]
|
| 302 |
+
if key in seen:
|
| 303 |
+
continue
|
| 304 |
+
seen.add(key)
|
| 305 |
+
display_date, sort_date = normalise_date(date_raw)
|
| 306 |
+
events.append(
|
| 307 |
+
FactEvent(
|
| 308 |
+
event_id="",
|
| 309 |
+
date=display_date,
|
| 310 |
+
sort_date=sort_date,
|
| 311 |
+
actor=resolve_actor(fact),
|
| 312 |
+
fact=re.sub(r"\s+", " ", fact).strip()[:1000],
|
| 313 |
+
source_doc=source_doc,
|
| 314 |
+
source_role=source_role,
|
| 315 |
+
tags=tag_text(fact),
|
| 316 |
+
confidence=0.82 if source_doc == "CHRONOLOGY" else 0.72,
|
| 317 |
+
)
|
| 318 |
+
)
|
| 319 |
+
events.sort(key=lambda event: (event.sort_date or "9999-99-99", event.source_doc, event.fact))
|
| 320 |
+
for index, event in enumerate(events, 1):
|
| 321 |
+
event.event_id = f"EVT-{index:03d}"
|
| 322 |
+
return events
|
| 323 |
+
|
| 324 |
+
|
| 325 |
+
def _excerpt(text: str, needle: str, radius: int = 160) -> str:
|
| 326 |
+
index = text.lower().find(needle.lower())
|
| 327 |
+
if index < 0:
|
| 328 |
+
return ""
|
| 329 |
+
value = text[max(0, index - radius): index + len(needle) + radius]
|
| 330 |
+
return re.sub(r"\s+", " ", value).strip()
|
| 331 |
+
|
| 332 |
+
|
| 333 |
+
def find_contradictions(documents: list[SourceDocument]) -> list[Contradiction]:
|
| 334 |
+
terms = {
|
| 335 |
+
"warranty": ["warranty"],
|
| 336 |
+
"refund or return": ["refund", "return", "reject"],
|
| 337 |
+
"product defect": ["defect", "faulty", "brake"],
|
| 338 |
+
"account access": ["account closed", "frozen", "blocked"],
|
| 339 |
+
"reasonable adjustment": ["reasonable adjustment", "disability"],
|
| 340 |
+
"evidence handling": ["missing evidence", "ignored evidence", "omitted"],
|
| 341 |
+
}
|
| 342 |
+
contradictions: list[Contradiction] = []
|
| 343 |
+
seen: set[tuple[str, str, str]] = set()
|
| 344 |
+
for left_index, left in enumerate(documents):
|
| 345 |
+
for right in documents[left_index + 1:]:
|
| 346 |
+
for topic, needles in terms.items():
|
| 347 |
+
left_needle = next((n for n in needles if n in left.extracted_text.lower()), "")
|
| 348 |
+
right_needle = next((n for n in needles if n in right.extracted_text.lower()), "")
|
| 349 |
+
if not left_needle or not right_needle:
|
| 350 |
+
continue
|
| 351 |
+
left_excerpt = _excerpt(left.extracted_text, left_needle)
|
| 352 |
+
right_excerpt = _excerpt(right.extracted_text, right_needle)
|
| 353 |
+
left_words = set(re.findall(r"\w+", left_excerpt.lower()))
|
| 354 |
+
right_words = set(re.findall(r"\w+", right_excerpt.lower()))
|
| 355 |
+
overlap = len(left_words & right_words) / max(1, len(left_words | right_words))
|
| 356 |
+
negation_mismatch = any(
|
| 357 |
+
marker in left_excerpt.lower() and marker not in right_excerpt.lower()
|
| 358 |
+
or marker in right_excerpt.lower() and marker not in left_excerpt.lower()
|
| 359 |
+
for marker in (" not ", " never ", " no ", " deny", "refus")
|
| 360 |
+
)
|
| 361 |
+
if overlap > 0.82 and not negation_mismatch:
|
| 362 |
+
continue
|
| 363 |
+
key = (left.doc_id, right.doc_id, topic)
|
| 364 |
+
if key in seen:
|
| 365 |
+
continue
|
| 366 |
+
seen.add(key)
|
| 367 |
+
contradictions.append(
|
| 368 |
+
Contradiction(
|
| 369 |
+
contradiction_id=f"CONTR-{len(contradictions) + 1:03d}",
|
| 370 |
+
kind=f"Potentially conflicting accounts: {topic}",
|
| 371 |
+
left_doc=left.doc_id,
|
| 372 |
+
right_doc=right.doc_id,
|
| 373 |
+
issue="The documents discuss the same topic differently. Human review is required.",
|
| 374 |
+
left_excerpt=left_excerpt[:500],
|
| 375 |
+
right_excerpt=right_excerpt[:500],
|
| 376 |
+
severity="high" if negation_mismatch else "medium",
|
| 377 |
+
confidence=0.76 if negation_mismatch else 0.58,
|
| 378 |
+
)
|
| 379 |
+
)
|
| 380 |
+
return contradictions
|
| 381 |
+
|
| 382 |
+
|
| 383 |
+
def find_omissions(documents: list[SourceDocument]) -> list[OmissionFlag]:
|
| 384 |
+
checks = {
|
| 385 |
+
"admission or acknowledgement": ["admitted", "confirmed", "acknowledged"],
|
| 386 |
+
"disability or vulnerability": ["disability", "pip", "reasonable adjustment", "vulnerab"],
|
| 387 |
+
"hardship": ["hardship", "eviction", "arrears", "financial difficulty"],
|
| 388 |
+
"product safety or compliance": ["illegal", "not compliant", "unsafe", "defect", "brake"],
|
| 389 |
+
"burden or standard of proof": ["burden", "onus", "standard of proof"],
|
| 390 |
+
"data protection": ["subject access", "sar", "gdpr", "personal data"],
|
| 391 |
+
}
|
| 392 |
+
all_text = " ".join(doc.extracted_text.lower() for doc in documents)
|
| 393 |
+
omissions: list[OmissionFlag] = []
|
| 394 |
+
decision_roles = {"adjudicator", "respondent_bank", "respondent_solicitor"}
|
| 395 |
+
for category, keywords in checks.items():
|
| 396 |
+
if not any(keyword in all_text for keyword in keywords):
|
| 397 |
+
continue
|
| 398 |
+
for doc in documents:
|
| 399 |
+
if doc.role not in decision_roles:
|
| 400 |
+
continue
|
| 401 |
+
if not any(keyword in doc.extracted_text.lower() for keyword in keywords):
|
| 402 |
+
omissions.append(
|
| 403 |
+
OmissionFlag(
|
| 404 |
+
doc=doc.doc_id,
|
| 405 |
+
category=category,
|
| 406 |
+
reason=f"Material about {category} appears elsewhere but was not detected in this document.",
|
| 407 |
+
confidence=0.68,
|
| 408 |
+
)
|
| 409 |
+
)
|
| 410 |
+
return omissions
|
| 411 |
+
|
| 412 |
+
|
| 413 |
+
def legal_significance(tags: list[str]) -> str:
|
| 414 |
+
mapping = {
|
| 415 |
+
"vulnerability": "Potential Equality Act / vulnerability issue; verify duty, knowledge, and evidence.",
|
| 416 |
+
"consumer_rights": "Potential Consumer Rights Act issue; verify trader, timing, defect, and remedy.",
|
| 417 |
+
"account_interference": "Potential banking complaint issue; verify account terms, notice, and FCA/FOS rules.",
|
| 418 |
+
"evidence_failure": "Potential evidence-handling issue; preserve originals and compare the decision trail.",
|
| 419 |
+
"regulatory": "Potential regulatory route; verify jurisdiction, deadline, and the current rule text.",
|
| 420 |
+
"admission": "Potential admission or acknowledgement; retain the complete document and context.",
|
| 421 |
+
"loss_or_harm": "Potential loss item; document amount, date, causation, and mitigation.",
|
| 422 |
+
}
|
| 423 |
+
return " ".join(mapping[tag] for tag in tags if tag in mapping) or "Review for relevance and corroboration."
|
| 424 |
+
|
| 425 |
+
|
| 426 |
+
def build_routes(documents: list[SourceDocument]) -> list[str]:
|
| 427 |
+
routes: list[str] = []
|
| 428 |
+
for title, (keywords, recommendation) in LEGAL_ROUTES.items():
|
| 429 |
+
hits: list[str] = []
|
| 430 |
+
for doc in documents:
|
| 431 |
+
for sentence in _sentences(doc.extracted_text):
|
| 432 |
+
if any(keyword in sentence.lower() for keyword in keywords):
|
| 433 |
+
hits.append(f"[{doc.doc_id}] {sentence[:260]}")
|
| 434 |
+
break
|
| 435 |
+
if len(hits) == 3:
|
| 436 |
+
break
|
| 437 |
+
if hits:
|
| 438 |
+
routes.append(f"{title}\n{recommendation}\n" + "\n".join(hits))
|
| 439 |
+
return routes or ["No specific route was detected. Review the source documents manually."]
|
| 440 |
+
|
| 441 |
+
|
| 442 |
+
def score_case(events: list[FactEvent], contradictions: list[Contradiction], omissions: list[OmissionFlag]) -> dict[str, Any]:
|
| 443 |
+
tag_counts = {tag: 0 for tag in TAG_RULES}
|
| 444 |
+
corroborated_sources = set()
|
| 445 |
+
for event in events:
|
| 446 |
+
corroborated_sources.add(event.source_doc)
|
| 447 |
+
for tag in event.tags:
|
| 448 |
+
if tag in tag_counts:
|
| 449 |
+
tag_counts[tag] += 1
|
| 450 |
+
evidence_points = min(35, len(events) * 2)
|
| 451 |
+
source_points = min(25, len(corroborated_sources) * 5)
|
| 452 |
+
issue_points = min(20, sum(1 for value in tag_counts.values() if value))
|
| 453 |
+
review_points = min(20, len(contradictions) * 2 + len(omissions))
|
| 454 |
+
completeness = min(100, evidence_points + source_points + issue_points + review_points)
|
| 455 |
+
return {
|
| 456 |
+
"completeness_score": completeness,
|
| 457 |
+
"event_count": len(events),
|
| 458 |
+
"source_count": len(corroborated_sources),
|
| 459 |
+
"tag_counts": tag_counts,
|
| 460 |
+
"contradiction_count": len(contradictions),
|
| 461 |
+
"omission_count": len(omissions),
|
| 462 |
+
"note": (
|
| 463 |
+
"This is an evidence-pack completeness heuristic, not a prediction of legal "
|
| 464 |
+
"success or damages."
|
| 465 |
+
),
|
| 466 |
+
}
|
| 467 |
+
|
| 468 |
+
|
| 469 |
+
def analyze(owner: str, documents: list[SourceDocument], chronology: str = "") -> AnalysisResult:
|
| 470 |
+
documents = [doc for doc in documents if doc.extracted_text.strip()]
|
| 471 |
+
events = extract_events(documents, chronology)
|
| 472 |
+
contradictions = find_contradictions(documents)
|
| 473 |
+
omissions = find_omissions(documents)
|
| 474 |
+
matrix: list[EvidenceRow] = []
|
| 475 |
+
for index, event in enumerate(events, 1):
|
| 476 |
+
contradiction = next(
|
| 477 |
+
(item.contradiction_id for item in contradictions if event.source_doc in (item.left_doc, item.right_doc)),
|
| 478 |
+
"",
|
| 479 |
+
)
|
| 480 |
+
omission = next((item.category for item in omissions if item.doc == event.source_doc), "")
|
| 481 |
+
matrix.append(
|
| 482 |
+
EvidenceRow(
|
| 483 |
+
item_id=f"EVD-{index:03d}",
|
| 484 |
+
source_doc=event.source_doc,
|
| 485 |
+
source_role=event.source_role,
|
| 486 |
+
date=event.date,
|
| 487 |
+
actor=event.actor,
|
| 488 |
+
fact=event.fact,
|
| 489 |
+
tags=event.tags,
|
| 490 |
+
legal_significance=legal_significance(event.tags),
|
| 491 |
+
contradiction_link=contradiction,
|
| 492 |
+
omission_flag=omission,
|
| 493 |
+
confidence=event.confidence,
|
| 494 |
+
)
|
| 495 |
+
)
|
| 496 |
+
warnings = [
|
| 497 |
+
"Automated findings are leads for human verification, not factual or legal conclusions.",
|
| 498 |
+
"Check current law, limitation dates, jurisdiction, and source context before relying on the report.",
|
| 499 |
+
]
|
| 500 |
+
if not events:
|
| 501 |
+
warnings.append("No dated events were detected. Add a line such as '20 Jan 2025: event description'.")
|
| 502 |
+
result = AnalysisResult(
|
| 503 |
+
release_id=make_release_id(),
|
| 504 |
+
owner=owner.strip(),
|
| 505 |
+
documents=documents,
|
| 506 |
+
events=events,
|
| 507 |
+
contradictions=contradictions,
|
| 508 |
+
omissions=omissions,
|
| 509 |
+
evidence_matrix=matrix,
|
| 510 |
+
routes=build_routes(documents),
|
| 511 |
+
warnings=warnings,
|
| 512 |
+
)
|
| 513 |
+
result.score = score_case(events, contradictions, omissions)
|
| 514 |
+
return result
|
| 515 |
+
|
| 516 |
+
|
| 517 |
+
def build_text_report(result: AnalysisResult) -> str:
|
| 518 |
+
score = result.score
|
| 519 |
+
lines = [
|
| 520 |
+
"=" * 78,
|
| 521 |
+
APP_TITLE.upper(),
|
| 522 |
+
f"Version: {APP_VERSION}",
|
| 523 |
+
f"Reference: {result.release_id}",
|
| 524 |
+
f"Generated: {datetime.now(timezone.utc).strftime('%d %B %Y, %H:%M UTC')}",
|
| 525 |
+
f"Owner / claimant: {result.owner or 'Not specified'}",
|
| 526 |
+
"=" * 78,
|
| 527 |
+
"",
|
| 528 |
+
"EXECUTIVE DASHBOARD",
|
| 529 |
+
"-" * 78,
|
| 530 |
+
f"Documents: {len(result.documents)}",
|
| 531 |
+
f"Events: {len(result.events)}",
|
| 532 |
+
f"Potential contradictions: {len(result.contradictions)}",
|
| 533 |
+
f"Potential omissions: {len(result.omissions)}",
|
| 534 |
+
f"Evidence-pack completeness: {score.get('completeness_score', 0)}/100",
|
| 535 |
+
score.get("note", ""),
|
| 536 |
+
"",
|
| 537 |
+
"SOURCE DOCUMENTS",
|
| 538 |
+
"-" * 78,
|
| 539 |
+
]
|
| 540 |
+
lines.extend(
|
| 541 |
+
f"{doc.doc_id} | {doc.name} | {doc.doc_type} | role={doc.role} | status={doc.extraction_status}"
|
| 542 |
+
for doc in result.documents
|
| 543 |
+
)
|
| 544 |
+
lines += ["", "CHRONOLOGY", "-" * 78]
|
| 545 |
+
for event in result.events:
|
| 546 |
+
lines += [
|
| 547 |
+
f"{event.event_id} | {event.date or 'Undated'} | {event.actor} | {event.source_doc}",
|
| 548 |
+
event.fact,
|
| 549 |
+
f"Tags: {', '.join(event.tags)} | Confidence: {event.confidence:.0%}",
|
| 550 |
+
"",
|
| 551 |
+
]
|
| 552 |
+
lines += ["POTENTIAL CONTRADICTIONS", "-" * 78]
|
| 553 |
+
if not result.contradictions:
|
| 554 |
+
lines.append("None detected.")
|
| 555 |
+
for item in result.contradictions:
|
| 556 |
+
lines += [
|
| 557 |
+
f"{item.contradiction_id} | {item.severity.upper()} | {item.kind}",
|
| 558 |
+
f"{item.left_doc}: {item.left_excerpt}",
|
| 559 |
+
f"{item.right_doc}: {item.right_excerpt}",
|
| 560 |
+
f"Review note: {item.issue}",
|
| 561 |
+
"",
|
| 562 |
+
]
|
| 563 |
+
lines += ["POTENTIAL OMISSIONS", "-" * 78]
|
| 564 |
+
if not result.omissions:
|
| 565 |
+
lines.append("None detected.")
|
| 566 |
+
for item in result.omissions:
|
| 567 |
+
lines.append(f"{item.doc} | {item.category} | {item.reason}")
|
| 568 |
+
lines += ["", "LEGAL / REGULATORY ROUTES", "-" * 78]
|
| 569 |
+
for route in result.routes:
|
| 570 |
+
lines += [route, ""]
|
| 571 |
+
lines += ["EVIDENCE MATRIX", "-" * 78]
|
| 572 |
+
for row in result.evidence_matrix:
|
| 573 |
+
lines += [
|
| 574 |
+
f"{row.item_id} | {row.date or 'Undated'} | {row.actor} | {row.source_doc}",
|
| 575 |
+
f"Fact: {row.fact}",
|
| 576 |
+
f"Significance: {row.legal_significance}",
|
| 577 |
+
f"Links: contradiction={row.contradiction_link or 'none'}; omission={row.omission_flag or 'none'}",
|
| 578 |
+
f"Next action: {row.next_action}",
|
| 579 |
+
"",
|
| 580 |
+
]
|
| 581 |
+
lines += ["WARNINGS", "-" * 78]
|
| 582 |
+
lines.extend(f"- {warning}" for warning in result.warnings)
|
| 583 |
+
return "\n".join(lines).strip() + "\n"
|
| 584 |
+
|
| 585 |
+
|
| 586 |
+
def format_panels(result: AnalysisResult) -> dict[str, str]:
|
| 587 |
+
events = "\n\n".join(
|
| 588 |
+
f"{event.event_id} | {event.date or 'Undated'} | {event.actor} | {event.source_doc}\n"
|
| 589 |
+
f"{event.fact}\nTags: {', '.join(event.tags)}"
|
| 590 |
+
for event in result.events
|
| 591 |
+
) or "No dated events detected."
|
| 592 |
+
contradictions = "\n\n".join(
|
| 593 |
+
f"{item.contradiction_id} [{item.severity.upper()}] {item.kind}\n"
|
| 594 |
+
f"{item.left_doc}: {item.left_excerpt}\n{item.right_doc}: {item.right_excerpt}\n"
|
| 595 |
+
f"{item.issue}"
|
| 596 |
+
for item in result.contradictions
|
| 597 |
+
) or "No potential contradictions detected."
|
| 598 |
+
omissions = "\n".join(
|
| 599 |
+
f"{item.doc} | {item.category} | {item.reason}" for item in result.omissions
|
| 600 |
+
) or "No potential omissions detected."
|
| 601 |
+
routes = "\n\n".join(result.routes)
|
| 602 |
+
matrix = "\n\n".join(
|
| 603 |
+
f"{row.item_id} | {row.date or 'Undated'} | {row.actor} | {row.source_doc}\n"
|
| 604 |
+
f"Fact: {row.fact}\nSignificance: {row.legal_significance}\n"
|
| 605 |
+
f"Next: {row.next_action}"
|
| 606 |
+
for row in result.evidence_matrix
|
| 607 |
+
) or "No evidence rows generated."
|
| 608 |
+
dashboard = (
|
| 609 |
+
f"Reference: {result.release_id}\n"
|
| 610 |
+
f"Documents: {len(result.documents)} | Events: {len(result.events)} | "
|
| 611 |
+
f"Contradictions: {len(result.contradictions)} | Omissions: {len(result.omissions)}\n"
|
| 612 |
+
f"Evidence-pack completeness: {result.score.get('completeness_score', 0)}/100\n"
|
| 613 |
+
f"{result.score.get('note', '')}"
|
| 614 |
+
)
|
| 615 |
+
return {
|
| 616 |
+
"dashboard": dashboard,
|
| 617 |
+
"events": events,
|
| 618 |
+
"contradictions": contradictions,
|
| 619 |
+
"omissions": omissions,
|
| 620 |
+
"routes": routes,
|
| 621 |
+
"matrix": matrix,
|
| 622 |
+
"report": build_text_report(result),
|
| 623 |
+
}
|