Upload app.py

app.py

@@ -19,7 +19,7 @@ from sentence_transformers import SentenceTransformer
 from extracted_phase2_core import AgenticSelfRAG, Chunk, K_PASSAGES
 
 
-APP_NAME = "SourceTruth Testing"
+APP_NAME = "SourceTruth"
 APP_TAGLINE = "Ask grounded questions over the preloaded Phase 2 project corpus and inspect cited evidence."
 
 APP_ROOT = Path(__file__).resolve().parent
@@ -27,13 +27,14 @@ UPLOAD_ROOT = APP_ROOT / "testing_uploads"
 LOG_ROOT = APP_ROOT / "testing_logs"
 EVENT_LOG_PATH = LOG_ROOT / "events.jsonl"
 INTERACTION_LOG_PATH = LOG_ROOT / "interactions.jsonl"
-LOCAL_CORPUS_DIR = Path(r"C:\4 Sem Project\Phase 2\phase 2 corpus")
 CORPUS_CANDIDATES = [
     APP_ROOT / "phase2_corpus",
     APP_ROOT / "phase 2 corpus",
     APP_ROOT,
-    LOCAL_CORPUS_DIR,
 ]
+LOCAL_CORPUS_DIR = os.getenv("LOCAL_CORPUS_DIR", "").strip()
+if LOCAL_CORPUS_DIR:
+    CORPUS_CANDIDATES.append(Path(LOCAL_CORPUS_DIR).expanduser())
 
 MAX_FILE_SIZE_MB = int(os.getenv("MAX_FILE_SIZE_MB", "20"))
 MAX_PAGES = int(os.getenv("MAX_PAGES", "75"))
@@ -48,10 +49,10 @@ LOAD_IN_4BIT = os.getenv("LOAD_IN_4BIT", "0") == "1"
 MAX_SUMMARY_SENTENCES = int(os.getenv("MAX_SUMMARY_SENTENCES", "3"))
 
 PRIVACY_NOTICE = (
-    "Upload only PDF files. Documents are processed only to answer your questions and "
-    "produce citations. Files are not used to train models. Interaction logs may store "
-    "an anonymized file hash, question, answer, citation, and proxy evaluation metrics "
-    "for testing analysis. Avoid uploading confidential, personal, medical, or legal files."
+    "The preloaded project PDFs are processed only to answer your questions and produce citations. "
+    "Documents are not used to train models. Interaction logs may store the question, answer, citation, "
+    "and proxy evaluation metrics for testing analysis. Avoid using the application for confidential, "
+    "personal, medical, or legal decisions without direct document verification."
 )
 
 CSS = """
@@ -69,8 +70,8 @@ CSS = """
 """
 
 PERSON_RE = re.compile(r"\b(?:Dr\.?\s+)?[A-Z][a-z]+(?:\s+[A-Z][a-z]+){1,2}\b")
-CURRENCY_RE = re.compile(r"\b(?:INR|Rs\.?)\s*[\d,]+(?:\.\d+)?\b", re.I)
-AMOUNT_RE = re.compile(r"\b(?:paid amount|amount paid|total price|price|amount)\b[:\s-]*(INR|Rs\.?)?\s*([\d,]+(?:\.\d+)?)", re.I)
+CURRENCY_RE = re.compile(r"(?:₹\s*[\d,]+(?:\.\d+)?|\b(?:INR|Rs\.?)\s*[\d,]+(?:\.\d+)?)", re.I)
+AMOUNT_RE = re.compile(r"\b(?:paid amount|amount paid|total price|price|amount|budget|cost)\b[:\s-]*(₹|INR|Rs\.?)?\s*([\d,]+(?:\.\d+)?)", re.I)
 VERSION_RE = re.compile(r"\b\d+(?:\.\d+){1,3}\b")
 DATE_RE = re.compile(
     r"\b\d{1,2}\s+(?:January|February|March|April|May|June|July|August|"
@@ -255,7 +256,7 @@ def question_plan(question: str) -> QuestionPlan:
     if any(cue in q for cue in ["how to", "how do", "how should", "steps", "process", "procedure", "workflow", "manage ", "handling "]):
         mode = "procedural"
         expected = "procedure"
-        allow_agentic_fallback = False
+        allow_agentic_fallback = True
     elif q.startswith("who") or "who is" in q or "who was" in q:
         mode = "factoid"
         expected = "person"
@@ -377,6 +378,8 @@ def cleanup_expired_sessions():
     expired: List[str] = []
     with SESSIONS_LOCK:
         for session_id, session in list(SESSIONS.items()):
+            if session_id == "phase2-corpus":
+                continue
             if session.last_activity < cutoff:
                 expired.append(session_id)
         for session_id in expired:
@@ -1032,6 +1035,12 @@ def parse_vmp_table(page_records: List[PageRecord]) -> Dict[str, Dict[str, str]]
             idx += 5
             continue
         idx += 1
+    if not rows:
+        log_event(
+            "vmp_table_parse_empty",
+            source_file="02_Validation_Master_Plan.pdf",
+            page_count=len(page_records),
+        )
     return rows
 
 
@@ -1246,16 +1255,37 @@ def ask_question(question: str, session_id: Optional[str]):
     abstained = False
     hallucination_rate = 0.0
 
+    def run_agentic_fallback() -> Tuple[str, bool, float, Optional[Chunk], Optional[str]]:
+        nonlocal output, best_chunk
+        try:
+            output = session.agent.run(question)
+            best_chunk = output.best_chunk or best_chunk
+            answer = (
+                "I don't have enough evidence in the project corpus to answer that reliably."
+                if output.abstained
+                else (output.answer or "No answer produced.")
+            )
+            return answer, output.abstained, output.hallucination_rate or 0.0, best_chunk, None
+        except Exception as exc:
+            log_event("inference_failed", session_id=session.session_id, question=question, error=str(exc))
+            return "", False, 0.0, best_chunk, str(exc)
+
     if not evidence_pairs:
         abstained = True
         answer_text = "I don't have enough evidence in the project corpus to answer that reliably."
     elif plan.mode in {"procedural", "descriptive"}:
-        if not evidence_has_expected_type(plan, evidence_sentences):
-            abstained = True
-            answer_text = "I don't have enough evidence in the project corpus to answer that reliably."
+        summary_answer = summarize_procedural_answer(evidence_pairs) if evidence_has_expected_type(plan, evidence_sentences) else None
+        if summary_answer:
+            answer_text = summary_answer
+            abstained = False
         else:
-            answer_text = summarize_procedural_answer(evidence_pairs) or "I don't have enough evidence in the project corpus to answer that reliably."
-            abstained = answer_text.startswith("I don't have enough evidence")
+            if plan.allow_agentic_fallback:
+                answer_text, abstained, hallucination_rate, best_chunk, inference_error = run_agentic_fallback()
+                if inference_error:
+                    return error_html(f"Inference failed: {inference_error}"), None, ""
+            else:
+                abstained = True
+                answer_text = "I don't have enough evidence in the project corpus to answer that reliably."
     else:
         concise_answer = concise_factoid_answer(question, plan, evidence_pairs)
         if concise_answer and evidence_has_expected_type(plan, evidence_sentences):
@@ -1264,19 +1294,9 @@ def ask_question(question: str, session_id: Optional[str]):
             abstained = True
             answer_text = "I don't have enough evidence in the project corpus to answer that reliably."
         elif plan.allow_agentic_fallback:
-            try:
-                output = session.agent.run(question)
-                best_chunk = output.best_chunk or best_chunk
-                answer_text = (
-                    "I don't have enough evidence in the project corpus to answer that reliably."
-                    if output.abstained
-                    else (output.answer or "No answer produced.")
-                )
-                abstained = output.abstained
-                hallucination_rate = output.hallucination_rate or 0.0
-            except Exception as exc:
-                log_event("inference_failed", session_id=session.session_id, question=question, error=str(exc))
-                return error_html(f"Inference failed: {exc}"), None, ""
+            answer_text, abstained, hallucination_rate, best_chunk, inference_error = run_agentic_fallback()
+            if inference_error:
+                return error_html(f"Inference failed: {inference_error}"), None, ""
         else:
             abstained = True
             answer_text = "I don't have enough evidence in the project corpus to answer that reliably."