Update app.py

app.py

@@ -1,8 +1,12 @@
 # ================================================================
 # Self-Sensing Concrete Assistant — Predictor (XGB) + Hybrid RAG
-# - Predictor tab: identical behavior (kept)
-# - Literature tab: Hybrid RAG; LLM runs silently when available
-# - UX: no visible "LLM & Controls" window; prediction=0.0 if incomplete
+# - Uses local 'papers/' folder for literature
+# - Robust MMR sentence selection (no list index errors)
+# - Predictor: safe model caching + safe feature alignment
+# - Stable categoricals ("NA"); no over-strict completeness gate
+# - Lightweight instrumentation (JSONL logs per RAG turn)
+# - Dark-blue theme + Evaluate tab + k-slider styling
+# - Citations use SHORT CODES (e.g., S71, S92) from filenames
 # ================================================================
 
 # ---------------------- Runtime flags (HF-safe) ----------------------
@@ -12,9 +16,9 @@ os.environ["TRANSFORMERS_NO_FLAX"] = "1"
 os.environ["TOKENIZERS_PARALLELISM"] = "false"
 
 # ------------------------------- Imports ------------------------------
-import re, time, joblib, warnings, json
+import re, joblib, warnings, json, traceback, time, uuid, subprocess, sys
 from pathlib import Path
-from typing import List, Dict, Any
+from typing import List, Dict, Any, Optional
 
 import numpy as np
 import pandas as pd
@@ -35,7 +39,7 @@ except Exception:
     BM25Okapi = None
     print("rank_bm25 not installed; BM25 disabled (TF-IDF still works).")
 
-# Optional OpenAI (for LLM paraphrase)
+# Optional OpenAI (for LLM synthesis)
 OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
 OPENAI_MODEL   = os.getenv("OPENAI_MODEL", "gpt-5")
 try:
@@ -49,6 +53,7 @@ LLM_AVAILABLE = (OPENAI_API_KEY is not None and OPENAI_API_KEY.strip() != "" and
 # ========================= Predictor (kept) =========================
 CF_COL     = "Conductive Filler Conc. (wt%)"
 TARGET_COL = "Stress GF (MPa-1)"
+CANON_NA   = "NA"  # canonical placeholder for categoricals
 
 MAIN_VARIABLES = [
     "Filler 1 Type",
@@ -105,84 +110,133 @@ CATEGORICAL_COLS = {
     "Current Type"
 }
 
-OPTIONAL_FIELDS = {
-    "Filler 2 Type",
-    "Filler 2 Diameter (µm)",
-    "Filler 2 Length (mm)",
-    "Filler 2 Dimensionality",
-}
-
-DIM_CHOICES     = ["0D", "1D", "2D", "3D", "NA"]
-CURRENT_CHOICES = ["DC", "AC", "NA"]
+DIM_CHOICES     = ["0D", "1D", "2D", "3D", CANON_NA]
+CURRENT_CHOICES = ["DC", "AC", CANON_NA]
 
 MODEL_CANDIDATES = [
     "stress_gf_xgb.joblib",
     "models/stress_gf_xgb.joblib",
     "/home/user/app/stress_gf_xgb.joblib",
+    os.getenv("MODEL_PATH", "")
 ]
 
-def _load_model_or_error():
-    for p in MODEL_CANDIDATES:
+# ---------- Model caching + status ----------
+MODEL = None
+MODEL_STATUS = "🔴 Model not loaded"
+
+def _try_load_model():
+    global MODEL, MODEL_STATUS
+    for p in [x for x in MODEL_CANDIDATES if x]:
         if os.path.exists(p):
             try:
-                return joblib.load(p)
+                MODEL = joblib.load(p)
+                MODEL_STATUS = f"🟢 Loaded model: {Path(p).name}"
+                print("[ModelLoad] Loaded:", p)
+                return
             except Exception as e:
-                return f"Could not load model from {p}: {e}"
-    return ("Model file not found. Upload your trained pipeline as "
-            "stress_gf_xgb.joblib (or put it in models/).")
+                print(f"[ModelLoad] Error from {p}: {e}")
+                traceback.print_exc()
+                MODEL = None
+    if MODEL is None:
+        MODEL_STATUS = "🔴 Model not found (place stress_gf_xgb.joblib at repo root or models/, or set MODEL_PATH)"
+        print("[ModelLoad]", MODEL_STATUS)
+
+_try_load_model()  # load at import time
+
+def _canon_cat(v: Any) -> str:
+    """Stable, canonical category placeholder normalization."""
+    if v is None:
+        return CANON_NA
+    s = str(v).strip()
+    if s == "" or s.upper() in {"N/A", "NONE", "NULL"}:
+        return CANON_NA
+    return s
+
+def _to_float_or_nan(v):
+    if v in ("", None):
+        return np.nan
+    try:
+        return float(str(v).replace(",", ""))
+    except Exception:
+        return np.nan
 
 def _coerce_to_row(form_dict: dict) -> pd.DataFrame:
     row = {}
     for col in MAIN_VARIABLES:
         v = form_dict.get(col, None)
         if col in NUMERIC_COLS:
-            if v in ("", None):
-                row[col] = np.nan
-            else:
-                try:
-                    row[col] = float(v)
-                except Exception:
-                    row[col] = np.nan
+            row[col] = _to_float_or_nan(v)
+        elif col in CATEGORICAL_COLS:
+            row[col] = _canon_cat(v)
         else:
-            row[col] = "" if v in (None, "NA") else str(v).strip()
+            s = str(v).strip() if v is not None else ""
+            row[col] = s if s else CANON_NA
     return pd.DataFrame([row], columns=MAIN_VARIABLES)
 
-def _is_complete(form_dict: dict) -> bool:
-    for col in MAIN_VARIABLES:
-        if col in OPTIONAL_FIELDS:
-            continue
-        v = form_dict.get(col, None)
-        if col in NUMERIC_COLS:
+def _align_columns_to_model(df: pd.DataFrame, mdl) -> pd.DataFrame:
+    """
+    SAFE alignment:
+    - If mdl.feature_names_in_ exists AND is a subset of df.columns (raw names), reorder to it.
+    - Else, try a Pipeline step (e.g., 'preprocessor') with feature_names_in_ subset of df.columns.
+    - Else, DO NOT align (let the pipeline handle columns by name).
+    """
+    try:
+        feat = getattr(mdl, "feature_names_in_", None)
+        if isinstance(feat, (list, np.ndarray, pd.Index)):
+            feat = list(feat)
+            if all(c in df.columns for c in feat):
+                return df[feat]
+
+        if hasattr(mdl, "named_steps"):
+            for key in ["preprocessor", "columntransformer"]:
+                if key in mdl.named_steps:
+                    step = mdl.named_steps[key]
+                    feat2 = getattr(step, "feature_names_in_", None)
+                    if isinstance(feat2, (list, np.ndarray, pd.Index)):
+                        feat2 = list(feat2)
+                        if all(c in df.columns for c in feat2):
+                            return df[feat2]
+            # fallback to first step if it exposes input names
             try:
-                if v in ("", None) or (isinstance(v, float) and np.isnan(v)):
-                    return False
+                first_key = list(mdl.named_steps.keys())[0]
+                step = mdl.named_steps[first_key]
+                feat3 = getattr(step, "feature_names_in_", None)
+                if isinstance(feat3, (list, np.ndarray, pd.Index)):
+                    feat3 = list(feat3)
+                    if all(c in df.columns for c in feat3):
+                        return df[feat3]
             except Exception:
-                return False
-        elif col in CATEGORICAL_COLS:
-            s = "" if v in (None, "NA") else str(v).strip()
-            if s == "":
-                return False
-        else:
-            s = "" if v is None else str(v).strip()
-            if s == "":
-                return False
-    return True
+                pass
+
+        return df
+    except Exception as e:
+        print(f"[Align] Skip aligning due to: {e}")
+        traceback.print_exc()
+        return df
 
 def predict_fn(**kwargs):
-    if not _is_complete(kwargs):
+    """
+    Always attempt prediction.
+    - Missing numerics -> NaN (imputer handles)
+    - Categoricals -> 'NA'
+    - If model missing or inference error -> 0.0 (keeps UI stable)
+    """
+    if MODEL is None:
         return 0.0
-    mdl = _load_model_or_error()
-    if isinstance(mdl, str):
-        return mdl
     X_new = _coerce_to_row(kwargs)
+    X_new = _align_columns_to_model(X_new, MODEL)
     try:
-        y_log = mdl.predict(X_new)        # model predicts log1p(target)
-        y = float(np.expm1(y_log)[0])     # back to original scale MPa^-1
-        if -1e-10 < y < 0:
-            y = 0.0
-        return y
+        y_raw = MODEL.predict(X_new)  # log1p or original scale depending on training
+        if getattr(MODEL, "target_is_log1p_", False):
+            y = np.expm1(y_raw)
+        else:
+            y = y_raw
+        y = float(np.asarray(y).ravel()[0])
+        return max(y, 0.0)
     except Exception as e:
-        return f"Prediction error: {e}"
+        print(f"[Predict] {e}")
+        traceback.print_exc()
+        return 0.0
 
 EXAMPLE = {
     "Filler 1 Type": "CNT",
@@ -191,7 +245,7 @@ EXAMPLE = {
     "Filler 1 Length (mm)": 1.2,
     CF_COL: 0.5,
     "Filler 2 Type": "",
-    "Filler 2 Dimensionality": "NA",
+    "Filler 2 Dimensionality": CANON_NA,
     "Filler 2 Diameter (µm)": None,
     "Filler 2 Length (mm)": None,
     "Specimen Volume (mm3)": 1000,
@@ -219,9 +273,9 @@ def _clear_all():
         if col in NUMERIC_COLS:
             cleared.append(None)
         elif col in {"Filler 1 Dimensionality", "Filler 2 Dimensionality"}:
-            cleared.append("NA")
+            cleared.append(CANON_NA)
         elif col == "Current Type":
-            cleared.append("NA")
+            cleared.append(CANON_NA)
         else:
             cleared.append("")
     return cleared
@@ -291,6 +345,7 @@ def _safe_init_st_model(name: str):
         return None
 
 def build_or_load_hybrid(pdf_dir: Path):
+    # Build or load the hybrid retriever cache
     have_cache = (TFIDF_VECT_PATH.exists() and TFIDF_MAT_PATH.exists()
                   and RAG_META_PATH.exists()
                   and (BM25_TOK_PATH.exists() or BM25Okapi is None)
@@ -343,7 +398,7 @@ def build_or_load_hybrid(pdf_dir: Path):
             emb = None
 
     joblib.dump(vectorizer, TFIDF_VECT_PATH)
-    joblib.dump(X_tfidF:=X_tfidf, TFIDF_MAT_PATH)  # assign + save
+    joblib.dump(X_tfidf, TFIDF_MAT_PATH)
     if BM25Okapi is not None:
         joblib.dump(all_tokens, BM25_TOK_PATH)
     meta.to_parquet(RAG_META_PATH, index=False)
@@ -354,9 +409,29 @@ bm25 = BM25Okapi(bm25_tokens) if (BM25Okapi is not None and bm25_tokens is not N
 st_query_model = _safe_init_st_model(os.getenv("EMB_MODEL_NAME", "sentence-transformers/all-MiniLM-L6-v2"))
 
 def _extract_page(text_chunk: str) -> str:
-    m = list(re.finditer(r"\\[\\[PAGE=(\\d+)\\]\\]", text_chunk or ""))
+    # Correct: [[PAGE=123]]
+    m = list(re.finditer(r"\[\[PAGE=(\d+)\]\]", text_chunk or ""))
     return (m[-1].group(1) if m else "?")
 
+def _short_doc_code(doc_path: str) -> str:
+    """
+    Turn a full filename like:
+      'S92-Research-on-the-self-sensing-and-mechanical-properties-of_2021_Cement-and-Co.pdf'
+    into a short code:
+      'S92'
+    For generic names, falls back to the first token of the stem.
+    """
+    if not doc_path:
+        return "Source"
+    name = Path(doc_path).name
+    stem = name.rsplit(".", 1)[0]
+    # Split on whitespace, hyphen, underscore
+    parts = re.split(r"[ \t\n\r\-_]+", stem)
+    for p in parts:
+        if p:
+            return p
+    return stem or "Source"
+
 def hybrid_search(query: str, k=8, w_tfidf=W_TFIDF_DEFAULT, w_bm25=W_BM25_DEFAULT, w_emb=W_EMB_DEFAULT):
     if rag_meta is None or rag_meta.empty:
         return pd.DataFrame()
@@ -383,7 +458,7 @@ def hybrid_search(query: str, k=8, w_tfidf=W_TFIDF_DEFAULT, w_bm25=W_BM25_DEFAUL
 
     # BM25 scores
     if bm25 is not None:
-        q_tokens = [t.lower() for t in re.findall(r"[A-Za-z0-9_#+\\-\\/\\.%%]+", query)]
+        q_tokens = [t.lower() for t in re.findall(r"[A-Za-z0-9_#+\-\/\.%]+", query)]
         bm25_scores = np.array(bm25.get_scores(q_tokens), dtype=float)
     else:
         bm25_scores = np.zeros(len(rag_meta), dtype=float); w_bm25 = 0.0
@@ -415,68 +490,114 @@ def split_sentences(text: str) -> List[str]:
     return [s for s in sents if 6 <= len(s.split()) <= 60]
 
 def mmr_select_sentences(question: str, hits: pd.DataFrame, top_n=4, pool_per_chunk=6, lambda_div=0.7):
+    """
+    Robust MMR sentence picker:
+    - Handles empty pools
+    - Clamps top_n to pool size
+    - Avoids 'list index out of range'
+    """
+    # Build pool
     pool = []
     for _, row in hits.iterrows():
-        doc = Path(row["doc_path"]).name
+        doc_code = _short_doc_code(row["doc_path"])
         page = _extract_page(row["text"])
-        for s in split_sentences(row["text"])[:pool_per_chunk]:
-            pool.append({"sent": s, "doc": doc, "page": page})
+        sents = split_sentences(row["text"])
+        if not sents:
+            continue
+        for s in sents[:max(1, int(pool_per_chunk))]:
+            pool.append({"sent": s, "doc": doc_code, "page": page})
+
     if not pool:
         return []
 
+    # Relevance vectors
     sent_texts = [p["sent"] for p in pool]
-
     use_dense = USE_DENSE and st_query_model is not None
-    if use_dense:
-        try:
+    try:
+        if use_dense:
             from sklearn.preprocessing import normalize as sk_normalize
-            texts = [question] + sent_texts
-            enc = st_query_model.encode(texts, convert_to_numpy=True)
+            enc = st_query_model.encode([question] + sent_texts, convert_to_numpy=True)
             q_vec = sk_normalize(enc[:1])[0]
             S     = sk_normalize(enc[1:])
             rel   = (S @ q_vec)
             def sim_fn(i, j): return float(S[i] @ S[j])
-        except Exception:
-            use_dense = False
+        else:
+            from sklearn.feature_extraction.text import TfidfVectorizer
+            vect = TfidfVectorizer().fit(sent_texts + [question])
+            Q = vect.transform([question]); S = vect.transform(sent_texts)
+            rel = (S @ Q.T).toarray().ravel()
+            def sim_fn(i, j):
+                num = (S[i] @ S[j].T)
+                return float(num.toarray()[0, 0]) if hasattr(num, "toarray") else float(num)
+    except Exception:
+        # Fallback: uniform relevance if vectorization fails
+        rel = np.ones(len(sent_texts), dtype=float)
+        def sim_fn(i, j): return 0.0
 
-    if not use_dense:
-        from sklearn.feature_extraction.text import TfidfVectorizer
-        vect = TfidfVectorizer().fit(sent_texts + [question])
-        Q = vect.transform([question]); S = vect.transform(sent_texts)
-        rel = (S @ Q.T).toarray().ravel()
-        def sim_fn(i, j): return float((S[i] @ S[j].T).toarray()[0, 0])
+    # Normalize lambda_div
+    lambda_div = float(np.clip(lambda_div, 0.0, 1.0))
 
-    selected, selected_idx = [], []
+    # Select first by highest relevance
     remain = list(range(len(pool)))
+    if not remain:
+        return []
     first = int(np.argmax(rel))
-    selected.append(pool[first]); selected_idx.append(first); remain.remove(first)
+    selected_idx = [first]
+    selected     = [pool[first]]
+    remain.remove(first)
 
-    while len(selected) < top_n and remain:
+    # Clamp top_n
+    max_pick = min(int(top_n), len(pool))
+    while len(selected) < max_pick and remain:
         cand_scores = []
         for i in remain:
-            sim_to_sel = max(sim_fn(i, j) for j in selected_idx) if selected_idx else 0.0
-            score = lambda_div * rel[i] - (1 - lambda_div) * sim_to_sel
+            div_i = max(sim_fn(i, j) for j in selected_idx) if selected_idx else 0.0
+            score = lambda_div * float(rel[i]) - (1.0 - lambda_div) * div_i
             cand_scores.append((score, i))
+        if not cand_scores:
+            break
         cand_scores.sort(reverse=True)
-        best_i = cand_scores[0][1]
-        selected.append(pool[best_i]); selected_idx.append(best_i); remain.remove(best_i)
+        _, best_i = cand_scores[0]
+        selected_idx.append(best_i)
+        selected.append(pool[best_i])
+        remain.remove(best_i)
+
     return selected
 
 def compose_extractive(selected: List[Dict[str, Any]]) -> str:
     if not selected:
         return ""
-    return " ".join(f"{s['sent']} ({s['doc']}, p.{s['page']})" for s in selected)
+    # Citations inside answer are short codes only, e.g. (S92), (S71)
+    return " ".join(f"{s['sent']} ({s['doc']})" for s in selected)
 
-def synthesize_with_llm(question: str, sentence_lines: List[str], model: str = None, temperature: float = 0.2) -> str:
-    if not LLM_AVAILABLE:
+# ========================= NEW: Instrumentation helpers =========================
+LOG_PATH = ARTIFACT_DIR / "rag_logs.jsonl"
+OPENAI_IN_COST_PER_1K  = float(os.getenv("OPENAI_COST_IN_PER_1K", "0"))
+OPENAI_OUT_COST_PER_1K = float(os.getenv("OPENAI_COST_OUT_PER_1K", "0"))
+
+def _safe_write_jsonl(path: Path, record: dict):
+    try:
+        with open(path, "a", encoding="utf-8") as f:
+            f.write(json.dumps(record, ensure_ascii=False) + "\n")
+    except Exception as e:
+        print("[Log] write failed:", e)
+
+def _calc_cost_usd(prompt_toks, completion_toks):
+    if prompt_toks is None or completion_toks is None:
         return None
+    return (prompt_toks / 1000.0) * OPENAI_IN_COST_PER_1K + (completion_toks / 1000.0) * OPENAI_OUT_COST_PER_1K
+
+# ----------------- Modified to return (text, usage_dict) -----------------
+def synthesize_with_llm(question: str, sentence_lines: List[str], model: str = None, temperature: float = 0.2):
+    if not LLM_AVAILABLE:
+        return None, None
     client = OpenAI(api_key=OPENAI_API_KEY)
     model = model or OPENAI_MODEL
     SYSTEM_PROMPT = (
         "You are a scientific assistant for self-sensing cementitious materials.\n"
         "Answer STRICTLY using the provided sentences.\n"
         "Do not invent facts. Keep it concise (3–6 sentences).\n"
-        "Retain inline citations like (Doc.pdf, p.X) exactly as given."
+        "Retain inline citations exactly as given (e.g., (S92), (S92; S71))."
     )
     user_prompt = (
         f"Question: {question}\n\n"
@@ -492,9 +613,19 @@ def synthesize_with_llm(question: str, sentence_lines: List[str], model: str = N
             ],
             temperature=temperature,
         )
-        return getattr(resp, "output_text", None) or str(resp)
+        out_text = getattr(resp, "output_text", None) or str(resp)
+        usage = None
+        try:
+            u = getattr(resp, "usage", None)
+            if u:
+                pt = getattr(u, "prompt_tokens", None) if hasattr(u, "prompt_tokens") else u.get("prompt_tokens", None)
+                ct = getattr(u, "completion_tokens", None) if hasattr(u, "completion_tokens") else u.get("completion_tokens", None)
+                usage = {"prompt_tokens": pt, "completion_tokens": ct}
+        except Exception:
+            usage = None
+        return out_text, usage
     except Exception:
-        return None
+        return None, None
 
 def rag_reply(
     question: str,
@@ -509,42 +640,159 @@ def rag_reply(
     w_bm25: float  = W_BM25_DEFAULT,
     w_emb: float   = W_EMB_DEFAULT
 ) -> str:
+    run_id = str(uuid.uuid4())
+    t0_total = time.time()
+    t0_retr  = time.time()
+
+    # --- Retrieval ---
     hits = hybrid_search(question, k=k, w_tfidf=w_tfidf, w_bm25=w_bm25, w_emb=w_emb)
-    if hits is None or hits.empty:
-        return "No indexed PDFs found. Upload PDFs to the 'papers/' folder and reload the Space."
+    t1_retr = time.time()
+    latency_ms_retriever = int((t1_retr - t0_retr) * 1000)
 
+    if hits is None or hits.empty:
+        final = "No indexed PDFs found. Upload PDFs to the 'papers/' folder and reload the Space."
+        record = {
+            "run_id": run_id,
+            "ts": int(time.time()*1000),
+            "inputs": {
+                "question": question, "top_k": int(k), "n_sentences": int(n_sentences),
+                "w_tfidf": float(w_tfidf), "w_bm25": float(w_bm25), "w_emb": float(w_emb),
+                "use_llm": bool(use_llm), "model": model, "temperature": float(temperature)
+            },
+            "retrieval": {"hits": [], "latency_ms_retriever": latency_ms_retriever},
+            "output": {"final_answer": final, "used_sentences": []},
+            "latency_ms_total": int((time.time()-t0_total)*1000),
+            "openai": None
+        }
+        _safe_write_jsonl(LOG_PATH, record)
+        return final
+
+    # Select sentences
     selected = mmr_select_sentences(question, hits, top_n=int(n_sentences), pool_per_chunk=6, lambda_div=0.7)
-    header_cites = "; ".join(f"{Path(r['doc_path']).name} (p.{_extract_page(r['text'])})" for _, r in hits.head(6).iterrows())
-    srcs = {Path(r['doc_path']).name for _, r in hits.iterrows()}
-    coverage_note = "" if len(srcs) >= 3 else f"\n\n> Note: Only {len(srcs)} unique source(s) contributed. Add more PDFs or increase Top-K."
 
-    # Hidden policy: if strict==True → no paraphrasing; else try LLM if available
+    # Header citations: short codes only, joined by '; ' (e.g., "S55; S71; S92")
+    header_codes = []
+    for _, r in hits.head(6).iterrows():
+        code = _short_doc_code(r["doc_path"])
+        if code not in header_codes:
+            header_codes.append(code)
+    header_cites = "; ".join(header_codes)
+    src_codes = set(header_codes)
+    coverage_note = "" if len(src_codes) >= 3 else f"\n\n> Note: Only {len(src_codes)} unique source(s) contributed. Add more PDFs or increase Top-K."
+
+    # Prepare retrieval list for logging (full filenames kept here)
+    retr_list = []
+    for _, r in hits.iterrows():
+        retr_list.append({
+            "doc": Path(r["doc_path"]).name,
+            "page": _extract_page(r["text"]),
+            "score_tfidf": float(r.get("score_tfidf", 0.0)),
+            "score_bm25": float(r.get("score_bm25", 0.0)),
+            "score_dense": float(r.get("score_dense", 0.0)),
+            "combo_score": float(r.get("score", 0.0)),
+        })
+
+    # Strict quotes only (no LLM)
     if strict_quotes_only:
         if not selected:
-            return f"**Quoted Passages:**\n\n---\n" + "\n\n".join(hits['text'].tolist()[:2]) + f"\n\n**Citations:** {header_cites}{coverage_note}"
-        msg = "**Quoted Passages:**\n- " + "\n- ".join(f"{s['sent']} ({s['doc']}, p.{s['page']})" for s in selected)
-        msg += f"\n\n**Citations:** {header_cites}{coverage_note}"
-        if include_passages:
-            msg += "\n\n---\n" + "\n\n".join(hits['text'].tolist()[:2])
-        return msg
-
+            final = (
+                "**Quoted Passages:**\n\n---\n" +
+                "\n\n".join(hits['text'].tolist()[:2]) +
+                f"\n\n**Citations:** {header_cites}{coverage_note}"
+            )
+        else:
+            bullets = "\n- ".join(f"{s['sent']} ({s['doc']})" for s in selected)
+            final = f"**Quoted Passages:**\n- {bullets}\n\n**Citations:** {header_cites}{coverage_note}"
+            if include_passages:
+                final += "\n\n---\n" + "\n\n".join(hits['text'].tolist()[:2])
+
+        record = {
+            "run_id": run_id,
+            "ts": int(time.time()*1000),
+            "inputs": {
+                "question": question, "top_k": int(k), "n_sentences": int(n_sentences),
+                "w_tfidf": float(w_tfidf), "w_bm25": float(w_bm25), "w_emb": float(w_emb),
+                "use_llm": False, "model": None, "temperature": float(temperature)
+            },
+            "retrieval": {"hits": retr_list, "latency_ms_retriever": latency_ms_retriever},
+            "output": {
+                "final_answer": final,
+                "used_sentences": [{"sent": s["sent"], "doc": s["doc"], "page": s["page"]} for s in selected]
+            },
+            "latency_ms_total": int((time.time()-t0_total)*1000),
+            "openai": None
+        }
+        _safe_write_jsonl(LOG_PATH, record)
+        return final
+
+    # Extractive or LLM synthesis
     extractive = compose_extractive(selected)
+    llm_usage = None
+    llm_latency_ms = None
     if use_llm and selected:
-        lines = [f"{s['sent']} ({s['doc']}, p.{s['page']})" for s in selected]
-        llm_text = synthesize_with_llm(question, lines, model=model, temperature=temperature)
+        # Lines already carry short-code citations, e.g. "... (S92)"
+        lines = [f"{s['sent']} ({s['doc']})" for s in selected]
+        t0_llm = time.time()
+        llm_text, llm_usage = synthesize_with_llm(question, lines, model=model, temperature=temperature)
+        t1_llm = time.time()
+        llm_latency_ms = int((t1_llm - t0_llm) * 1000)
+
         if llm_text:
-            msg = f"**Answer (LLM synthesis):** {llm_text}\n\n**Citations:** {header_cites}{coverage_note}"
+            final = f"**Answer (LLM synthesis):** {llm_text}\n\n**Citations:** {header_cites}{coverage_note}"
             if include_passages:
-                msg += "\n\n---\n" + "\n\n".join(hits['text'].tolist()[:2])
-            return msg
-
-    if not extractive:
-        return f"**Answer:** Here are relevant passages.\n\n**Citations:** {header_cites}{coverage_note}\n\n---\n" + "\n\n".join(hits['text'].tolist()[:2])
-
-    msg = f"**Answer:** {extractive}\n\n**Citations:** {header_cites}{coverage_note}"
-    if include_passages:
-        msg += "\n\n---\n" + "\n\n".join(hits['text'].tolist()[:2])
-    return msg
+                final += "\n\n---\n" + "\n\n".join(hits['text'].tolist()[:2])
+        else:
+            if not extractive:
+                final = (
+                    f"**Answer:** Here are relevant passages.\n\n"
+                    f"**Citations:** {header_cites}{coverage_note}\n\n---\n" +
+                    "\n\n".join(hits['text'].tolist()[:2])
+                )
+            else:
+                final = f"**Answer:** {extractive}\n\n**Citations:** {header_cites}{coverage_note}"
+                if include_passages:
+                    final += "\n\n---\n" + "\n\n".join(hits['text'].tolist()[:2])
+    else:
+        if not extractive:
+            final = (
+                f"**Answer:** Here are relevant passages.\n\n"
+                f"**Citations:** {header_cites}{coverage_note}\n\n---\n" +
+                "\n\n".join(hits['text'].tolist()[:2])
+            )
+        else:
+            final = f"**Answer:** {extractive}\n\n**Citations:** {header_cites}{coverage_note}"
+            if include_passages:
+                final += "\n\n---\n" + "\n\n".join(hits['text'].tolist()[:2])
+
+    # --------- Log full run ---------
+    prompt_toks = llm_usage.get("prompt_tokens") if llm_usage else None
+    completion_toks = llm_usage.get("completion_tokens") if llm_usage else None
+    cost_usd = _calc_cost_usd(prompt_toks, completion_toks)
+
+    total_ms = int((time.time() - t0_total) * 1000)
+    record = {
+        "run_id": run_id,
+        "ts": int(time.time()*1000),
+        "inputs": {
+            "question": question, "top_k": int(k), "n_sentences": int(n_sentences),
+            "w_tfidf": float(w_tfidf), "w_bm25": float(w_bm25), "w_emb": float(w_emb),
+            "use_llm": bool(use_llm), "model": model, "temperature": float(temperature)
+        },
+        "retrieval": {"hits": retr_list, "latency_ms_retriever": latency_ms_retriever},
+        "output": {
+            "final_answer": final,
+            "used_sentences": [{"sent": s['sent'], "doc": s['doc'], "page": s['page']} for s in selected]
+        },
+        "latency_ms_total": total_ms,
+        "latency_ms_llm": llm_latency_ms,
+        "openai": {
+            "prompt_tokens": prompt_toks,
+            "completion_tokens": completion_toks,
+            "cost_usd": cost_usd
+        } if use_llm else None
+    }
+    _safe_write_jsonl(LOG_PATH, record)
+    return final
 
 def rag_chat_fn(message, history, top_k, n_sentences, include_passages,
                 use_llm, model_name, temperature, strict_quotes_only,
@@ -584,7 +832,7 @@ input[type="checkbox"], .gr-checkbox, .gr-checkbox > * { pointer-events: auto !i
 .gr-checkbox label, .gr-check-radio label { pointer-events: auto !important; cursor: pointer; }
 #rag-tab input[type="checkbox"] { accent-color: #60a5fa !important; }
 
-/* RAG tab background and elements */
+/* RAG tab styling */
 #rag-tab .block, #rag-tab .group, #rag-tab .accordion {
     background: linear-gradient(160deg, #1f2937 0%, #14532d 55%, #0b3b68 100%) !important;
     border-radius: 12px;
@@ -611,8 +859,173 @@ input[type="checkbox"], .gr-checkbox, .gr-checkbox > * { pointer-events: auto !i
     color: #eef6ff !important;
 }
 
+/* Evaluate tab dark/high-contrast styling */
+#eval-tab .block, #eval-tab .group, #eval-tab .accordion {
+    background: linear-gradient(165deg, #0a0f1f 0%, #0d1a31 60%, #0a1c2e 100%) !important;
+    border-radius: 12px;
+    border: 1px solid rgba(139, 197, 255, 0.28);
+}
+#eval-tab label, #eval-tab .markdown, #eval-tab .prose, #eval-tab p, #eval-tab span {
+    color: #e6f2ff !important;
+}
+#eval-tab input, #eval-tab .gr-file, #eval-tab .scroll-hide, #eval-tab textarea, #eval-tab select {
+    background: rgba(8, 13, 26, 0.9) !important;
+    border: 1px solid #3b82f6 !important;
+    color: #dbeafe !important;
+}
+#eval-tab input[type="range"] { accent-color: #22c55e !important; }
+#eval-tab button { 
+    border-radius: 10px !important; 
+    font-weight: 700 !important; 
+    background: #0ea5e9 !important; 
+    color: #001321 !important; 
+    border: 1px solid #7dd3fc !important;
+}
+#eval-tab .gr-json, #eval-tab .markdown pre, #eval-tab .markdown code {
+    background: rgba(2, 6, 23, 0.85) !important;
+    color: #e2e8f0 !important;
+    border: 1px solid rgba(148, 163, 184, 0.3) !important;
+    border-radius: 10px !important;
+}
+
 /* Predictor output emphasis */
 #pred-out .wrap { font-size: 20px; font-weight: 700; color: #ecfdf5; }
+
+/* Tab header: darker blue theme for all tabs */
+.gradio-container .tab-nav button[role="tab"] {
+  background: #0b1b34 !important;
+  color: #cfe6ff !important;
+  border: 1px solid #1e3a8a !important;
+}
+.gradio-container .tab-nav button[role="tab"][aria-selected="true"] {
+  background: #0e2a57 !important;
+  color: #e0f2fe !important;
+  border-color: #3b82f6 !important;
+}
+
+/* Evaluate tab: enforce dark-blue text for labels/marks */
+#eval-tab .label, 
+#eval-tab label, 
+#eval-tab .gr-slider .label,
+#eval-tab .wrap .label,
+#eval-tab .prose, 
+#eval-tab .markdown, 
+#eval-tab p, 
+#eval-tab span {
+  color: #cfe6ff !important;
+}
+
+/* Target the specific k-slider label strongly */
+#k-slider .label, 
+#k-slider label, 
+#k-slider .wrap .label {
+  color: #cfe6ff !important;
+  text-shadow: 0 1px 0 rgba(0,0,0,0.35);
+}
+
+/* Slider track/thumb (dark blue gradient + blue thumb) */
+#eval-tab input[type="range"] { 
+  accent-color: #3b82f6 !important;
+}
+
+/* WebKit */
+#eval-tab input[type="range"]::-webkit-slider-runnable-track {
+  height: 6px;
+  background: linear-gradient(90deg, #0b3b68, #1e3a8a);
+  border-radius: 4px;
+}
+#eval-tab input[type="range"]::-webkit-slider-thumb {
+  -webkit-appearance: none;
+  appearance: none;
+  margin-top: -6px;
+  width: 18px; height: 18px;
+  background: #1d4ed8;
+  border: 1px solid #60a5fa;
+  border-radius: 50%;
+}
+
+/* Firefox */
+#eval-tab input[type="range"]::-moz-range-track {
+  height: 6px;
+  background: linear-gradient(90deg, #0b3b68, #1e3a8a);
+  border-radius: 4px;
+}
+#eval-tab input[type="range"]::-moz-range-thumb {
+  width: 18px; height: 18px;
+  background: #1d4ed8;
+  border: 1px solid #60a5fa;
+  border-radius: 50%;
+}
+
+/* ======== PATCH: Style the File + JSON outputs by ID ======== */
+#perq-file, #agg-file {
+  background: rgba(8, 13, 26, 0.9) !important;
+  border: 1px solid #3b82f6 !important;
+  border-radius: 12px !important;
+  padding: 8px !important;
+}
+#perq-file * , #agg-file * { color: #dbeafe !important; }
+#perq-file a, #agg-file a {
+  background: #0e2a57 !important;
+  color: #e0f2fe !important;
+  border: 1px solid #60a5fa !important;
+  border-radius: 8px !important;
+  padding: 6px 10px !important;
+  text-decoration: none !important;
+}
+#perq-file a:hover, #agg-file a:hover {
+  background: #10356f !important;
+  border-color: #93c5fd !important;
+}
+/* File preview wrappers (covers multiple Gradio render modes) */
+#perq-file .file-preview, #agg-file .file-preview,
+#perq-file .wrap, #agg-file .wrap {
+  background: rgba(2, 6, 23, 0.85) !important;
+  border-radius: 10px !important;
+  border: 1px solid rgba(148,163,184,.3) !important;
+}
+
+/* JSON output: dark panel + readable text */
+#agg-json {
+  background: rgba(2, 6, 23, 0.85) !important;
+  border: 1px solid rgba(148,163,184,.35) !important;
+  border-radius: 12px !important;
+  padding: 8px !important;
+}
+#agg-json *, #agg-json .json, #agg-json .wrap { color: #e6f2ff !important; }
+#agg-json pre, #agg-json code {
+  background: rgba(4, 10, 24, 0.9) !important;
+  color: #e2e8f0 !important;
+  border: 1px solid rgba(148,163,184,.35) !important;
+  border-radius: 10px !important;
+}
+/* Tree/overflow modes */
+#agg-json [data-testid="json-tree"],
+#agg-json [role="tree"],
+#agg-json .overflow-auto {
+  background: rgba(4, 10, 24, 0.9) !important;
+  color: #e6f2ff !important;
+  border-radius: 10px !important;
+  border: 1px solid rgba(148,163,184,.35) !important;
+}
+
+/* Eval log markdown */
+#eval-log, #eval-log * { color: #cfe6ff !important; }
+#eval-log pre, #eval-log code {
+  background: rgba(2, 6, 23, 0.85) !important;
+  color: #e2e8f0 !important;
+  border: 1px solid rgba(148,163,184,.3) !important;
+  border-radius: 10px !important;
+}
+
+/* When Evaluate tab is active and JS has added .eval-active, bump contrast subtly */
+#eval-tab.eval-active .block, 
+#eval-tab.eval-active .group {
+  border-color: #60a5fa !important;
+}
+#eval-tab.eval-active .label {
+  color: #e6f2ff !important;
+}
 """
 
 theme = gr.themes.Soft(
@@ -630,11 +1043,37 @@ theme = gr.themes.Soft(
 )
 
 with gr.Blocks(css=CSS, theme=theme, fill_height=True) as demo:
+    # Optional: JS to toggle .eval-active when Evaluate tab selected
+    gr.HTML("""
+    <script>
+    (function(){
+      const applyEvalActive = () => {
+        const selected = document.querySelector('.tab-nav button[role="tab"][aria-selected="true"]');
+        const evalPanel = document.querySelector('#eval-tab');
+        if (!evalPanel) return;
+        if (selected && /Evaluate/.test(selected.textContent)) {
+          evalPanel.classList.add('eval-active');
+        } else {
+          evalPanel.classList.remove('eval-active');
+        }
+      };
+      document.addEventListener('click', function(e) {
+        if (e.target && e.target.getAttribute('role') === 'tab') {
+          setTimeout(applyEvalActive, 50);
+        }
+      }, true);
+      document.addEventListener('DOMContentLoaded', applyEvalActive);
+      setTimeout(applyEvalActive, 300);
+    })();
+    </script>
+    """)
+    
     gr.Markdown(
         "<h1 style='margin:0'>Self-Sensing Concrete Assistant</h1>"
         "<p style='opacity:.9'>"
         "Left: ML prediction for Stress Gauge Factor (original scale, MPa<sup>-1</sup>). "
-        "Right: Literature Q&A via Hybrid RAG (BM25 + TF-IDF + optional dense) with MMR sentence selection."
+        "Right: Literature Q&A via Hybrid RAG (BM25 + TF-IDF + optional dense) with MMR sentence selection. "
+        "Answers cite short document codes (e.g., <code>S71</code>, <code>S92</code>)."
         "</p>"
     )
 
@@ -644,27 +1083,27 @@ with gr.Blocks(css=CSS, theme=theme, fill_height=True) as demo:
             with gr.Row():
                 with gr.Column(scale=7):
                     with gr.Accordion("Primary conductive filler", open=True, elem_classes=["card"]):
-                        f1_type = gr.Textbox(label="Filler 1 Type", placeholder="e.g., CNT, Graphite, Steel fiber")
-                        f1_diam = gr.Number(label="Filler 1 Diameter (µm)")
-                        f1_len  = gr.Number(label="Filler 1 Length (mm)")
-                        cf_conc = gr.Number(label=f"{CF_COL}", info="Weight percent of total binder")
-                        f1_dim  = gr.Dropdown(DIM_CHOICES, value="NA", label="Filler 1 Dimensionality")
+                        f1_type = gr.Textbox(label="Filler 1 Type *", placeholder="e.g., CNT, Graphite, Steel fiber")
+                        f1_diam = gr.Number(label="Filler 1 Diameter (µm) *")
+                        f1_len  = gr.Number(label="Filler 1 Length (mm) *")
+                        cf_conc = gr.Number(label=f"{CF_COL} *", info="Weight percent of total binder")
+                        f1_dim  = gr.Dropdown(DIM_CHOICES, value=CANON_NA, label="Filler 1 Dimensionality *")
 
                     with gr.Accordion("Secondary filler (optional)", open=False, elem_classes=["card"]):
                         f2_type = gr.Textbox(label="Filler 2 Type", placeholder="Optional")
                         f2_diam = gr.Number(label="Filler 2 Diameter (µm)")
                         f2_len  = gr.Number(label="Filler 2 Length (mm)")
-                        f2_dim  = gr.Dropdown(DIM_CHOICES, value="NA", label="Filler 2 Dimensionality")
+                        f2_dim  = gr.Dropdown(DIM_CHOICES, value=CANON_NA, label="Filler 2 Dimensionality")
 
                     with gr.Accordion("Mix design & specimen", open=False, elem_classes=["card"]):
-                        spec_vol  = gr.Number(label="Specimen Volume (mm3)")
-                        probe_cnt = gr.Number(label="Probe Count")
-                        probe_mat = gr.Textbox(label="Probe Material", placeholder="e.g., Copper, Silver paste")
-                        wb        = gr.Number(label="W/B")
-                        sb        = gr.Number(label="S/B")
-                        gauge_len = gr.Number(label="Gauge Length (mm)")
-                        curing    = gr.Textbox(label="Curing Condition", placeholder="e.g., 28d water, 20°C")
-                        n_fillers = gr.Number(label="Number of Fillers")
+                        spec_vol  = gr.Number(label="Specimen Volume (mm3) *")
+                        probe_cnt = gr.Number(label="Probe Count *")
+                        probe_mat = gr.Textbox(label="Probe Material *", placeholder="e.g., Copper, Silver paste")
+                        wb        = gr.Number(label="W/B *")
+                        sb        = gr.Number(label="S/B *")
+                        gauge_len = gr.Number(label="Gauge Length (mm) *")
+                        curing    = gr.Textbox(label="Curing Condition *", placeholder="e.g., 28d water, 20°C")
+                        n_fillers = gr.Number(label="Number of Fillers *")
 
                     with gr.Accordion("Processing", open=False, elem_classes=["card"]):
                         dry_temp = gr.Number(label="Drying Temperature (°C)")
@@ -672,13 +1111,14 @@ with gr.Blocks(css=CSS, theme=theme, fill_height=True) as demo:
 
                     with gr.Accordion("Mechanical & electrical loading", open=False, elem_classes=["card"]):
                         load_rate = gr.Number(label="Loading Rate (MPa/s)")
-                        E_mod     = gr.Number(label="Modulus of Elasticity (GPa)")
-                        current   = gr.Dropdown(CURRENT_CHOICES, value="NA", label="Current Type")
+                        E_mod     = gr.Number(label="Modulus of Elasticity (GPa) *")
+                        current   = gr.Dropdown(CURRENT_CHOICES, value=CANON_NA, label="Current Type")
                         voltage   = gr.Number(label="Applied Voltage (V)")
 
                 with gr.Column(scale=5):
                     with gr.Group(elem_classes=["card"]):
                         out_pred = gr.Number(label="Predicted Stress GF (MPa-1)", value=0.0, precision=6, elem_id="pred-out")
+                        gr.Markdown(f"<small>{MODEL_STATUS}</small>")
                         with gr.Row():
                             btn_pred  = gr.Button("Predict", variant="primary")
                             btn_clear = gr.Button("Clear")
@@ -687,7 +1127,7 @@ with gr.Blocks(css=CSS, theme=theme, fill_height=True) as demo:
                     with gr.Accordion("About this model", open=False, elem_classes=["card"]):
                         gr.Markdown(
                             "- Pipeline: ColumnTransformer → (RobustScaler + OneHot) → XGBoost\n"
-                            "- Target: Stress GF (MPa<sup>-1</sup>) on original scale (model trains on log1p).\n"
+                            "- Target: Stress GF (MPa<sup>-1</sup>) on original scale (model may train on log1p; saved flag used at inference).\n"
                             "- Missing values are safely imputed per-feature.\n"
                             "- Trained columns:\n"
                             f"  `{', '.join(MAIN_VARIABLES)}`",
@@ -713,9 +1153,11 @@ with gr.Blocks(css=CSS, theme=theme, fill_height=True) as demo:
 
         # ------------------------- Literature Tab -------------------------
         with gr.Tab("📚 Ask the Literature (Hybrid RAG + MMR)", elem_id="rag-tab"):
+            pdf_count = len(list(LOCAL_PDF_DIR.glob("**/*.pdf")))
             gr.Markdown(
-                "Upload PDFs into the repository folder <code>papers/</code> then reload the Space. "
-                "Answers cite (Doc.pdf, p.X)."
+                f"Using local folder <code>papers/</code> — **{pdf_count} PDF(s)** indexed. "
+                "Upload more PDFs and reload the Space to expand coverage. "
+                "Answers cite short document codes such as <code>S71</code>, <code>S92</code>."
             )
             with gr.Row():
                 top_k = gr.Slider(5, 12, value=8, step=1, label="Top-K chunks")
@@ -727,11 +1169,11 @@ with gr.Blocks(css=CSS, theme=theme, fill_height=True) as demo:
                 w_bm25  = gr.Slider(0.0, 1.0, value=W_BM25_DEFAULT,  step=0.05, label="BM25 weight")
                 w_emb   = gr.Slider(0.0, 1.0, value=(0.0 if not USE_DENSE else 0.40), step=0.05, label="Dense weight (set 0 if disabled)")
 
-            # ---- Hidden states for LLM behavior (no visible controls) ----
-            state_use_llm     = gr.State(LLM_AVAILABLE)   # True when key present; else False
+            # Hidden states (unchanged)
+            state_use_llm     = gr.State(LLM_AVAILABLE)
             state_model_name  = gr.State(os.getenv("OPENAI_MODEL", OPENAI_MODEL))
             state_temperature = gr.State(0.2)
-            state_strict      = gr.State(False)           # hidden: default to not-strict
+            state_strict      = gr.State(False)
 
             gr.ChatInterface(
                 fn=rag_chat_fn,
@@ -741,9 +1183,73 @@ with gr.Blocks(css=CSS, theme=theme, fill_height=True) as demo:
                     w_tfidf, w_bm25, w_emb
                 ],
                 title="Literature Q&A",
-                description="Hybrid retrieval with diversity. Answers carry inline (Doc, p.X) citations."
+                description="Hybrid retrieval with diversity. Answers carry inline short-code citations (e.g., (S92), (S71))."
             )
 
+        # ====== Evaluate (Gold vs Logs) ======
+        with gr.Tab("📏 Evaluate (Gold vs Logs)", elem_id="eval-tab"):
+            gr.Markdown("Upload your **gold.csv** and compute metrics against the app logs.")
+            with gr.Row():
+                gold_file = gr.File(label="gold.csv", file_types=[".csv"], interactive=True)
+                k_slider  = gr.Slider(3, 12, value=8, step=1, label="k for Hit/Recall/nDCG", elem_id="k-slider")
+            with gr.Row():
+                btn_eval = gr.Button("Compute Metrics", variant="primary")
+            with gr.Row():
+                out_perq = gr.File(label="Per-question metrics (CSV)", elem_id="perq-file")
+                out_agg  = gr.File(label="Aggregate metrics (JSON)", elem_id="agg-file")
+            out_json = gr.JSON(label="Aggregate summary", elem_id="agg-json")
+            out_log  = gr.Markdown(label="Run log", elem_id="eval-log")
+
+            def _run_eval_inproc(gold_path: str, k: int = 8):
+                import json as _json
+                out_dir = str(ARTIFACT_DIR)
+                logs = str(LOG_PATH)
+                cmd = [
+                    sys.executable, "rag_eval_metrics.py",
+                    "--gold_csv", gold_path,
+                    "--logs_jsonl", logs,
+                    "--k", str(k),
+                    "--out_dir", out_dir
+                ]
+                try:
+                    p = subprocess.run(cmd, capture_output=True, text=True, check=False)
+                    stdout = p.stdout or ""
+                    stderr = p.stderr or ""
+                    perq = ARTIFACT_DIR / "metrics_per_question.csv"
+                    agg  = ARTIFACT_DIR / "metrics_aggregate.json"
+                    agg_json = {}
+                    if agg.exists():
+                        agg_json = _json.loads(agg.read_text(encoding="utf-8"))
+                    report = "```\n" + (stdout.strip() or "(no stdout)") + ("\n" + stderr.strip() if stderr else "") + "\n```"
+                    return (str(perq) if perq.exists() else None,
+                            str(agg)  if agg.exists()  else None,
+                            agg_json,
+                            report)
+                except Exception as e:
+                    return (None, None, {}, f"**Eval error:** {e}")
+
+            def _eval_wrapper(gf, k):
+                from pathlib import Path as _Path
+                if gf is None:
+                    default_gold = _Path("gold.csv")
+                    if not default_gold.exists():
+                        return None, None, {}, "**No gold.csv provided or found in repo root.**"
+                    gold_path = str(default_gold)
+                else:
+                    gold_path = gf.name
+                return _run_eval_inproc(gold_path, int(k))
+
+            btn_eval.click(_eval_wrapper, inputs=[gold_file, k_slider],
+                           outputs=[out_perq, out_agg, out_json, out_log])
+
 # ------------- Launch -------------
 if __name__ == "__main__":
     demo.queue().launch()
+
+    # After launch: export a simple list of PDFs as paper_list.csv
+    import os as _os
+    import pandas as _pd
+    folder = "papers"
+    files = sorted(_os.listdir(folder)) if _os.path.exists(folder) else []
+    _pd.DataFrame({"doc": files}).to_csv("paper_list.csv", index=False)
+    print("✅ Saved paper_list.csv with", len(files), "papers")