Update app.py

app.py

@@ -1,1221 +1,289 @@
-# ================================================================
-# Self-Sensing Concrete Assistant — Predictor (XGB) + Hybrid RAG
-# - Uses local 'papers/' folder for literature
-# - Robust MMR sentence selection (no list index errors)
-# - Predictor: safe model caching + safe feature alignment
-# - Stable categoricals ("NA"); no over-strict completeness gate
-# - Fixed [[PAGE=...]] regex
-# - NEW: Lightweight instrumentation (JSONL logs per RAG turn)
-# - UPDATED THEME: Dark-blue tabs + Evaluate tab + k-slider styling
-# - PATCH: Per-question/aggregate File + JSON outputs now dark-themed via elem_id hooks
-# - OPTIONAL JS: Adds .eval-active class when Evaluate tab is selected
-# ================================================================
+# app.py  — RAG chat + RAG evaluation (HF Space)
 
-# ---------------------- Runtime flags (HF-safe) ----------------------
 import os
-os.environ["TRANSFORMERS_NO_TF"] = "1"
-os.environ["TRANSFORMERS_NO_FLAX"] = "1"
-os.environ["TOKENIZERS_PARALLELISM"] = "false"
-
-# ------------------------------- Imports ------------------------------
-import re, joblib, warnings, json, traceback, time, uuid, subprocess, sys
 from pathlib import Path
-from typing import List, Dict, Any, Optional
+import json
 
-import numpy as np
-import pandas as pd
 import gradio as gr
 
-warnings.filterwarnings("ignore", category=UserWarning)
-
-# Optional deps (handled gracefully if missing)
-USE_DENSE = True
-try:
-    from sentence_transformers import SentenceTransformer
-except Exception:
-    USE_DENSE = False
-
-try:
-    from rank_bm25 import BM25Okapi
-except Exception:
-    BM25Okapi = None
-    print("rank_bm25 not installed; BM25 disabled (TF-IDF still works).")
-
-# Optional OpenAI (for LLM paraphrase)
-OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
-OPENAI_MODEL   = os.getenv("OPENAI_MODEL", "gpt-5")
-try:
-    from openai import OpenAI
-except Exception:
-    OpenAI = None
-
-# LLM availability flag — used internally; UI remains hidden
-LLM_AVAILABLE = (OPENAI_API_KEY is not None and OPENAI_API_KEY.strip() != "" and OpenAI is not None)
-
-# ========================= Predictor (kept) =========================
-CF_COL     = "Conductive Filler Conc. (wt%)"
-TARGET_COL = "Stress GF (MPa-1)"
-CANON_NA   = "NA"  # canonical placeholder for categoricals
-
-MAIN_VARIABLES = [
-    "Filler 1 Type",
-    "Filler 1 Diameter (µm)",
-    "Filler 1 Length (mm)",
-    CF_COL,
-    "Filler 1 Dimensionality",
-    "Filler 2 Type",
-    "Filler 2 Diameter (µm)",
-    "Filler 2 Length (mm)",
-    "Filler 2 Dimensionality",
-    "Specimen Volume (mm3)",
-    "Probe Count",
-    "Probe Material",
-    "W/B",
-    "S/B",
-    "Gauge Length (mm)",
-    "Curing Condition",
-    "Number of Fillers",
-    "Drying Temperature (°C)",
-    "Drying Duration (hr)",
-    "Loading Rate (MPa/s)",
-    "Modulus of Elasticity (GPa)",
-    "Current Type",
-    "Applied Voltage (V)"
-]
-
-NUMERIC_COLS = {
-    "Filler 1 Diameter (µm)",
-    "Filler 1 Length (mm)",
-    CF_COL,
-    "Filler 2 Diameter (µm)",
-    "Filler 2 Length (mm)",
-    "Specimen Volume (mm3)",
-    "Probe Count",
-    "W/B",
-    "S/B",
-    "Gauge Length (mm)",
-    "Number of Fillers",
-    "Drying Temperature (°C)",
-    "Drying Duration (hr)",
-    "Loading Rate (MPa/s)",
-    "Modulus of Elasticity (GPa)",
-    "Applied Voltage (V)"
-}
-
-CATEGORICAL_COLS = {
-    "Filler 1 Type",
-    "Filler 1 Dimensionality",
-    "Filler 2 Type",
-    "Filler 2 Dimensionality",
-    "Probe Material",
-    "Curing Condition",
-    "Current Type"
-}
-
-DIM_CHOICES     = ["0D", "1D", "2D", "3D", CANON_NA]
-CURRENT_CHOICES = ["DC", "AC", CANON_NA]
-
-MODEL_CANDIDATES = [
-    "stress_gf_xgb.joblib",
-    "models/stress_gf_xgb.joblib",
-    "/home/user/app/stress_gf_xgb.joblib",
-    os.getenv("MODEL_PATH", "")
-]
-
-# ---------- Model caching + status ----------
-MODEL = None
-MODEL_STATUS = "🔴 Model not loaded"
-
-def _try_load_model():
-    global MODEL, MODEL_STATUS
-    for p in [x for x in MODEL_CANDIDATES if x]:
-        if os.path.exists(p):
-            try:
-                MODEL = joblib.load(p)
-                MODEL_STATUS = f"🟢 Loaded model: {Path(p).name}"
-                print("[ModelLoad] Loaded:", p)
-                return
-            except Exception as e:
-                print(f"[ModelLoad] Error from {p}: {e}")
-                traceback.print_exc()
-                MODEL = None
-    if MODEL is None:
-        MODEL_STATUS = "🔴 Model not found (place stress_gf_xgb.joblib at repo root or models/, or set MODEL_PATH)"
-        print("[ModelLoad]", MODEL_STATUS)
-
-_try_load_model()  # load at import time
-
-def _canon_cat(v: Any) -> str:
-    """Stable, canonical category placeholder normalization."""
-    if v is None:
-        return CANON_NA
-    s = str(v).strip()
-    if s == "" or s.upper() in {"N/A", "NONE", "NULL"}:
-        return CANON_NA
-    return s
-
-def _to_float_or_nan(v):
-    if v in ("", None):
-        return np.nan
-    try:
-        return float(str(v).replace(",", ""))
-    except Exception:
-        return np.nan
-
-def _coerce_to_row(form_dict: dict) -> pd.DataFrame:
-    row = {}
-    for col in MAIN_VARIABLES:
-        v = form_dict.get(col, None)
-        if col in NUMERIC_COLS:
-            row[col] = _to_float_or_nan(v)
-        elif col in CATEGORICAL_COLS:
-            row[col] = _canon_cat(v)
-        else:
-            s = str(v).strip() if v is not None else ""
-            row[col] = s if s else CANON_NA
-    return pd.DataFrame([row], columns=MAIN_VARIABLES)
-
-def _align_columns_to_model(df: pd.DataFrame, mdl) -> pd.DataFrame:
-    """
-    SAFE alignment:
-    - If mdl.feature_names_in_ exists AND is a subset of df.columns (raw names), reorder to it.
-    - Else, try a Pipeline step (e.g., 'preprocessor') with feature_names_in_ subset of df.columns.
-    - Else, DO NOT align (let the pipeline handle columns by name).
-    """
-    try:
-        feat = getattr(mdl, "feature_names_in_", None)
-        if isinstance(feat, (list, np.ndarray, pd.Index)):
-            feat = list(feat)
-            if all(c in df.columns for c in feat):
-                return df[feat]
-
-        if hasattr(mdl, "named_steps"):
-            for key in ["preprocessor", "columntransformer"]:
-                if key in mdl.named_steps:
-                    step = mdl.named_steps[key]
-                    feat2 = getattr(step, "feature_names_in_", None)
-                    if isinstance(feat2, (list, np.ndarray, pd.Index)):
-                        feat2 = list(feat2)
-                        if all(c in df.columns for c in feat2):
-                            return df[feat2]
-            # fallback to first step if it exposes input names
-            try:
-                first_key = list(mdl.named_steps.keys())[0]
-                step = mdl.named_steps[first_key]
-                feat3 = getattr(step, "feature_names_in_", None)
-                if isinstance(feat3, (list, np.ndarray, pd.Index)):
-                    feat3 = list(feat3)
-                    if all(c in df.columns for c in feat3):
-                        return df[feat3]
-            except Exception:
-                pass
-
-        return df
-    except Exception as e:
-        print(f"[Align] Skip aligning due to: {e}")
-        traceback.print_exc()
-        return df
-
-def predict_fn(**kwargs):
-    """
-    Always attempt prediction.
-    - Missing numerics -> NaN (imputer handles)
-    - Categoricals -> 'NA'
-    - If model missing or inference error -> 0.0 (keeps UI stable)
-    """
-    if MODEL is None:
-        return 0.0
-    X_new = _coerce_to_row(kwargs)
-    X_new = _align_columns_to_model(X_new, MODEL)
-    try:
-        y_raw = MODEL.predict(X_new)  # log1p or original scale depending on training
-        if getattr(MODEL, "target_is_log1p_", False):
-            y = np.expm1(y_raw)
-        else:
-            y = y_raw
-        y = float(np.asarray(y).ravel()[0])
-        return max(y, 0.0)
-    except Exception as e:
-        print(f"[Predict] {e}")
-        traceback.print_exc()
-        return 0.0
-
-EXAMPLE = {
-    "Filler 1 Type": "CNT",
-    "Filler 1 Dimensionality": "1D",
-    "Filler 1 Diameter (µm)": 0.02,
-    "Filler 1 Length (mm)": 1.2,
-    CF_COL: 0.5,
-    "Filler 2 Type": "",
-    "Filler 2 Dimensionality": CANON_NA,
-    "Filler 2 Diameter (µm)": None,
-    "Filler 2 Length (mm)": None,
-    "Specimen Volume (mm3)": 1000,
-    "Probe Count": 2,
-    "Probe Material": "Copper",
-    "W/B": 0.4,
-    "S/B": 2.5,
-    "Gauge Length (mm)": 20,
-    "Curing Condition": "28d water, 20°C",
-    "Number of Fillers": 1,
-    "Drying Temperature (°C)": 60,
-    "Drying Duration (hr)": 24,
-    "Loading Rate (MPa/s)": 0.1,
-    "Modulus of Elasticity (GPa)": 25,
-    "Current Type": "DC",
-    "Applied Voltage (V)": 5.0,
-}
-
-def _fill_example():
-    return [EXAMPLE.get(k, None) for k in MAIN_VARIABLES]
-
-def _clear_all():
-    cleared = []
-    for col in MAIN_VARIABLES:
-        if col in NUMERIC_COLS:
-            cleared.append(None)
-        elif col in {"Filler 1 Dimensionality", "Filler 2 Dimensionality"}:
-            cleared.append(CANON_NA)
-        elif col == "Current Type":
-            cleared.append(CANON_NA)
-        else:
-            cleared.append("")
-    return cleared
-
-# ========================= Hybrid RAG =========================
-ARTIFACT_DIR    = Path("rag_artifacts"); ARTIFACT_DIR.mkdir(exist_ok=True)
-TFIDF_VECT_PATH = ARTIFACT_DIR / "tfidf_vectorizer.joblib"
-TFIDF_MAT_PATH  = ARTIFACT_DIR / "tfidf_matrix.joblib"
-BM25_TOK_PATH   = ARTIFACT_DIR / "bm25_tokens.joblib"
-EMB_NPY_PATH    = ARTIFACT_DIR / "chunk_embeddings.npy"
-RAG_META_PATH   = ARTIFACT_DIR / "chunks.parquet"
-
-LOCAL_PDF_DIR = Path("papers"); LOCAL_PDF_DIR.mkdir(exist_ok=True)
-USE_ONLINE_SOURCES = os.getenv("USE_ONLINE_SOURCES", "false").lower() == "true"
-
-W_TFIDF_DEFAULT = 0.50 if not USE_DENSE else 0.30
-W_BM25_DEFAULT  = 0.50 if not USE_DENSE else 0.30
-W_EMB_DEFAULT   = 0.00 if USE_DENSE is False else 0.40
-
-_SENT_SPLIT_RE = re.compile(r"(?<=[.!?])\s+|\n+")
-TOKEN_RE       = re.compile(r"[A-Za-z0-9_#+\-/\.%]+")
-def sent_split(text: str) -> List[str]:
-    sents = [s.strip() for s in _SENT_SPLIT_RE.split(text) if s.strip()]
-    return [s for s in sents if len(s.split()) >= 5]
-def tokenize(text: str) -> List[str]:
-    return [t.lower() for t in TOKEN_RE.findall(text)]
-
-def _extract_pdf_text(pdf_path: Path) -> str:
-    try:
-        import fitz
-        doc = fitz.open(pdf_path)
-        out = []
-        for i, page in enumerate(doc):
-            out.append(f"[[PAGE={i+1}]]\n{page.get_text('text') or ''}")
-        return "\n\n".join(out)
-    except Exception:
-        try:
-            from pypdf import PdfReader
-            reader = PdfReader(str(pdf_path))
-            out = []
-            for i, p in enumerate(reader.pages):
-                txt = p.extract_text() or ""
-                out.append(f"[[PAGE={i+1}]]\n{txt}")
-            return "\n\n".join(out)
-        except Exception as e:
-            print(f"PDF read error ({pdf_path}): {e}")
-            return ""
-
-def chunk_by_sentence_windows(text: str, win_size=8, overlap=2) -> List[str]:
-    sents = sent_split(text)
-    chunks, step = [], max(1, win_size - overlap)
-    for i in range(0, len(sents), step):
-        window = sents[i:i+win_size]
-        if not window: break
-        chunks.append(" ".join(window))
-    return chunks
-
-def _safe_init_st_model(name: str):
-    global USE_DENSE
-    if not USE_DENSE:
-        return None
-    try:
-        return SentenceTransformer(name)
-    except Exception as e:
-        print("Dense embeddings unavailable:", e)
-        USE_DENSE = False
-        return None
-
-def build_or_load_hybrid(pdf_dir: Path):
-    # Build or load the hybrid retriever cache
-    have_cache = (TFIDF_VECT_PATH.exists() and TFIDF_MAT_PATH.exists()
-                  and RAG_META_PATH.exists()
-                  and (BM25_TOK_PATH.exists() or BM25Okapi is None)
-                  and (EMB_NPY_PATH.exists() or not USE_DENSE))
-    if have_cache:
-        vectorizer = joblib.load(TFIDF_VECT_PATH)
-        X_tfidf    = joblib.load(TFIDF_MAT_PATH)
-        meta       = pd.read_parquet(RAG_META_PATH)
-        bm25_toks  = joblib.load(BM25_TOK_PATH) if BM25Okapi is not None else None
-        emb        = np.load(EMB_NPY_PATH) if (USE_DENSE and EMB_NPY_PATH.exists()) else None
-        return vectorizer, X_tfidf, meta, bm25_toks, emb
-
-    rows, all_tokens = [], []
-    pdf_paths = list(Path(pdf_dir).glob("**/*.pdf"))
-    print(f"Indexing PDFs in {pdf_dir} — found {len(pdf_paths)} files.")
-    for pdf in pdf_paths:
-        raw = _extract_pdf_text(pdf)
-        if not raw.strip():
-            continue
-        for i, ch in enumerate(chunk_by_sentence_windows(raw, win_size=8, overlap=2)):
-            rows.append({"doc_path": str(pdf), "chunk_id": i, "text": ch})
-            all_tokens.append(tokenize(ch))
-    if not rows:
-        meta = pd.DataFrame(columns=["doc_path", "chunk_id", "text"])
-        vectorizer = None; X_tfidf = None; emb = None; all_tokens = None
-        return vectorizer, X_tfidf, meta, all_tokens, emb
-
-    meta = pd.DataFrame(rows)
-    from sklearn.feature_extraction.text import TfidfVectorizer
-    vectorizer = TfidfVectorizer(
-        ngram_range=(1,2),
-        min_df=1, max_df=0.95,
-        sublinear_tf=True, smooth_idf=True,
-        lowercase=True,
-        token_pattern=r"(?u)\b\w[\w\-\./%+#]*\b"
+from rag_core import (
+    rag_reply,
+    W_TFIDF_DEFAULT,
+    W_BM25_DEFAULT,
+    W_EMB_DEFAULT,
+    LOG_PATH,
+    ARTIFACT_DIR,
+)
+from rag_eval_metrics import evaluate_rag
+
+
+# ------------------ RAG chat wrapper ------------------ #
+
+def rag_chat_fn(
+    message,
+    history,
+    top_k,
+    n_sentences,
+    include_passages,
+    w_tfidf,
+    w_bm25,
+    w_emb,
+):
+    """Gradio chat wrapper around rag_reply."""
+    if not message or not message.strip():
+        return "Ask a literature question (e.g., *How does CNT length affect gauge factor?*)."
+
+    return rag_reply(
+        question=message,
+        k=int(top_k),
+        n_sentences=int(n_sentences),
+        include_passages=bool(include_passages),
+        use_llm=False,              # retrieval-only answers
+        model=None,
+        temperature=0.2,
+        strict_quotes_only=False,
+        w_tfidf=float(w_tfidf),
+        w_bm25=float(w_bm25),
+        w_emb=float(w_emb),
+        config_id=None,             # you can later set a name if you want
     )
-    X_tfidf = vectorizer.fit_transform(meta["text"].tolist())
-
-    emb = None
-    if USE_DENSE:
-        try:
-            st_model = _safe_init_st_model(os.getenv("EMB_MODEL_NAME", "sentence-transformers/all-MiniLM-L6-v2"))
-            if st_model is not None:
-                from sklearn.preprocessing import normalize as sk_normalize
-                em = st_model.encode(meta["text"].tolist(), batch_size=64, show_progress_bar=False, convert_to_numpy=True)
-                emb = sk_normalize(em)
-                np.save(EMB_NPY_PATH, emb)
-        except Exception as e:
-            print("Dense embedding failed:", e)
-            emb = None
-
-    joblib.dump(vectorizer, TFIDF_VECT_PATH)
-    joblib.dump(X_tfidf, TFIDF_MAT_PATH)
-    if BM25Okapi is not None:
-        joblib.dump(all_tokens, BM25_TOK_PATH)
-    meta.to_parquet(RAG_META_PATH, index=False)
-    return vectorizer, X_tfidf, meta, all_tokens, emb
 
-tfidf_vectorizer, tfidf_matrix, rag_meta, bm25_tokens, emb_matrix = build_or_load_hybrid(LOCAL_PDF_DIR)
-bm25 = BM25Okapi(bm25_tokens) if (BM25Okapi is not None and bm25_tokens is not None) else None
-st_query_model = _safe_init_st_model(os.getenv("EMB_MODEL_NAME", "sentence-transformers/all-MiniLM-L6-v2"))
 
-def _extract_page(text_chunk: str) -> str:
-    # Correct: [[PAGE=123]]
-    m = list(re.finditer(r"\[\[PAGE=(\d+)\]\]", text_chunk or ""))
-    return (m[-1].group(1) if m else "?")
+# ------------------ Evaluation wrapper ------------------ #
 
-def hybrid_search(query: str, k=8, w_tfidf=W_TFIDF_DEFAULT, w_bm25=W_BM25_DEFAULT, w_emb=W_EMB_DEFAULT):
-    if rag_meta is None or rag_meta.empty:
-        return pd.DataFrame()
-
-    # Dense scores
-    if USE_DENSE and st_query_model is not None and emb_matrix is not None and w_emb > 0:
-        try:
-            from sklearn.preprocessing import normalize as sk_normalize
-            q_emb = st_query_model.encode([query], convert_to_numpy=True)
-            q_emb = sk_normalize(q_emb)[0]
-            dense_scores = emb_matrix @ q_emb
-        except Exception as e:
-            print("Dense query encoding failed:", e)
-            dense_scores = np.zeros(len(rag_meta), dtype=float); w_emb = 0.0
-    else:
-        dense_scores = np.zeros(len(rag_meta), dtype=float); w_emb = 0.0
-
-    # TF-IDF scores
-    if tfidf_vectorizer is not None and tfidf_matrix is not None:
-        q_vec = tfidf_vectorizer.transform([query])
-        tfidf_scores = (tfidf_matrix @ q_vec.T).toarray().ravel()
-    else:
-        tfidf_scores = np.zeros(len(rag_meta), dtype=float); w_tfidf = 0.0
-
-    # BM25 scores
-    if bm25 is not None:
-        q_tokens = [t.lower() for t in re.findall(r"[A-Za-z0-9_#+\-\/\.%]+", query)]
-        bm25_scores = np.array(bm25.get_scores(q_tokens), dtype=float)
-    else:
-        bm25_scores = np.zeros(len(rag_meta), dtype=float); w_bm25 = 0.0
-
-    def _norm(x):
-        x = np.asarray(x, dtype=float)
-        if np.allclose(x.max(), x.min()):
-            return np.zeros_like(x)
-        return (x - x.min()) / (x.max() - x.min())
-
-    s_dense = _norm(dense_scores)
-    s_tfidf = _norm(tfidf_scores)
-    s_bm25  = _norm(bm25_scores)
-
-    total_w = (w_tfidf + w_bm25 + w_emb) or 1.0
-    w_tfidf, w_bm25, w_emb = w_tfidf/total_w, w_bm25/total_w, w_emb/total_w
-
-    combo = w_emb * s_dense + w_tfidf * s_tfidf + w_bm25 * s_bm25
-    idx = np.argsort(-combo)[:k]
-    hits = rag_meta.iloc[idx].copy()
-    hits["score_dense"] = s_dense[idx]
-    hits["score_tfidf"] = s_tfidf[idx]
-    hits["score_bm25"]  = s_bm25[idx]
-    hits["score"]       = combo[idx]
-    return hits.reset_index(drop=True)
-
-def split_sentences(text: str) -> List[str]:
-    sents = sent_split(text)
-    return [s for s in sents if 6 <= len(s.split()) <= 60]
-
-def mmr_select_sentences(question: str, hits: pd.DataFrame, top_n=4, pool_per_chunk=6, lambda_div=0.7):
+def run_eval_ui(gold_file, k):
     """
-    Robust MMR sentence picker:
-    - Handles empty pools
-    - Clamps top_n to pool size
-    - Avoids 'list index out of range'
+    Run RAG retrieval evaluation against a gold set and return:
+      - Markdown log string
+      - metrics_per_question.csv path (for File download)
+      - metrics_aggregate.json path (for File download)
+      - metrics_by_weights.csv path (for File download)
+      - aggregate metrics dict (for JSON preview)
     """
-    # Build pool
-    pool = []
-    for _, row in hits.iterrows():
-        doc  = Path(row["doc_path"]).name
-        page = _extract_page(row["text"])
-        sents = split_sentences(row["text"])
-        if not sents:
-            continue
-        for s in sents[:max(1, int(pool_per_chunk))]:
-            pool.append({"sent": s, "doc": doc, "page": page})
-
-    if not pool:
-        return []
-
-    # Relevance vectors
-    sent_texts = [p["sent"] for p in pool]
-    use_dense = USE_DENSE and st_query_model is not None
-    try:
-        if use_dense:
-            from sklearn.preprocessing import normalize as sk_normalize
-            enc = st_query_model.encode([question] + sent_texts, convert_to_numpy=True)
-            q_vec = sk_normalize(enc[:1])[0]
-            S     = sk_normalize(enc[1:])
-            rel   = (S @ q_vec)
-            def sim_fn(i, j): return float(S[i] @ S[j])
-        else:
-            from sklearn.feature_extraction.text import TfidfVectorizer
-            vect = TfidfVectorizer().fit(sent_texts + [question])
-            Q = vect.transform([question]); S = vect.transform(sent_texts)
-            rel = (S @ Q.T).toarray().ravel()
-            def sim_fn(i, j):
-                num = (S[i] @ S[j].T)
-                return float(num.toarray()[0, 0]) if hasattr(num, "toarray") else float(num)
-    except Exception:
-        # Fallback: uniform relevance if vectorization fails
-        rel = np.ones(len(sent_texts), dtype=float)
-        def sim_fn(i, j): return 0.0
-
-    # Normalize lambda_div
-    lambda_div = float(np.clip(lambda_div, 0.0, 1.0))
-
-    # Select first by highest relevance
-    remain = list(range(len(pool)))
-    if not remain:
-        return []
-    first = int(np.argmax(rel))
-    selected_idx = [first]
-    selected     = [pool[first]]
-    remain.remove(first)
-
-    # Clamp top_n
-    max_pick = min(int(top_n), len(pool))
-    while len(selected) < max_pick and remain:
-        cand_scores = []
-        for i in remain:
-            div_i = max(sim_fn(i, j) for j in selected_idx) if selected_idx else 0.0
-            score = lambda_div * float(rel[i]) - (1.0 - lambda_div) * div_i
-            cand_scores.append((score, i))
-        if not cand_scores:
-            break
-        cand_scores.sort(reverse=True)
-        _, best_i = cand_scores[0]
-        selected_idx.append(best_i)
-        selected.append(pool[best_i])
-        remain.remove(best_i)
-
-    return selected
-
-def compose_extractive(selected: List[Dict[str, Any]]) -> str:
-    if not selected:
-        return ""
-    return " ".join(f"{s['sent']} ({s['doc']}, p.{s['page']})" for s in selected)
-
-# ========================= NEW: Instrumentation helpers =========================
-LOG_PATH = ARTIFACT_DIR / "rag_logs.jsonl"
-OPENAI_IN_COST_PER_1K  = float(os.getenv("OPENAI_COST_IN_PER_1K", "0"))
-OPENAI_OUT_COST_PER_1K = float(os.getenv("OPENAI_COST_OUT_PER_1K", "0"))
-
-def _safe_write_jsonl(path: Path, record: dict):
-    try:
-        with open(path, "a", encoding="utf-8") as f:
-            f.write(json.dumps(record, ensure_ascii=False) + "\n")
-    except Exception as e:
-        print("[Log] write failed:", e)
-
-def _calc_cost_usd(prompt_toks, completion_toks):
-    if prompt_toks is None or completion_toks is None:
-        return None
-    return (prompt_toks / 1000.0) * OPENAI_IN_COST_PER_1K + (completion_toks / 1000.0) * OPENAI_OUT_COST_PER_1K
-
-# ----------------- Modified to return (text, usage_dict) -----------------
-def synthesize_with_llm(question: str, sentence_lines: List[str], model: str = None, temperature: float = 0.2):
-    if not LLM_AVAILABLE:
-        return None, None
-    client = OpenAI(api_key=OPENAI_API_KEY)
-    model = model or OPENAI_MODEL
-    SYSTEM_PROMPT = (
-        "You are a scientific assistant for self-sensing cementitious materials.\n"
-        "Answer STRICTLY using the provided sentences.\n"
-        "Do not invent facts. Keep it concise (3–6 sentences).\n"
-        "Retain inline citations like (Doc.pdf, p.X) exactly as given."
-    )
-    user_prompt = (
-        f"Question: {question}\n\n"
-        f"Use ONLY these sentences to answer; keep their inline citations:\n" +
-        "\n".join(f"- {s}" for s in sentence_lines)
-    )
-    try:
-        resp = client.responses.create(
-            model=model,
-            input=[
-                {"role": "system", "content": SYSTEM_PROMPT},
-                {"role": "user", "content": user_prompt},
-            ],
-            temperature=temperature,
-        )
-        out_text = getattr(resp, "output_text", None) or str(resp)
-        usage = None
-        try:
-            u = getattr(resp, "usage", None)
-            if u:
-                pt = getattr(u, "prompt_tokens", None) if hasattr(u, "prompt_tokens") else u.get("prompt_tokens", None)
-                ct = getattr(u, "completion_tokens", None) if hasattr(u, "completion_tokens") else u.get("completion_tokens", None)
-                usage = {"prompt_tokens": pt, "completion_tokens": ct}
-        except Exception:
-            usage = None
-        return out_text, usage
-    except Exception:
-        return None, None
-
-def rag_reply(
-    question: str,
-    k: int = 8,
-    n_sentences: int = 4,
-    include_passages: bool = False,
-    use_llm: bool = False,
-    model: str = None,
-    temperature: float = 0.2,
-    strict_quotes_only: bool = False,
-    w_tfidf: float = W_TFIDF_DEFAULT,
-    w_bm25: float  = W_BM25_DEFAULT,
-    w_emb: float   = W_EMB_DEFAULT
-) -> str:
-    run_id = str(uuid.uuid4())
-    t0_total = time.time()
-    t0_retr  = time.time()
-
-    # --- Retrieval ---
-    hits = hybrid_search(question, k=k, w_tfidf=w_tfidf, w_bm25=w_bm25, w_emb=w_emb)
-    t1_retr = time.time()
-    latency_ms_retriever = int((t1_retr - t0_retr) * 1000)
-
-    if hits is None or hits.empty:
-        final = "No indexed PDFs found. Upload PDFs to the 'papers/' folder and reload the Space."
-        record = {
-            "run_id": run_id,
-            "ts": int(time.time()*1000),
-            "inputs": {
-                "question": question, "top_k": int(k), "n_sentences": int(n_sentences),
-                "w_tfidf": float(w_tfidf), "w_bm25": float(w_bm25), "w_emb": float(w_emb),
-                "use_llm": bool(use_llm), "model": model, "temperature": float(temperature)
-            },
-            "retrieval": {"hits": [], "latency_ms_retriever": latency_ms_retriever},
-            "output": {"final_answer": final, "used_sentences": []},
-            "latency_ms_total": int((time.time()-t0_total)*1000),
-            "openai": None
-        }
-        _safe_write_jsonl(LOG_PATH, record)
-        return final
-
-    # Select sentences
-    selected = mmr_select_sentences(question, hits, top_n=int(n_sentences), pool_per_chunk=6, lambda_div=0.7)
-    header_cites = "; ".join(f"{Path(r['doc_path']).name} (p.{_extract_page(r['text'])})" for _, r in hits.head(6).iterrows())
-    srcs = {Path(r['doc_path']).name for _, r in hits.iterrows()}
-    coverage_note = "" if len(srcs) >= 3 else f"\n\n> Note: Only {len(srcs)} unique source(s) contributed. Add more PDFs or increase Top-K."
-
-    # Prepare retrieval list for logging
-    retr_list = []
-    for _, r in hits.iterrows():
-        retr_list.append({
-            "doc": Path(r["doc_path"]).name,
-            "page": _extract_page(r["text"]),
-            "score_tfidf": float(r.get("score_tfidf", 0.0)),
-            "score_bm25": float(r.get("score_bm25", 0.0)),
-            "score_dense": float(r.get("score_dense", 0.0)),
-            "combo_score": float(r.get("score", 0.0)),
-        })
-
-    # Strict quotes only (no LLM)
-    if strict_quotes_only:
-        if not selected:
-            final = f"**Quoted Passages:**\n\n---\n" + "\n\n".join(hits['text'].tolist()[:2]) + f"\n\n**Citations:** {header_cites}{coverage_note}"
-        else:
-            final = "**Quoted Passages:**\n- " + "\n- ".join(f"{s['sent']} ({s['doc']}, p.{s['page']})" for s in selected)
-            final += f"\n\n**Citations:** {header_cites}{coverage_note}"
-            if include_passages:
-                final += "\n\n---\n" + "\n\n".join(hits['text'].tolist()[:2])
-
-        record = {
-            "run_id": run_id,
-            "ts": int(time.time()*1000),
-            "inputs": {
-                "question": question, "top_k": int(k), "n_sentences": int(n_sentences),
-                "w_tfidf": float(w_tfidf), "w_bm25": float(w_bm25), "w_emb": float(w_emb),
-                "use_llm": False, "model": None, "temperature": float(temperature)
-            },
-            "retrieval": {"hits": retr_list, "latency_ms_retriever": latency_ms_retriever},
-            "output": {
-                "final_answer": final,
-                "used_sentences": [{"sent": s["sent"], "doc": s["doc"], "page": s["page"]} for s in selected]
-            },
-            "latency_ms_total": int((time.time()-t0_total)*1000),
-            "openai": None
-        }
-        _safe_write_jsonl(LOG_PATH, record)
-        return final
-
-    # Extractive or LLM synthesis
-    extractive = compose_extractive(selected)
-    llm_usage = None
-    llm_latency_ms = None
-    if use_llm and selected:
-        lines = [f"{s['sent']} ({s['doc']}, p.{s['page']})" for s in selected]
-        t0_llm = time.time()
-        llm_text, llm_usage = synthesize_with_llm(question, lines, model=model, temperature=temperature)
-        t1_llm = time.time()
-        llm_latency_ms = int((t1_llm - t0_llm) * 1000)
-
-        if llm_text:
-            final = f"**Answer (LLM synthesis):** {llm_text}\n\n**Citations:** {header_cites}{coverage_note}"
-            if include_passages:
-                final += "\n\n---\n" + "\n\n".join(hits['text'].tolist()[:2])
-        else:
-            if not extractive:
-                final = f"**Answer:** Here are relevant passages.\n\n**Citations:** {header_cites}{coverage_note}\n\n---\n" + "\n\n".join(hits['text'].tolist()[:2])
-            else:
-                final = f"**Answer:** {extractive}\n\n**Citations:** {header_cites}{coverage_note}"
-                if include_passages:
-                    final += "\n\n---\n" + "\n\n".join(hits['text'].tolist()[:2])
+    # 1) Determine gold CSV path
+    if gold_file is None:
+        # Try default gold.csv in repo root
+        default_gold = Path("gold.csv")
+        if not default_gold.exists():
+            return (
+                "**No gold.csv provided or found in the working directory.**\n\n"
+                "Upload a file or place `gold.csv` next to `app.py`.",
+                None,
+                None,
+                None,
+                None,
+            )
+        gold_csv = str(default_gold)
     else:
-        if not extractive:
-            final = f"**Answer:** Here are relevant passages.\n\n**Citations:** {header_cites}{coverage_note}\n\n---\n" + "\n\n".join(hits['text'].tolist()[:2])
-        else:
-            final = f"**Answer:** {extractive}\n\n**Citations:** {header_cites}{coverage_note}"
-            if include_passages:
-                final += "\n\n---\n" + "\n\n".join(hits['text'].tolist()[:2])
-
-    # --------- Log full run ---------
-    prompt_toks = llm_usage.get("prompt_tokens") if llm_usage else None
-    completion_toks = llm_usage.get("completion_tokens") if llm_usage else None
-    cost_usd = _calc_cost_usd(prompt_toks, completion_toks)
+        # Gradio File object has a .name pointing to temp path in the Space
+        gold_csv = str(Path(gold_file.name))
 
-    total_ms = int((time.time() - t0_total) * 1000)
-    record = {
-        "run_id": run_id,
-        "ts": int(time.time()*1000),
-        "inputs": {
-            "question": question, "top_k": int(k), "n_sentences": int(n_sentences),
-            "w_tfidf": float(w_tfidf), "w_bm25": float(w_bm25), "w_emb": float(w_emb),
-            "use_llm": bool(use_llm), "model": model, "temperature": float(temperature)
-        },
-        "retrieval": {"hits": retr_list, "latency_ms_retriever": latency_ms_retriever},
-        "output": {
-            "final_answer": final,
-            "used_sentences": [{"sent": s['sent'], "doc": s['doc'], "page": s['page']} for s in selected]
-        },
-        "latency_ms_total": total_ms,
-        "latency_ms_llm": llm_latency_ms,
-        "openai": {
-            "prompt_tokens": prompt_toks,
-            "completion_tokens": completion_toks,
-            "cost_usd": cost_usd
-        } if use_llm else None
-    }
-    _safe_write_jsonl(LOG_PATH, record)
-    return final
+    # 2) Paths for logs + output directory
+    logs_jsonl = str(LOG_PATH)          # e.g., rag_artifacts/rag_logs.jsonl (from rag_core)
+    out_dir = str(ARTIFACT_DIR)         # e.g., rag_artifacts/
 
-def rag_chat_fn(message, history, top_k, n_sentences, include_passages,
-                use_llm, model_name, temperature, strict_quotes_only,
-                w_tfidf, w_bm25, w_emb):
-    if not message or not message.strip():
-        return "Ask a literature question (e.g., *How does CNT length affect gauge factor?*)"
+    # 3) Run evaluation (writes CSV/JSON under ARTIFACT_DIR)
+    #    evaluate_rag prints to console; we only care about the files it creates.
     try:
-        return rag_reply(
-            question=message,
-            k=int(top_k),
-            n_sentences=int(n_sentences),
-            include_passages=bool(include_passages),
-            use_llm=bool(use_llm),
-            model=(model_name or None),
-            temperature=float(temperature),
-            strict_quotes_only=bool(strict_quotes_only),
-            w_tfidf=float(w_tfidf),
-            w_bm25=float(w_bm25),
-            w_emb=float(w_emb),
+        evaluate_rag(
+            gold_csv=gold_csv,
+            logs_jsonl=logs_jsonl,
+            k=int(k),
+            out_dir=out_dir,
+            group_by_weights=True,      # also writes metrics_by_weights.csv
         )
     except Exception as e:
-        return f"RAG error: {e}"
-
-# ========================= UI (science-oriented styling) =========================
-CSS = """
-/* Science-oriented: crisp contrast + readable numerics */
-* {font-family: ui-sans-serif, system-ui, -apple-system, 'Segoe UI', Roboto, 'Helvetica Neue', Arial;}
-.gradio-container {
-    background: linear-gradient(135deg, #0b1020 0%, #0c2b1a 60%, #0a2b4d 100%) !important;
-}
-.card {background: rgba(255,255,255,0.06) !important; border: 1px solid rgba(255,255,255,0.14); border-radius: 12px;}
-label {color: #e8f7ff !important; text-shadow: 0 1px 0 rgba(0,0,0,0.35); cursor: pointer;}
-input[type="number"] {font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", monospace;}
-
-/* Checkbox clickability fixes */
-input[type="checkbox"], .gr-checkbox, .gr-checkbox > * { pointer-events: auto !important; }
-.gr-checkbox label, .gr-check-radio label { pointer-events: auto !important; cursor: pointer; }
-#rag-tab input[type="checkbox"] { accent-color: #60a5fa !important; }
-
-/* RAG tab styling */
-#rag-tab .block, #rag-tab .group, #rag-tab .accordion {
-    background: linear-gradient(160deg, #1f2937 0%, #14532d 55%, #0b3b68 100%) !important;
-    border-radius: 12px;
-    border: 1px solid rgba(255,255,255,0.14);
-}
-#rag-tab input, #rag-tab textarea, #rag-tab select, #rag-tab .scroll-hide, #rag-tab .chatbot textarea {
-    background: rgba(17, 24, 39, 0.85) !important;
-    border: 1px solid #60a5fa !important;
-    color: #e5f2ff !important;
-}
-#rag-tab input[type="range"] { accent-color: #22c55e !important; }
-#rag-tab button { border-radius: 10px !important; font-weight: 600 !important; }
-#rag-tab .chatbot {
-    background: rgba(15, 23, 42, 0.6) !important;
-    border: 1px solid rgba(148, 163, 184, 0.35) !important;
-}
-#rag-tab .message.user {
-    background: rgba(34, 197, 94, 0.15) !important;
-    border-left: 3px solid #22c55e !important;
-}
-#rag-tab .message.bot {
-    background: rgba(59, 130, 246, 0.15) !important;
-    border-left: 3px solid #60a5fa !important;
-    color: #eef6ff !important;
-}
-
-/* Evaluate tab dark/high-contrast styling */
-#eval-tab .block, #eval-tab .group, #eval-tab .accordion {
-    background: linear-gradient(165deg, #0a0f1f 0%, #0d1a31 60%, #0a1c2e 100%) !important;
-    border-radius: 12px;
-    border: 1px solid rgba(139, 197, 255, 0.28);
-}
-#eval-tab label, #eval-tab .markdown, #eval-tab .prose, #eval-tab p, #eval-tab span {
-    color: #e6f2ff !important;
-}
-#eval-tab input, #eval-tab .gr-file, #eval-tab .scroll-hide, #eval-tab textarea, #eval-tab select {
-    background: rgba(8, 13, 26, 0.9) !important;
-    border: 1px solid #3b82f6 !important;
-    color: #dbeafe !important;
-}
-#eval-tab input[type="range"] { accent-color: #22c55e !important; }
-#eval-tab button { 
-    border-radius: 10px !important; 
-    font-weight: 700 !important; 
-    background: #0ea5e9 !important; 
-    color: #001321 !important; 
-    border: 1px solid #7dd3fc !important;
-}
-#eval-tab .gr-json, #eval-tab .markdown pre, #eval-tab .markdown code {
-    background: rgba(2, 6, 23, 0.85) !important;
-    color: #e2e8f0 !important;
-    border: 1px solid rgba(148, 163, 184, 0.3) !important;
-    border-radius: 10px !important;
-}
-
-/* Predictor output emphasis */
-#pred-out .wrap { font-size: 20px; font-weight: 700; color: #ecfdf5; }
-
-/* Tab header: darker blue theme for all tabs */
-.gradio-container .tab-nav button[role="tab"] {
-  background: #0b1b34 !important;
-  color: #cfe6ff !important;
-  border: 1px solid #1e3a8a !important;
-}
-.gradio-container .tab-nav button[role="tab"][aria-selected="true"] {
-  background: #0e2a57 !important;
-  color: #e0f2fe !important;
-  border-color: #3b82f6 !important;
-}
-
-/* Evaluate tab: enforce dark-blue text for labels/marks */
-#eval-tab .label, 
-#eval-tab label, 
-#eval-tab .gr-slider .label,
-#eval-tab .wrap .label,
-#eval-tab .prose, 
-#eval-tab .markdown, 
-#eval-tab p, 
-#eval-tab span {
-  color: #cfe6ff !important; /* softer than pure white */
-}
-
-/* Target the specific k-slider label strongly */
-#k-slider .label, 
-#k-slider label, 
-#k-slider .wrap .label {
-  color: #cfe6ff !important;
-  text-shadow: 0 1px 0 rgba(0,0,0,0.35);
-}
+        return (f"**Evaluation error:** {e}", None, None, None, None)
 
-/* Slider track/thumb (dark blue gradient + blue thumb) */
-#eval-tab input[type="range"] { 
-  accent-color: #3b82f6 !important; /* fallback */
-}
+    # 4) Build paths to outputs
+    perq_path = ARTIFACT_DIR / "metrics_per_question.csv"
+    agg_path  = ARTIFACT_DIR / "metrics_aggregate.json"
+    surf_path = ARTIFACT_DIR / "metrics_by_weights.csv"
 
-/* WebKit */
-#eval-tab input[type="range"]::-webkit-slider-runnable-track {
-  height: 6px;
-  background: linear-gradient(90deg, #0b3b68, #1e3a8a);
-  border-radius: 4px;
-}
-#eval-tab input[type="range"]::-webkit-slider-thumb {
-  -webkit-appearance: none;
-  appearance: none;
-  margin-top: -6px;            /* centers thumb on 6px track */
-  width: 18px; height: 18px;
-  background: #1d4ed8;
-  border: 1px solid #60a5fa;
-  border-radius: 50%;
-}
-
-/* Firefox */
-#eval-tab input[type="range"]::-moz-range-track {
-  height: 6px;
-  background: linear-gradient(90deg, #0b3b68, #1e3a8a);
-  border-radius: 4px;
-}
-#eval-tab input[type="range"]::-moz-range-thumb {
-  width: 18px; height: 18px;
-  background: #1d4ed8;
-  border: 1px solid #60a5fa;
-  border-radius: 50%;
-}
-
-/* ======== PATCH: Style the File + JSON outputs by ID ======== */
-#perq-file, #agg-file {
-  background: rgba(8, 13, 26, 0.9) !important;
-  border: 1px solid #3b82f6 !important;
-  border-radius: 12px !important;
-  padding: 8px !important;
-}
-#perq-file * , #agg-file * { color: #dbeafe !important; }
-#perq-file a, #agg-file a {
-  background: #0e2a57 !important;
-  color: #e0f2fe !important;
-  border: 1px solid #60a5fa !important;
-  border-radius: 8px !important;
-  padding: 6px 10px !important;
-  text-decoration: none !important;
-}
-#perq-file a:hover, #agg-file a:hover {
-  background: #10356f !important;
-  border-color: #93c5fd !important;
-}
-/* File preview wrappers (covers multiple Gradio render modes) */
-#perq-file .file-preview, #agg-file .file-preview,
-#perq-file .wrap, #agg-file .wrap {
-  background: rgba(2, 6, 23, 0.85) !important;
-  border-radius: 10px !important;
-  border: 1px solid rgba(148,163,184,.3) !important;
-}
-
-/* JSON output: dark panel + readable text */
-#agg-json {
-  background: rgba(2, 6, 23, 0.85) !important;
-  border: 1px solid rgba(148,163,184,.35) !important;
-  border-radius: 12px !important;
-  padding: 8px !important;
-}
-#agg-json *, #agg-json .json, #agg-json .wrap { color: #e6f2ff !important; }
-#agg-json pre, #agg-json code {
-  background: rgba(4, 10, 24, 0.9) !important;
-  color: #e2e8f0 !important;
-  border: 1px solid rgba(148,163,184,.35) !important;
-  border-radius: 10px !important;
-}
-/* Tree/overflow modes */
-#agg-json [data-testid="json-tree"],
-#agg-json [role="tree"],
-#agg-json .overflow-auto {
-  background: rgba(4, 10, 24, 0.9) !important;
-  color: #e6f2ff !important;
-  border-radius: 10px !important;
-  border: 1px solid rgba(148,163,184,.35) !important;
-}
-
-/* Eval log markdown */
-#eval-log, #eval-log * { color: #cfe6ff !important; }
-#eval-log pre, #eval-log code {
-  background: rgba(2, 6, 23, 0.85) !important;
-  color: #e2e8f0 !important;
-  border: 1px solid rgba(148,163,184,.3) !important;
-  border-radius: 10px !important;
-}
+    # 5) Load aggregate JSON for preview (if available)
+    agg_json_data = None
+    if agg_path.exists():
+        try:
+            with open(agg_path, "r", encoding="utf-8") as f:
+                agg_json_data = json.load(f)
+        except Exception as e:
+            agg_json_data = {"error": f"Failed to load metrics_aggregate.json: {e}"}
+
+    # 6) Build a short log summary for the UI
+    log_lines = ["### ✅ Evaluation finished\n"]
+    log_lines.append(f"- Gold file: `{gold_csv}`")
+    log_lines.append(f"- Logs: `{logs_jsonl}`")
+    log_lines.append(f"- k (cutoff): **{int(k)}**")
+    log_lines.append("")
+    if agg_json_data and isinstance(agg_json_data, dict):
+        # Show some key values if present
+        for key in [
+            "questions_total_gold",
+            "questions_covered_in_logs",
+            "questions_missing_in_logs",
+            "questions_in_logs_not_in_gold",
+            "mean_hit@k_doc",
+            "mean_precision@k_doc",
+            "mean_recall@k_doc",
+            "mean_ndcg@k_doc",
+        ]:
+            if key in agg_json_data:
+                log_lines.append(f"- **{key}**: `{agg_json_data[key]}`")
+    log_md = "\n".join(log_lines)
+
+    # 7) Return everything for Gradio:
+    #    - markdown log
+    #    - each file path (or None if missing)
+    #    - JSON preview
+    return (
+        log_md,
+        str(perq_path) if perq_path.exists() else None,
+        str(agg_path)  if agg_path.exists()  else None,
+        str(surf_path) if surf_path.exists() else None,
+        agg_json_data,
+    )
 
-/* When Evaluate tab is active and JS has added .eval-active, bump contrast subtly */
-#eval-tab.eval-active .block, 
-#eval-tab.eval-active .group {
-  border-color: #60a5fa !important;
-}
-#eval-tab.eval-active .label {
-  color: #e6f2ff !important;
-}
-"""
 
-theme = gr.themes.Soft(
-    primary_hue="blue",
-    neutral_hue="green"
-).set(
-    body_background_fill="#0b1020",
-    body_text_color="#e0f2fe",
-    input_background_fill="#0f172a",
-    input_border_color="#1e40af",
-    button_primary_background_fill="#2563eb",
-    button_primary_text_color="#ffffff",
-    button_secondary_background_fill="#14532d",
-    button_secondary_text_color="#ecfdf5",
-)
+# ------------------ Build Gradio UI ------------------ #
 
-with gr.Blocks(css=CSS, theme=theme, fill_height=True) as demo:
-    # Optional: JS to toggle .eval-active when Evaluate tab selected
-    gr.HTML("""
-    <script>
-    (function(){
-      const applyEvalActive = () => {
-        const selected = document.querySelector('.tab-nav button[role="tab"][aria-selected="true"]');
-        const evalPanel = document.querySelector('#eval-tab');
-        if (!evalPanel) return;
-        if (selected && /Evaluate/.test(selected.textContent)) {
-          evalPanel.classList.add('eval-active');
-        } else {
-          evalPanel.classList.remove('eval-active');
-        }
-      };
-      document.addEventListener('click', function(e) {
-        if (e.target && e.target.getAttribute('role') === 'tab') {
-          setTimeout(applyEvalActive, 50);
-        }
-      }, true);
-      document.addEventListener('DOMContentLoaded', applyEvalActive);
-      setTimeout(applyEvalActive, 300);
-    })();
-    </script>
-    """)
-    
+with gr.Blocks(title="Self-Sensing Concrete RAG") as demo:
     gr.Markdown(
-        "<h1 style='margin:0'>Self-Sensing Concrete Assistant</h1>"
-        "<p style='opacity:.9'>"
-        "Left: ML prediction for Stress Gauge Factor (original scale, MPa<sup>-1</sup>). "
-        "Right: Literature Q&A via Hybrid RAG (BM25 + TF-IDF + optional dense) with MMR sentence selection."
-        "</p>"
+        "<h1>Self-Sensing Concrete Assistant — Hybrid RAG</h1>"
+        "<p>Ask questions about self-sensing/self-sensing concrete; "
+        "answers are grounded in your local PDFs in <code>papers/</code>.</p>"
     )
 
     with gr.Tabs():
-        # ------------------------- Predictor Tab -------------------------
-        with gr.Tab("🔮 Predict Gauge Factor (XGB)"):
-            with gr.Row():
-                with gr.Column(scale=7):
-                    with gr.Accordion("Primary conductive filler", open=True, elem_classes=["card"]):
-                        f1_type = gr.Textbox(label="Filler 1 Type *", placeholder="e.g., CNT, Graphite, Steel fiber")
-                        f1_diam = gr.Number(label="Filler 1 Diameter (µm) *")
-                        f1_len  = gr.Number(label="Filler 1 Length (mm) *")
-                        cf_conc = gr.Number(label=f"{CF_COL} *", info="Weight percent of total binder")
-                        f1_dim  = gr.Dropdown(DIM_CHOICES, value=CANON_NA, label="Filler 1 Dimensionality *")
-
-                    with gr.Accordion("Secondary filler (optional)", open=False, elem_classes=["card"]):
-                        f2_type = gr.Textbox(label="Filler 2 Type", placeholder="Optional")
-                        f2_diam = gr.Number(label="Filler 2 Diameter (µm)")
-                        f2_len  = gr.Number(label="Filler 2 Length (mm)")
-                        f2_dim  = gr.Dropdown(DIM_CHOICES, value=CANON_NA, label="Filler 2 Dimensionality")
-
-                    with gr.Accordion("Mix design & specimen", open=False, elem_classes=["card"]):
-                        spec_vol  = gr.Number(label="Specimen Volume (mm3) *")
-                        probe_cnt = gr.Number(label="Probe Count *")
-                        probe_mat = gr.Textbox(label="Probe Material *", placeholder="e.g., Copper, Silver paste")
-                        wb        = gr.Number(label="W/B *")
-                        sb        = gr.Number(label="S/B *")
-                        gauge_len = gr.Number(label="Gauge Length (mm) *")
-                        curing    = gr.Textbox(label="Curing Condition *", placeholder="e.g., 28d water, 20°C")
-                        n_fillers = gr.Number(label="Number of Fillers *")
-
-                    with gr.Accordion("Processing", open=False, elem_classes=["card"]):
-                        dry_temp = gr.Number(label="Drying Temperature (°C)")
-                        dry_hrs  = gr.Number(label="Drying Duration (hr)")
-
-                    with gr.Accordion("Mechanical & electrical loading", open=False, elem_classes=["card"]):
-                        load_rate = gr.Number(label="Loading Rate (MPa/s)")
-                        E_mod     = gr.Number(label="Modulus of Elasticity (GPa) *")
-                        current   = gr.Dropdown(CURRENT_CHOICES, value=CANON_NA, label="Current Type")
-                        voltage   = gr.Number(label="Applied Voltage (V)")
-
-                with gr.Column(scale=5):
-                    with gr.Group(elem_classes=["card"]):
-                        out_pred = gr.Number(label="Predicted Stress GF (MPa-1)", value=0.0, precision=6, elem_id="pred-out")
-                        gr.Markdown(f"<small>{MODEL_STATUS}</small>")
-                        with gr.Row():
-                            btn_pred  = gr.Button("Predict", variant="primary")
-                            btn_clear = gr.Button("Clear")
-                            btn_demo  = gr.Button("Fill Example")
-
-                    with gr.Accordion("About this model", open=False, elem_classes=["card"]):
-                        gr.Markdown(
-                            "- Pipeline: ColumnTransformer → (RobustScaler + OneHot) → XGBoost\n"
-                            "- Target: Stress GF (MPa<sup>-1</sup>) on original scale (model may train on log1p; saved flag used at inference).\n"
-                            "- Missing values are safely imputed per-feature.\n"
-                            "- Trained columns:\n"
-                            f"  `{', '.join(MAIN_VARIABLES)}`",
-                            elem_classes=["prose"]
-                        )
-
-            inputs_in_order = [
-                f1_type, f1_diam, f1_len, cf_conc,
-                f1_dim,  f2_type, f2_diam, f2_len,
-                f2_dim,  spec_vol, probe_cnt, probe_mat,
-                wb, sb, gauge_len, curing, n_fillers,
-                dry_temp, dry_hrs, load_rate,
-                E_mod, current, voltage
-            ]
-
-            def _predict_wrapper(*vals):
-                data = {k: v for k, v in zip(MAIN_VARIABLES, vals)}
-                return predict_fn(**data)
-
-            btn_pred.click(_predict_wrapper, inputs=inputs_in_order, outputs=out_pred)
-            btn_clear.click(lambda: _clear_all(), inputs=None, outputs=inputs_in_order).then(lambda: 0.0, outputs=out_pred)
-            btn_demo.click(lambda: _fill_example(), inputs=None, outputs=inputs_in_order)
-
-        # ------------------------- Literature Tab -------------------------
-        with gr.Tab("📚 Ask the Literature (Hybrid RAG + MMR)", elem_id="rag-tab"):
-            pdf_count = len(list(LOCAL_PDF_DIR.glob("**/*.pdf")))
+        # --------- RAG Chat tab ---------
+        with gr.Tab("📚 RAG Chat"):
             gr.Markdown(
-                f"Using local folder <code>papers/</code> — **{pdf_count} PDF(s)** indexed. "
-                "Upload more PDFs and reload the Space to expand coverage. Answers cite (Doc.pdf, p.X)."
+                "Use this tab to query the literature. Retrieval combines TF-IDF, BM25, and dense embeddings.\n"
+                "You can tune the weights below to explore different configurations."
             )
-            with gr.Row():
-                top_k = gr.Slider(5, 12, value=8, step=1, label="Top-K chunks")
-                n_sentences = gr.Slider(2, 6, value=4, step=1, label="Answer length (sentences)")
-                include_passages = gr.Checkbox(value=False, label="Include supporting passages", interactive=True)
 
-            with gr.Accordion("Retriever weights (advanced)", open=False):
-                w_tfidf = gr.Slider(0.0, 1.0, value=W_TFIDF_DEFAULT, step=0.05, label="TF-IDF weight")
-                w_bm25  = gr.Slider(0.0, 1.0, value=W_BM25_DEFAULT,  step=0.05, label="BM25 weight")
-                w_emb   = gr.Slider(0.0, 1.0, value=(0.0 if not USE_DENSE else 0.40), step=0.05, label="Dense weight (set 0 if disabled)")
+            with gr.Row():
+                top_k = gr.Slider(
+                    minimum=3,
+                    maximum=15,
+                    value=8,
+                    step=1,
+                    label="Top-K chunks"
+                )
+                n_sentences = gr.Slider(
+                    minimum=2,
+                    maximum=8,
+                    value=4,
+                    step=1,
+                    label="Answer length (sentences)"
+                )
+                include_passages = gr.Checkbox(
+                    value=False,
+                    label="Include supporting passages"
+                )
 
-            # Hidden states (unchanged)
-            state_use_llm     = gr.State(LLM_AVAILABLE)
-            state_model_name  = gr.State(os.getenv("OPENAI_MODEL", OPENAI_MODEL))
-            state_temperature = gr.State(0.2)
-            state_strict      = gr.State(False)
+            with gr.Row():
+                w_tfidf = gr.Slider(
+                    minimum=0.0,
+                    maximum=1.0,
+                    value=W_TFIDF_DEFAULT,
+                    step=0.05,
+                    label="TF-IDF weight"
+                )
+                w_bm25 = gr.Slider(
+                    minimum=0.0,
+                    maximum=1.0,
+                    value=W_BM25_DEFAULT,
+                    step=0.05,
+                    label="BM25 weight"
+                )
+                w_emb = gr.Slider(
+                    minimum=0.0,
+                    maximum=1.0,
+                    value=W_EMB_DEFAULT,
+                    step=0.05,
+                    label="Dense (embedding) weight"
+                )
 
             gr.ChatInterface(
                 fn=rag_chat_fn,
                 additional_inputs=[
-                    top_k, n_sentences, include_passages,
-                    state_use_llm, state_model_name, state_temperature, state_strict,
-                    w_tfidf, w_bm25, w_emb
+                    top_k,
+                    n_sentences,
+                    include_passages,
+                    w_tfidf,
+                    w_bm25,
+                    w_emb,
                 ],
-                title="Literature Q&A",
-                description="Hybrid retrieval with diversity. Answers carry inline (Doc, p.X) citations."
+                title="Hybrid RAG Q&A",
+                description=(
+                    "Hybrid BM25 + TF-IDF + dense retrieval with MMR sentence selection. "
+                    "Answers are citation-aware and grounded in the uploaded PDFs."
+                ),
+            )
+
+        # --------- Evaluation tab ---------
+        with gr.Tab("📏 Evaluate RAG"):
+            gr.Markdown(
+                "Upload a **gold.csv** file and compute retrieval metrics against the app logs "
+                "stored in <code>rag_artifacts/rag_logs.jsonl</code>.\n\n"
+                "**Requirements for gold.csv:**\n"
+                "- Must contain a column named <code>question</code>.\n"
+                "- And either:\n"
+                "  - a <code>doc</code> column (one relevant document per row), or\n"
+                "  - a <code>relevant_docs</code> column with a list of docs separated by `,` or `;`.\n"
             )
 
-        # ====== Evaluate (Gold vs Logs) — darker, higher-contrast ======
-        with gr.Tab("📏 Evaluate (Gold vs Logs)", elem_id="eval-tab"):
-            gr.Markdown("Upload your **gold.csv** and compute metrics against the app logs.")
-            with gr.Row():
-                gold_file = gr.File(label="gold.csv", file_types=[".csv"], interactive=True)
-                k_slider  = gr.Slider(3, 12, value=8, step=1, label="k for Hit/Recall/nDCG", elem_id="k-slider")
             with gr.Row():
-                btn_eval = gr.Button("Compute Metrics", variant="primary")
+                gold_file = gr.File(
+                    label="gold.csv",
+                    file_types=[".csv"],
+                    type="file",
+                )
+                k_slider = gr.Slider(
+                    minimum=1,
+                    maximum=20,
+                    value=8,
+                    step=1,
+                    label="k for Hit/Recall/nDCG",
+                )
+
+            run_button = gr.Button("Run Evaluation", variant="primary")
+
+            eval_log = gr.Markdown(
+                label="Evaluation log / summary"
+            )
+
             with gr.Row():
-                out_perq = gr.File(label="Per-question metrics (CSV)", elem_id="perq-file")
-                out_agg  = gr.File(label="Aggregate metrics (JSON)", elem_id="agg-file")
-            out_json = gr.JSON(label="Aggregate summary", elem_id="agg-json")
-            out_log  = gr.Markdown(label="Run log", elem_id="eval-log")
+                perq_file = gr.File(
+                    label="metrics_per_question.csv (download)",
+                    interactive=False,
+                )
+                agg_file = gr.File(
+                    label="metrics_aggregate.json (download)",
+                    interactive=False,
+                )
+                surf_file = gr.File(
+                    label="metrics_by_weights.csv (download)",
+                    interactive=False,
+                )
+
+            agg_json = gr.JSON(
+                label="Aggregate metrics (preview as JSON)"
+            )
 
-            def _run_eval_inproc(gold_path: str, k: int = 8):
-                import json as _json
-                out_dir = str(ARTIFACT_DIR)
-                logs = str(LOG_PATH)
-                cmd = [
-                    sys.executable, "rag_eval_metrics.py",
-                    "--gold_csv", gold_path,
-                    "--logs_jsonl", logs,
-                    "--k", str(k),
-                    "--out_dir", out_dir
-                ]
-                try:
-                    p = subprocess.run(cmd, capture_output=True, text=True, check=False)
-                    stdout = p.stdout or ""
-                    stderr = p.stderr or ""
-                    perq = ARTIFACT_DIR / "metrics_per_question.csv"
-                    agg  = ARTIFACT_DIR / "metrics_aggregate.json"
-                    agg_json = {}
-                    if agg.exists():
-                        agg_json = _json.loads(agg.read_text(encoding="utf-8"))
-                    report = "```\n" + (stdout.strip() or "(no stdout)") + ("\n" + stderr.strip() if stderr else "") + "\n```"
-                    return (str(perq) if perq.exists() else None,
-                            str(agg)  if agg.exists()  else None,
-                            agg_json,
-                            report)
-                except Exception as e:
-                    return (None, None, {}, f"**Eval error:** {e}")
+            run_button.click(
+                fn=run_eval_ui,
+                inputs=[gold_file, k_slider],
+                outputs=[eval_log, perq_file, agg_file, surf_file, agg_json],
+            )
 
-            def _eval_wrapper(gf, k):
-                from pathlib import Path
-                if gf is None:
-                    default_gold = Path("gold.csv")
-                    if not default_gold.exists():
-                        return None, None, {}, "**No gold.csv provided or found in repo root.**"
-                    gold_path = str(default_gold)
-                else:
-                    gold_path = gf.name
-                return _run_eval_inproc(gold_path, int(k))
 
-            btn_eval.click(_eval_wrapper, inputs=[gold_file, k_slider],
-                           outputs=[out_perq, out_agg, out_json, out_log])
+# ------------------ Launch app ------------------ #
 
-# ------------- Launch -------------
 if __name__ == "__main__":
-    demo.queue().launch()
-    import os
-import pandas as pd
-
-# Folder where your RAG files are stored
-folder = "papers"  # change if needed
-
-# List all files in the folder
-files = sorted(os.listdir(folder))
-
-# Save them to a CSV file
-pd.DataFrame({"doc": files}).to_csv("paper_list.csv", index=False)
-
-print("✅ Saved paper_list.csv with", len(files), "papers")
+    demo.queue().launch(
+        server_name="0.0.0.0",
+        server_port=7860,
+        share=False,
+    )