Update app.py

app.py

@@ -1,227 +1,265 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 """
-MaterialMind (fixed corpus demo)
-- Uses YOUR PDFs from ./sources
-- Builds a tiny in-memory RAG index at startup (FastEmbed + cosine)
-- Cloud LLM scores candidates 0..400 (four 0..100 subscores)
-- Simple Gradio UI (no uploads)
+MaterialMind – Hugging Face Spaces app
+- Same UI & templates you already have
+- Reads PDFs from HF dataset Azizahalq/materialmind-corpus
+- Builds/updates a local Chroma index at startup
+- Calls an API LLM (OpenAI or Together) via Space secrets
 """
-import os, re, json, textwrap
-from pathlib import Path
-from typing import List, Tuple, Dict, Any
-
-import gradio as gr
-import requests
 
-from rag_utils import (
-    build_index_from_dir, retrieve, format_context_and_cites
+import os, re, json, textwrap
+import subprocess, shutil
+from typing import List, Tuple, Any, Dict
+from flask import Flask, request, render_template, redirect, url_for, flash
+from flask_cors import CORS
+from filelock import FileLock
+
+# Silence tokenizers warning in HF
+os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
+
+# ---- RAG helpers (portable) ----
+from rag_mini import (
+    ensure_dirs, bootstrap_corpus_and_index, search,
+    DATA_DIR, DEFAULT_TOPK
 )
 
-# -------------------- LLM client --------------------
-PROVIDER = os.getenv("LLM_PROVIDER", "openai").lower()     # "openai" | "together"
-API_KEY  = os.getenv("LLM_API_KEY", "")
-MODEL    = os.getenv("LLM_MODEL", "gpt-4o-mini")           # e.g. Together: "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo"
-TIMEOUT  = int(os.getenv("LLM_TIMEOUT", "60"))
-
-def call_llm(system: str, user: str) -> str:
-    if not API_KEY:
-        return "[Error] Missing LLM_API_KEY. Add a secret/env var."
-    if PROVIDER == "together":
-        base = "https://api.together.xyz/v1"
-        headers = {"Authorization": f"Bearer {API_KEY}", "Content-Type": "application/json"}
-    else:
-        base = "https://api.openai.com/v1"
-        headers = {"Authorization": f"Bearer {API_KEY}", "Content-Type": "application/json"}
-
-    payload = {
-        "model": MODEL,
-        "messages": [{"role":"system","content":system},{"role":"user","content":user}],
-        "temperature": 0.2,
-    }
-    r = requests.post(f"{base}/chat/completions", headers=headers, json=payload, timeout=TIMEOUT)
-    if r.status_code != 200:
-        return f"[Error] LLM HTTP {r.status_code}: {r.text[:500]}"
+app = Flask(__name__)
+app.secret_key = os.getenv("FLASK_SECRET", "change-me")
+CORS(app)
+
+# HF runs on port 7860
+PORT = int(os.environ.get("PORT", "7860"))
+
+LOCK_PATH = (DATA_DIR.parent / ".rag_lock")
+DEFAULT_MODEL = os.getenv("LLM_MODEL", "gpt-4o-mini")
+LLM_PROVIDER  = os.getenv("LLM_PROVIDER", "openai")   # "openai" or "together"
+LLM_API_KEY   = os.getenv("LLM_API_KEY", "")
+
+# ---------- LLM caller (remote) ----------
+def call_llm(model: str, system_prompt: str, user_prompt: str) -> str:
+    provider = LLM_PROVIDER.lower().strip()
+    if provider == "openai":
+        try:
+            from openai import OpenAI
+            client = OpenAI(api_key=LLM_API_KEY)
+            resp = client.chat.completions.create(
+                model=model,
+                temperature=0.2,
+                messages=[
+                    {"role": "system", "content": system_prompt},
+                    {"role": "user", "content": user_prompt},
+                ],
+            )
+            return resp.choices[0].message.content or ""
+        except Exception as e:
+            return f"[Error] OpenAI call failed: {e}"
+
+    elif provider == "together":
+        # Simple Together REST call (instruct/chat style)
+        import requests
+        url = "https://api.together.xyz/v1/chat/completions"
+        headers = {"Authorization": f"Bearer {LLM_API_KEY}", "Content-Type": "application/json"}
+        payload = {
+            "model": model,
+            "temperature": 0.2,
+            "messages": [
+                {"role": "system", "content": system_prompt},
+                {"role": "user", "content": user_prompt},
+            ],
+        }
+        try:
+            r = requests.post(url, headers=headers, json=payload, timeout=120)
+            r.raise_for_status()
+            j = r.json()
+            return j["choices"][0]["message"]["content"]
+        except Exception as e:
+            return f"[Error] Together call failed: {e}"
+
+    return "[Error] Unknown LLM_PROVIDER. Set LLM_PROVIDER to 'openai' or 'together'."
+
+def extract_json_block(text: str):
+    m = re.search(r"```json\s*(\{.*?\})\s*```", text, flags=re.S | re.I)
+    s = m.group(1) if m else None
+    if not s:
+        m2 = re.search(r"(\{(?:[^{}]|(?1))*\})", text, flags=re.S)
+        s = m2.group(1) if m2 else None
+    if not s:
+        return None
     try:
-        return r.json()["choices"][0]["message"]["content"]
+        return json.loads(s)
     except Exception:
-        return f"[Error] Unexpected LLM response: {r.text[:500]}"
-
-# -------------------- Prompting --------------------
-SYSTEM_RULES = """You are MaterialMind, a general-purpose materials-selection assistant.
-Return TWO things:
+        last = s.rfind("}")
+        if last != -1:
+            try:
+                return json.loads(s[:last+1])
+            except Exception:
+                return None
+        return None
 
-1) A JSON block with EXACT schema:
+def normalize_candidates_for_display(cands: List[Dict[str, Any]], max_total: float = 400.0) -> List[Dict[str, Any]]:
+    def _to_float(x):
+        try: return float(x)
+        except: return None
+
+    for c in cands:
+        if "score_pct" in c and c["score_pct"] is not None:
+            try:
+                p = float(c["score_pct"])
+                c["score_pct"] = max(0.0, min(100.0, p))
+                c.setdefault("score_raw", c["score_pct"] * 4.0)
+                continue
+            except: pass
+
+        raw = None
+        v = c.get("score")
+        if isinstance(v, (int, float)):
+            f = float(v)
+            raw = (f * max_total) if f <= 1.5 else f
+        elif isinstance(v, str):
+            s = v.strip()
+            m = re.search(r"^\s*([\d.]+)\s*/\s*([\d.]+)\s*$", s)
+            if m:
+                num, den = _to_float(m.group(1)), _to_float(m.group(2))
+                if num is not None and den and den > 0: raw = max_total * (num/den)
+            if raw is None:
+                m2 = re.search(r"^\s*([\d.]+)\s*%\s*$", s)
+                if m2:
+                    p = _to_float(m2.group(1))
+                    if p is not None: raw = max_total * (p/100.0)
+            if raw is None:
+                f = _to_float(s)
+                if f is not None: raw = (f * max_total) if f <= 1.5 else f
+
+        if raw is None:
+            subs = c.get("subscores") or {}
+            if isinstance(subs, dict) and subs:
+                raw = sum(max(0.0, min(100.0, _to_float(v) or 0.0)) for v in subs.values())
+
+        raw = 0.0 if raw is None else max(0.0, min(max_total, float(raw)))
+        c["score_raw"] = raw
+        c["score_pct"] = round((raw / max_total) * 100.0, 1)
+
+    cands.sort(key=lambda z: z.get("score_raw", 0.0), reverse=True)
+    return cands
+
+# ---------- Prompt text ----------
+SYSTEM_RULES = """You are MaterialMind, a materials-selection assistant.
+Return two things:
+1) JSON with a ranked shortlist:
 {
   "candidates": [
     {
       "name": "string",
-      "score": 0,  // integer 0..400 (sum of four 0..100 subscores)
-      "subscores": { "performance": 0, "stability": 0, "cost": 0, "availability": 0 },
-      "reasons": ["string", "..."],
-      "tradeoffs": ["string", "..."],
-      "citations": ["[1]", "[4]"]
+      "score": 0,            // 0..400 (sum of 4 independent 0..100 utilities)
+      "score_pct": 0,        // score/4 -> 0..100 for display
+      "reasons": ["..."],
+      "tradeoffs": ["..."],
+      "citations": ["[1]", "[2]"]
     }
   ]
 }
-
-SCORING (absolute, not weighted):
-- performance (0..100): strength/stiffness/thermal range vs user targets
-- stability   (0..100): corrosion/oxidation/chem/UV/thermal/creep, environment fit
-- cost        (0..100): relative cost vs user budget (If budget is "Not important", set cost=100)
-- availability(0..100): manufacturability, supply forms/lead time
-
-Total score = performance + stability + cost + availability (0..400). Be conservative; do not invent data.
-
-2) After the JSON, add 3–6 concise bullets explaining trade-offs.
-
+2) After the JSON, 3–6 concise bullets on trade-offs.
 Rules:
-- Use ONLY the provided context; cite like [n].
-- If critical info is missing, state what to clarify.
-- Keep units correct; state assumptions if needed.
+- Use only provided context; cite with [1], [2], etc. No fabrication.
+- Utilities per criterion are in [0,1]. Lower cost => higher cost-utility.
+- Weights (performance, stability, cost, availability) are independent 0..100 (NOT normalized).
+- Prefer pitting/crevice metrics in chloride questions; keep units explicit.
 """
 
-ANSWER_TEMPLATE = """User constraints
-- Application: {environment}
+ANSWER_TEMPLATE = """User constraints:
+- Environment: {environment}
 - Temperature: {temperature}
-- Targets: UTS ≥ {min_uts} MPa, density ≤ {max_density} g/cm^3
-- Budget: {budget}   • Process: {process}
-- Preferences: performance={pref_perf}, stability={pref_stab}, cost={pref_cost}, availability={pref_avail}
+- Min UTS (MPa): {min_uts}
+- Max density (g/cm^3): {max_density}
+- Budget: {budget}
+- Process: {process}
+
+Independent priorities (0..100 each):
+- performance={w_perf}, stability={w_stab}, cost={w_cost}, availability={w_avail}
 
-Task
-Shortlist suitable materials and score them 0..400 using the four 0..100 subscores (see rules).
-Explain trade-offs and include citations.
+Question:
+{question}
 
-Context snippets (numbered)
+Context snippets (numbered):
 {context}
 
-Citations
+Citations:
 {citations}
 
-Now first output ONLY the JSON block. Then the bullet narrative.
+Now, first output ONLY the JSON block, then the short narrative.
 """
 
-def extract_json_block(text: str):
-    m = re.search(r"```json\s*(\{.*?\})\s*```", text, flags=re.S | re.I)
-    s = m.group(1) if m else None
-    if not s:
-        m2 = re.search(r"(\{(?:[^{}]|(?1))*\})", text, flags=re.S)
-        s = m2.group(1) if m2 else None
-    if not s: return None
+def format_context(hits: List[Tuple[str, str]]) -> Tuple[str, str]:
+    blocks, cites = [], []
+    for i, (text, cite) in enumerate(hits, 1):
+        snippet = textwrap.shorten(text.replace("\n", " "), width=450, placeholder=" …")
+        blocks.append(f"[{i}] {snippet}")
+        cites.append(f"[{i}] {cite}")
+    return "\n".join(blocks), "\n".join(cites)
+
+# ---------- Routes ----------
+@app.get("/")
+def index():
+    return render_template("index.html", default_model=DEFAULT_MODEL, default_k=DEFAULT_TOPK)
+
+@app.post("/recommend")
+def recommend():
+    environment  = request.form.get("environment", "").strip() or "seawater"
+    temperature  = request.form.get("temperature", "").strip() or "20–25 °C"
+    min_uts      = request.form.get("min_uts", "").strip() or "0"
+    max_density  = request.form.get("max_density", "").strip() or "100"
+    budget       = request.form.get("budget", "").strip() or "open"
+    process      = request.form.get("process", "").strip() or "any"
+
+    # Hidden numeric weights set by JS (0..100 each; independent)
+    w_perf  = request.form.get("w_perf", "75")
+    w_stab  = request.form.get("w_stab", "100")
+    w_cost  = request.form.get("w_cost", "75")
+    w_avail = request.form.get("w_avail", "75")
+
     try:
-        return json.loads(s)
+        k = int(request.form.get("k", DEFAULT_TOPK))
     except Exception:
-        last = s.rfind("}")
-        if last != -1:
-            try: return json.loads(s[:last+1])
-            except Exception: return None
-        return None
-
-# -------------------- Build index once (your PDFs) --------------------
-SOURCES_DIR = Path(os.getenv("SOURCES_DIR", "sources")).resolve()
-INDEX = build_index_from_dir(SOURCES_DIR)  # texts, metas, embs (L2-normalized)
+        k = DEFAULT_TOPK
 
-# -------------------- UI callback --------------------
-PREF_CHOICES = ["Very high", "High", "Medium", "Low", "Very low"]
-COST_CHOICES = ["Not important", "High", "Medium", "Low", "Very low"]
-
-def recommend(environment, temperature, min_uts, max_density, budget, process,
-              pref_perf, pref_stab, pref_cost, pref_avail, topk):
-
-    if INDEX["embs"].shape[0] == 0:
-        return "No context available. Add PDFs to ./sources and redeploy.", None, None
+    question = (
+        f"For {environment} at {temperature}, shortlist materials that meet "
+        f"UTS ≥ {min_uts} MPa and density ≤ {max_density} g/cm^3. "
+        f"Consider budget={budget} and process={process}. "
+        f"Rank by performance, stability, cost, and availability."
+    )
 
-    # Retrieval
-    q = (f"For {environment or 'general'} at {temperature or 'room temperature'}, shortlist materials that meet "
-         f"UTS ≥ {min_uts or '0'} MPa and density ≤ {max_density or '100'} g/cm^3; "
-         f"consider budget={budget or 'open'}, process={process or 'any'}.")
-    hits = retrieve(INDEX, q, k=int(topk))
+    hits = search(question, k=k)
     if not hits:
-        return "No extractable context found (OCR may be needed).", None, None
-    ctx, cites = format_context_and_cites(hits)
-
-    # LLM
-    prompt = ANSWER_TEMPLATE.format(
-        environment=environment or "general",
-        temperature=temperature or "room temperature",
-        min_uts=min_uts or "0",
-        max_density=max_density or "100",
-        budget=budget or "open",
-        process=process or "any",
-        pref_perf=pref_perf, pref_stab=pref_stab, pref_cost=pref_cost, pref_avail=pref_avail,
-        context=ctx, citations=cites
+        flash("No context found. Make sure your dataset is reachable and indexed.", "error")
+        return redirect(url_for("index"))
+
+    ctx, cites = format_context(hits)
+    user_prompt = ANSWER_TEMPLATE.format(
+        environment=environment, temperature=temperature, min_uts=min_uts,
+        max_density=max_density, budget=budget, process=process,
+        w_perf=w_perf, w_stab=w_stab, w_cost=w_cost, w_avail=w_avail,
+        question=question, context=ctx, citations=cites
     )
-    raw = call_llm(SYSTEM_RULES, prompt)
+
+    raw = call_llm(DEFAULT_MODEL, SYSTEM_RULES, user_prompt)
     parsed = extract_json_block(raw) if raw else None
-    cands = (parsed or {}).get("candidates", []) if parsed else []
-
-    # Format outputs
-    if not cands:
-        return raw, None, cites
-
-    headers = ["Rank","Material","Score","Performance","Stability","Cost","Availability","Top reasons"]
-    rows = []
-    for i, c in enumerate(sorted(cands, key=lambda x: x.get("score",0), reverse=True), 1):
-        ss = c.get("subscores", {})
-        reasons = " • ".join(c.get("reasons", [])[:3])
-        rows.append([i, c.get("name","?"), c.get("score",0),
-                     ss.get("performance","—"), ss.get("stability","—"),
-                     ss.get("cost","—"), ss.get("availability","—"), reasons])
-
-    # Markdown table
-    table_md = "| " + " | ".join(headers) + " |\n|" + " --- |"*len(headers) + "\n"
-    for r in rows:
-        table_md += "| " + " | ".join(str(x) for x in r) + " |\n"
-
-    # Cards
-    cards = []
-    for i, c in enumerate(sorted(cands, key=lambda x: x.get("score",0), reverse=True), 1):
-        ss = c.get("subscores", {})
-        card = f"**{i}. {c.get('name','?')}**  \n"
-        card += f"Score {c.get('score',0)} (perf {ss.get('performance','—')}, stab {ss.get('stability','—')}, cost {ss.get('cost','—')}, avail {ss.get('availability','—')})\n\n"
-        if c.get("tradeoffs"):
-            card += "**Trade-offs:**\n- " + "\n- ".join(c["tradeoffs"]) + "\n\n"
-        if c.get("citations"):
-            card += "**Citations:** " + ", ".join(c["citations"])
-        cards.append(card)
-    cards_md = "\n---\n".join(cards)
-
-    return table_md + "\n\n" + raw, cards_md, cites
-
-# -------------------- Gradio UI --------------------
-with gr.Blocks(title="MaterialMind") as demo:
-    gr.Markdown("## MaterialMind — ranked materials shortlist with page-level citations")
-    with gr.Row():
-        environment = gr.Textbox(label="Application", placeholder="seawater / sour service / high-T oxidation")
-        temperature = gr.Textbox(label="Temperature", placeholder="e.g., 20–25 °C")
-    with gr.Row():
-        min_uts = gr.Textbox(label="Min UTS (MPa)", value="0")
-        max_density = gr.Textbox(label="Max density (g/cm³)", value="100")
-    with gr.Row():
-        budget = gr.Dropdown(["open","low","medium","high","Not important"], value="open", label="Budget")
-        process = gr.Textbox(label="Process", placeholder="wrought / casting / AM / any", value="any")
-
-    gr.Markdown("**Priorities (qualitative; scoring is absolute 0..100 each, total 0..400)**")
-    with gr.Row():
-        pref_perf = gr.Dropdown(["Very high","High","Medium","Low","Very low"], value="High", label="Performance")
-        pref_stab = gr.Dropdown(["Very high","High","Medium","Low","Very low"], value="High", label="Stability")
-        pref_cost = gr.Dropdown(["Not important","High","Medium","Low","Very low"], value="Medium", label="Cost")
-        pref_avail = gr.Dropdown(["Very high","High","Medium","Low","Very low"], value="Medium", label="Availability")
-
-    topk = gr.Slider(3, 10, step=1, value=5, label="Top-k context pages")
-
-    run_btn = gr.Button("Get ranked shortlist", variant="primary")
-    out_table = gr.Markdown(label="Shortlist & raw model output")
-    out_cards = gr.Markdown(label="Material cards")
-    out_cites = gr.Markdown(label="Citations (source mapping)")
-
-    run_btn.click(
-        recommend,
-        inputs=[environment, temperature, min_uts, max_density, budget, process,
-                pref_perf, pref_stab, pref_cost, pref_avail, topk],
-        outputs=[out_table, out_cards, out_cites],
-        api_name="recommend"
+    candidates = (parsed or {}).get("candidates", []) if parsed else []
+    candidates = normalize_candidates_for_display(candidates, max_total=400.0)
+
+    return render_template(
+        "results.html",
+        candidates=candidates,
+        citations=cites.splitlines(),
+        environment=environment,
+        temperature=temperature,
+        raw_output=raw,
+        default_model=DEFAULT_MODEL,
+        default_k=k,
     )
 
 if __name__ == "__main__":
-    demo.launch()
+    ensure_dirs()
+    bootstrap_corpus_and_index()       # <-- download dataset + build/update index
+    app.run(host="0.0.0.0", port=PORT, debug=False)