Update app.py

app.py

@@ -6,6 +6,7 @@ from PIL import Image
 import torch
 from transformers import CLIPModel, CLIPProcessor
 import torch.nn.functional as F
+import os, json, time
 
 # ---------- utils ----------
 def flux_to_gray(flux_array):
@@ -89,6 +90,65 @@ def search(text_query, image_query, k=5):
         return items, f"Returned {k} results."
 
 
+# ---------- evaluation helpers ----------
+def _search_topk_for_eval(text_query, image_query, k=5):
+    if INDEX["feats"] is None:
+        return [], [], "Build the index first."
+    with torch.no_grad():
+        if text_query and str(text_query).strip():
+            inputs = processor(text=[str(text_query).strip()], return_tensors="pt")
+            q = model.get_text_features(**inputs)
+        elif image_query is not None:
+            pil = image_query.convert("RGB")
+            inputs = processor(images=pil, return_tensors="pt")
+            q = model.get_image_features(**inputs)
+        else:
+            return [], [], "Enter text or upload an image."
+        q = F.normalize(q, p=2, dim=-1)[0]
+        sims = (INDEX["feats"] @ q).cpu()
+        k = min(int(k), sims.shape[0])
+        topk = torch.topk(sims, k=k)
+        idxs = topk.indices.tolist()
+        # reuse thumbs and captions like your main search
+        items = []
+        for idx in idxs:
+            cap = f"id: {INDEX['ids'][idx]}  score: {float(sims[idx]):.3f}  band: {INDEX['bands'][idx]}"
+            items.append((INDEX["thumbs"][idx], cap))
+        return items, idxs, f"Eval preview: top {k} ready."
+
+def _format_eval_summary(query, k, hits, p_at_k):
+    lines = []
+    lines.append(f"Query: {query or '[image query]'}")
+    lines.append(f"K: {k}")
+    lines.append(f"Relevant marked: {hits} of {k}")
+    lines.append(f"Precision@{k}: {p_at_k:.2f}")
+    lines.append("Saved to eval_runs.jsonl")
+    return "\n".join(lines)
+
+def _save_eval_run(record):
+    try:
+        with open("eval_runs.jsonl", "a", encoding="utf-8") as f:
+            f.write(json.dumps(record) + "\n")
+    except Exception:
+        pass
+
+def _compute_avg_from_file():
+    try:
+        total = 0.0
+        n = 0
+        with open("eval_runs.jsonl", "r", encoding="utf-8") as f:
+            for line in f:
+                rec = json.loads(line)
+                if "precision_at_k" in rec:
+                    total += float(rec["precision_at_k"])
+                    n += 1
+        if n == 0:
+            return "No runs recorded yet."
+        return f"Macro average Precision@K across {n} runs: {total/n:.2f}"
+    except FileNotFoundError:
+        return "No eval_runs.jsonl yet. Run at least one evaluation."
+
+
 # ---------- UI ----------
 with gr.Blocks() as demo:
     gr.Markdown("JWST multimodal search — build the index")
@@ -106,6 +166,64 @@ with gr.Blocks() as demo:
     k      = gr.Slider(1, 12, value=6, step=1, label="Top K")
 
     search_btn = gr.Button("Search")
+
+     # ---------- evaluation UI ----------
+    with gr.Accordion("Evaluation", open=False):
+        eval_query = gr.Textbox(label="Evaluation query", placeholder="Type a query or leave empty and upload an image")
+        eval_img   = gr.Image(label="Evaluation image (optional)", type="pil")
+        eval_k     = gr.Slider(1, 12, value=6, step=1, label="K for evaluation")
+
+        run_and_label = gr.Button("Run and label this query")
+
+        eval_gallery  = gr.Gallery(label="Eval top K results", columns=6, height=300)
+        relevant_picker = gr.CheckboxGroup(label="Select indices of relevant results (1..K)")
+        eval_md = gr.Markdown()
+
+        eval_state = gr.State({"result_indices": [], "k": 5, "query": ""})
+
+        def _run_eval_query(q_txt, q_img_in, k_in, state):
+            items, idxs, _ = _search_topk_for_eval(q_txt, q_img_in, k_in)
+            state["result_indices"] = idxs
+            state["k"] = int(k_in)
+            state["query"] = q_txt if (q_txt and q_txt.strip()) else "[image query]"
+            choice_labels = [str(i+1) for i in range(len(idxs))]
+            return items, gr.update(choices=choice_labels, value=[]), "Mark relevant then click Compute metrics.", state
+
+        run_and_label.click(
+            fn=_run_eval_query,
+            inputs=[eval_query, eval_img, eval_k, eval_state],
+            outputs=[eval_gallery, relevant_picker, eval_md, eval_state]
+        )
+
+        compute_btn = gr.Button("Compute metrics")
+
+        def _compute_pk(selected_indices, state):
+            k = int(state.get("k", 5))
+            query = state.get("query", "")
+            # user marks which of the K are relevant; count is the hits
+            hits = len(selected_indices)
+            p_at_k = hits / max(k, 1)
+            record = {
+                "ts": int(time.time()),
+                "query": query,
+                "k": k,
+                "relevant_indices": sorted([int(s) for s in selected_indices]),
+                "precision_at_k": p_at_k
+            }
+            _save_eval_run(record)
+            return _format_eval_summary(query, k, hits, p_at_k)
+
+        compute_btn.click(
+            fn=_compute_pk,
+            inputs=[relevant_picker, eval_state],
+            outputs=eval_md
+        )
+
+        avg_btn = gr.Button("Compute average across saved runs")
+        avg_md  = gr.Markdown()
+
+        avg_btn.click(fn=_compute_avg_from_file, outputs=avg_md)
+    
     gallery    = gr.Gallery(label="Results", columns=6, height=300)
     info2      = gr.Textbox(label="Search status", lines=1)