Polish Gradio UI layout/tooltips and remove dead helper code

app.py

@@ -1,21 +1,27 @@
-import gradio as gr
-import os
-import logging
-import time
-import json
-import csv
-from datetime import datetime
-from functools import lru_cache
-from PIL import Image
-from pathlib import Path
-from typing import Any, Dict, List, Set, Tuple
+import gradio as gr
+import os
+import logging
+import time
+import json
+import csv
+import base64
+from datetime import datetime
+from functools import lru_cache
+from PIL import Image
+from pathlib import Path
+from typing import Any, Dict, List, Set, Tuple
 from concurrent.futures import ThreadPoolExecutor, TimeoutError as FutureTimeoutError
 
 from psq_rag.pipeline.preproc import extract_user_provided_tags_upto_3_words
 from psq_rag.llm.rewrite import llm_rewrite_prompt
 from psq_rag.retrieval.psq_retrieval import psq_candidates_from_rewrite_phrases, _norm_tag_for_lookup
 from psq_rag.llm.select import llm_select_indices, llm_infer_structural_tags, llm_infer_probe_tags
-from psq_rag.retrieval.state import expand_tags_via_implications, get_tag_type_name, get_tag_implications
+from psq_rag.retrieval.state import (
+    expand_tags_via_implications,
+    get_tag_type_name,
+    get_tag_implications,
+    get_tag_counts,
+)
 from psq_rag.ui.group_ranked_display import rank_groups_from_tfidf, _load_enabled_groups
 
 
@@ -46,6 +52,15 @@ def _display_tag_text(tag: str) -> str:
     return tag.replace("_", " ")
 
 
+def _display_row_label(name: str) -> str:
+    n = (name or "").strip()
+    if not n:
+        return ""
+    if n == "selected_other":
+        return "Selected (Other)"
+    return n.replace("_", " ").title()
+
+
 def _normalize_selection_origin(origin: str) -> str:
     o = (origin or "").strip().lower()
     if o in {"rewrite", "selection", "probe", "structural", "user", "candidate"}:
@@ -53,11 +68,53 @@ def _normalize_selection_origin(origin: str) -> str:
     return "selection"
 
 
+@lru_cache(maxsize=1)
+def _load_tag_wiki_defs() -> Dict[str, str]:
+    p = Path("data/tag_wiki_defs.json")
+    if not p.exists():
+        return {}
+    try:
+        with p.open("r", encoding="utf-8") as f:
+            data = json.load(f)
+        out: Dict[str, str] = {}
+        if isinstance(data, dict):
+            for k, v in data.items():
+                tag = _norm_tag_for_lookup(str(k))
+                text = " ".join(str(v or "").split())
+                if tag and text:
+                    out[tag] = text
+        return out
+    except Exception:
+        return {}
+
+
+def _tooltip_text_for_tag(tag: str) -> str:
+    t = _norm_tag_for_lookup(tag)
+    parts: List[str] = []
+
+    try:
+        count = get_tag_counts().get(t)
+    except Exception:
+        count = None
+    if isinstance(count, int):
+        parts.append(f"Count: {count:,}")
+
+    d = _load_tag_wiki_defs().get(t, "")
+    if d:
+        parts.append(d)
+
+    return "\n".join(parts).strip()
+
+
 def _choice_label_with_source_meta(tag: str, *, origin: str, preselected: bool) -> str:
-    # Marker is stripped client-side and converted into data attributes for CSS-driven colors.
+    # Marker is stripped client-side and converted into data attributes for CSS-driven colors/tooltips.
     origin_norm = _normalize_selection_origin(origin)
     pre = "1" if preselected else "0"
-    return f"{_display_tag_text(tag)} [[psq:{origin_norm}:{pre}]]"
+    tooltip = _tooltip_text_for_tag(tag)
+    tip_b64 = ""
+    if tooltip:
+        tip_b64 = base64.urlsafe_b64encode(tooltip.encode("utf-8")).decode("ascii")
+    return f"{_display_tag_text(tag)} [[psq:{origin_norm}:{pre}:{tip_b64}]]"
 
 
 def _selection_source_rank(origin: str) -> int:
@@ -246,6 +303,7 @@ def _build_toggle_rows(
     *,
     seed_terms: List[str],
     selected_tags: List[str],
+    retrieved_candidate_tags: List[str],
     tag_selection_origins: Dict[str, str],
     implied_parent_map: Dict[str, str],
     top_groups: int,
@@ -267,18 +325,72 @@ def _build_toggle_rows(
         )
     )
     selected_index: Dict[str, int] = {t: i for i, t in enumerate(selected_active)}
+
+    row_defs: List[Dict[str, Any]] = []
+    enabled_group_tag_sets: Dict[str, Set[str]] = {
+        name: {t for t in tags if not _is_artist_tag(t)}
+        for name, tags in groups_map.items()
+    }
+    tags_in_any_enabled_group: Set[str] = set()
+    for tag_set in enabled_group_tag_sets.values():
+        tags_in_any_enabled_group.update(tag_set)
+
+    displayed_group_names = [r.group_name for r in ranked_rows]
+    displayed_group_tag_sets: Dict[str, Set[str]] = {
+        name: enabled_group_tag_sets.get(name, set())
+        for name in displayed_group_names
+    }
+    tags_in_any_displayed_group: Set[str] = set()
+    for tag_set in displayed_group_tag_sets.values():
+        tags_in_any_displayed_group.update(tag_set)
 
-    row_defs: List[Dict[str, Any]] = []
-    displayed_group_names = [r.group_name for r in ranked_rows]
-    displayed_group_tag_sets: Dict[str, Set[str]] = {
-        name: {t for t in groups_map.get(name, []) if not _is_artist_tag(t)}
-        for name in displayed_group_names
-    }
-    tags_in_any_displayed_group: Set[str] = set()
-    for tag_set in displayed_group_tag_sets.values():
-        tags_in_any_displayed_group.update(tag_set)
-
-    selected_other_raw = [t for t in selected_active if t not in tags_in_any_displayed_group]
+    retrieved_uncategorized_ranked = list(
+        dict.fromkeys(
+            _norm_tag_for_lookup(t)
+            for t in (retrieved_candidate_tags or [])
+            if t
+            and not _is_artist_tag(t)
+            and not _is_excluded_recommendation_tag(t)
+            and _norm_tag_for_lookup(t) not in tags_in_any_enabled_group
+        )
+    )
+    retrieved_other_row: Dict[str, Any] | None = None
+    if retrieved_uncategorized_ranked:
+        retrieved_uncategorized_set = set(retrieved_uncategorized_ranked)
+        selected_in_retrieved_other_raw = [
+            t for t in selected_active if t in retrieved_uncategorized_set
+        ]
+        selected_in_retrieved_other = _order_selected_tags_for_row(
+            row_selected_tags=selected_in_retrieved_other_raw,
+            selected_index=selected_index,
+            tag_selection_origins=tag_selection_origins,
+            implied_parent_map=implied_parent_map,
+        )
+        merged_retrieved_other = selected_in_retrieved_other + [
+            t for t in retrieved_uncategorized_ranked if t not in selected_in_retrieved_other
+        ]
+        keep_n = max(max(1, int(top_tags_per_group)), len(selected_in_retrieved_other))
+        merged_retrieved_other = merged_retrieved_other[:keep_n]
+        retrieved_other_meta = {
+            t: {
+                "origin": _normalize_selection_origin(tag_selection_origins.get(t, "selection")),
+                "preselected": t in selected_active,
+            }
+            for t in merged_retrieved_other
+        }
+        retrieved_other_row = {
+            "name": "other_retrieved",
+            "label": "Other (Retrieved)",
+            "tags": merged_retrieved_other,
+            "tag_meta": retrieved_other_meta,
+        }
+
+    # "Selected (Other)" should contain selected tags not already shown in any displayed row.
+    # Include "Other (Retrieved)" in that displayed-row set to avoid duplicates across those rows.
+    tags_in_displayed_rows = set(tags_in_any_displayed_group)
+    if retrieved_other_row:
+        tags_in_displayed_rows.update(retrieved_other_row.get("tags", []))
+    selected_other_raw = [t for t in selected_active if t not in tags_in_displayed_rows]
     selected_other = _order_selected_tags_for_row(
         row_selected_tags=selected_other_raw,
         selected_index=selected_index,
@@ -295,7 +407,7 @@ def _build_toggle_rows(
     row_defs.append(
         {
             "name": "selected_other",
-            "label": "Selected (Other)",
+            "label": _display_row_label("selected_other"),
             "tags": selected_other,
             "tag_meta": selected_other_meta,
         }
@@ -329,16 +441,20 @@ def _build_toggle_rows(
         row_defs.append(
             {
                 "name": group_name,
-                "label": f"{group_name} (E={row.expected_count:.2f})",
+                "label": _display_row_label(group_name),
                 "tags": merged,
                 "tag_meta": tag_meta,
             }
         )
-
-    return row_defs
+
+    # Keep this row at the bottom so category/group rows remain contiguous.
+    if retrieved_other_row:
+        row_defs.append(retrieved_other_row)
+
+    return row_defs
 
 
-def _build_display_audit_line(
+def _build_display_audit_line(
     row_defs: List[Dict[str, Any]],
     *,
     active_selected_tags: List[str],
@@ -366,12 +482,14 @@ def _build_display_audit_line(
         row_name = row.get("name", "")
         row_label = row.get("label", row_name)
         for tag in row.get("tags", []):
-            rec = info_by_tag.setdefault(tag, {"rows": [], "sources": set()})
-            rec["rows"].append(row_label)
-            if row_name == "selected_other":
-                rec["sources"].add("selected_other_row")
-            else:
-                rec["sources"].add("ranked_group_row")
+            rec = info_by_tag.setdefault(tag, {"rows": [], "sources": set()})
+            rec["rows"].append(row_label)
+            if row_name == "selected_other":
+                rec["sources"].add("selected_other_row")
+            elif row_name == "other_retrieved":
+                rec["sources"].add("other_retrieved_row")
+            else:
+                rec["sources"].add("ranked_group_row")
             if tag in active_set:
                 rec["sources"].add("selected_active")
             if tag in direct_set:
@@ -390,12 +508,12 @@ def _build_display_audit_line(
             for tag, rec in sorted(info_by_tag.items())
         ],
     }
-    return "Display Tag Audit: " + json.dumps(payload, ensure_ascii=True)
-
-
-def _build_row_component_updates(
-    row_defs: List[Dict[str, Any]],
-    selected_tags: List[str],
+    return "Display Tag Audit: " + json.dumps(payload, ensure_ascii=True)
+
+
+def _build_row_component_updates(
+    row_defs: List[Dict[str, Any]],
+    selected_tags: List[str],
     max_rows: int,
 ):
     selected = {t for t in (selected_tags or []) if t}
@@ -410,7 +528,7 @@ def _build_row_component_updates(
             values = [t for t in tags if t in selected]
             row_values_state.append(values)
             visible = bool(tags)
-            header_updates.append(gr.update(value=f"**{row.get('label', '')}**", visible=visible))
+            header_updates.append(gr.update(value=row.get("label", ""), visible=visible))
             tag_meta = row.get("tag_meta", {}) if isinstance(row.get("tag_meta", {}), dict) else {}
             choices = []
             for t in tags:
@@ -442,8 +560,9 @@ def _on_toggle_row(
     max_rows: int,
 ):
     row_defs = row_defs_state or []
+    row_defs_ui = row_defs[: max(0, int(max_rows))]
     selected = set(selected_tags_state or [])
-    row = row_defs[row_idx] if 0 <= row_idx < len(row_defs) else {}
+    row = row_defs_ui[row_idx] if 0 <= row_idx < len(row_defs_ui) else {}
     row_tags = list(dict.fromkeys(row.get("tags", [])))
     row_tag_set = set(row_tags)
     row_tag_by_norm = {_norm_tag_for_lookup(t): t for t in row_tags}
@@ -459,16 +578,32 @@ def _on_toggle_row(
         if mapped:
             new_set.add(mapped)
 
-    prev_row_selected = {t for t in selected if t in row_tag_set}
+    prev_values = list(row_values_state or [])
+    prev_row_values = prev_values[row_idx] if 0 <= row_idx < len(prev_values) else []
+    prev_row_selected = set()
+    for raw in (prev_row_values or []):
+        if raw in row_tag_set:
+            prev_row_selected.add(raw)
+            continue
+        raw_norm = _norm_tag_for_lookup(str(raw))
+        mapped = row_tag_by_norm.get(raw_norm)
+        if mapped:
+            prev_row_selected.add(mapped)
+
+    # Ignore non-user/no-op events (e.g., programmatic value re-sets) deterministically.
+    if new_set == prev_row_selected:
+        prompt_text = _compose_toggle_prompt_text(sorted(selected), row_defs_ui)
+        checkbox_updates = [gr.skip() for _ in range(max_rows)]
+        return [sorted(selected), prev_values, prompt_text, *checkbox_updates]
+
     selected.difference_update(row_tag_set)
     selected.update(new_set)
     toggled_tags = prev_row_selected ^ new_set
 
-    # Recompute row selections, but only push UI updates to rows touched by the toggled tags.
     new_row_values_state: List[List[str]] = []
     affected_rows: Set[int] = {row_idx}
-    for idx, row in enumerate(row_defs):
-        tags = list(dict.fromkeys(row.get("tags", [])))
+    for idx, row_item in enumerate(row_defs_ui):
+        tags = list(dict.fromkeys(row_item.get("tags", [])))
         values = [t for t in tags if t in selected]
         new_row_values_state.append(values)
         if toggled_tags and any(t in toggled_tags for t in tags):
@@ -476,40 +611,60 @@ def _on_toggle_row(
 
     checkbox_updates = []
     for idx in range(max_rows):
-        if idx < len(row_defs) and idx in affected_rows:
+        if idx >= len(row_defs_ui):
+            checkbox_updates.append(gr.skip())
+            continue
+        if idx in affected_rows:
             checkbox_updates.append(gr.update(value=new_row_values_state[idx]))
         else:
-            checkbox_updates.append(gr.update())
+            checkbox_updates.append(gr.skip())
 
-    prompt_text = _compose_toggle_prompt_text(sorted(selected), row_defs)
+    prompt_text = _compose_toggle_prompt_text(sorted(selected), row_defs_ui)
     return [sorted(selected), new_row_values_state, prompt_text, *checkbox_updates]
 
 
-def _build_ui_payload(
-    *,
-    console_text: str,
-    legacy_prompt_text: str,
-    row_defs: List[Dict[str, Any]],
-    selected_tags: List[str],
-):
+def _build_ui_payload(
+    *,
+    console_text: str,
+    row_defs: List[Dict[str, Any]],
+    selected_tags: List[str],
+):
     prompt_text, row_values_state, header_updates, checkbox_updates = _build_row_component_updates(
         row_defs=row_defs,
         selected_tags=selected_tags,
         max_rows=display_max_rows_default,
     )
-    return [
-        console_text,
-        legacy_prompt_text,
-        prompt_text,
-        sorted(set(selected_tags or [])),
-        row_defs,
-        row_values_state,
-        *header_updates,
+    return [
+        console_text,
+        gr.update(visible=bool(row_defs)),
+        prompt_text,
+        sorted(set(selected_tags or [])),
+        row_defs,
+        row_values_state,
+        *header_updates,
         *checkbox_updates,
-    ]
-
-
-def _build_selection_query(
+    ]
+
+
+def _prepare_run_ui() -> List[Any]:
+    header_updates = [gr.update(value="", visible=False) for _ in range(display_max_rows_default)]
+    checkbox_updates = [
+        gr.update(choices=[], value=[], visible=False)
+        for _ in range(display_max_rows_default)
+    ]
+    return [
+        "Running...",
+        gr.skip(),
+        "Running... usually completes in about 20 seconds.",
+        [],
+        [],
+        [],
+        *header_updates,
+        *checkbox_updates,
+    ]
+
+
+def _build_selection_query(
     prompt_in: str,
     rewritten: str,
     structural_tags: List[str],
@@ -799,42 +954,112 @@ css = """
 .source-legend {
   display: flex;
   flex-wrap: wrap;
+  align-items: center;
   gap: 8px;
   margin: 4px 0 10px 0;
 }
 
+.source-legend .legend-title {
+  font-size: 0.92rem;
+  font-weight: 900;
+  color: #334155;
+  margin-right: 4px;
+}
+
 .source-legend .chip {
   display: inline-flex;
   align-items: center;
-  gap: 8px;
-  border-radius: 999px;
-  border: 1px solid #8792a2;
-  padding: 5px 10px;
+  border-radius: 10px;
+  border: 1px solid #6c7788;
+  padding: 6px 12px;
   font-size: 0.85rem;
-  font-weight: 700;
-  color: #1f2430;
+  font-weight: 800;
+  color: #111827;
   background: #f3f6fb;
 }
 
-.source-legend .swatch {
-  width: 12px;
-  height: 12px;
-  border-radius: 50%;
-  border: 1px solid rgba(0,0,0,0.2);
+.source-legend .chip.rewrite { background: #26b9a3; color: #062923; border-color: #187869; }
+.source-legend .chip.selection { background: #f0a93c; color: #382206; border-color: #a66f1f; }
+.source-legend .chip.probe { background: #9a6cff; color: #ffffff; border-color: #6745b0; }
+.source-legend .chip.structural { background: #53c368; color: #102d17; border-color: #2f8442; }
+.source-legend .chip.implied { background: #a8b3c4; color: #1d2633; border-color: #6f7e95; }
+.source-legend .chip.user { background: #4f86ff; color: #ffffff; border-color: #2f5fbf; }
+.source-legend .chip.unselected { background: #c7ced8; color: #2d3440; border-color: #7d8897; }
+
+.row-heading p {
+  margin: 8px 0 0 0 !important;
+  font-size: 1.18rem !important;
+  font-weight: 850 !important;
+  line-height: 1.2 !important;
+}
+
+.row-instruction {
+  text-align: center;
+  margin: 8px 0 12px 0;
+}
+
+.row-instruction p {
+  margin: 0 !important;
+  font-size: 1.02rem !important;
+  font-style: italic !important;
+  font-weight: 800 !important;
+  color: #1d4ed8 !important;
+}
+
+.top-instruction {
+  text-align: center;
+  margin: 2px 0 6px 0;
+}
+
+.top-instruction p {
+  margin: 0 !important;
+  font-size: 1.02rem !important;
+  font-style: italic !important;
+  font-weight: 800 !important;
+  color: #1d4ed8 !important;
+}
+
+.run-hint {
+  margin-top: 6px;
+  text-align: center;
 }
 
-.source-legend .rewrite { background: #26b9a3; }
-.source-legend .selection { background: #f0a93c; }
-.source-legend .probe { background: #9a6cff; }
-.source-legend .structural { background: #53c368; }
-.source-legend .implied { background: #a8b3c4; }
-.source-legend .user { background: #4f86ff; }
-.source-legend .unselected { background: #c7ced8; }
+.run-hint p {
+  margin: 0 !important;
+  font-size: 0.9rem !important;
+  font-style: italic !important;
+  color: #475569 !important;
+}
+
+.prompt-card {
+  background: transparent !important;
+  border: none !important;
+  box-shadow: none !important;
+  padding: 0 !important;
+}
+
+.suggested-prompt-box {
+  margin-top: 2px !important;
+}
+
+.suggested-prompt-card {
+  margin-top: 10px !important;
+}
 """
 
 client_js = """
 () => {
-  const markerRe = /\\s*\\[\\[psq:([a-z_]+):(0|1)\\]\\]\\s*$/;
+  const markerRe = /\\s*\\[\\[psq:([a-z_]+):(0|1):([A-Za-z0-9_\\-=]*)\\]\\]\\s*$/;
+  const decodeTip = (b64) => {
+    if (!b64) return "";
+    try {
+      const binary = atob((b64 || "").replace(/-/g, "+").replace(/_/g, "/"));
+      const bytes = Uint8Array.from(binary, (c) => c.charCodeAt(0));
+      return new TextDecoder("utf-8").decode(bytes);
+    } catch (_) {
+      return "";
+    }
+  };
   const applyTagMeta = () => {
     const labels = document.querySelectorAll(".lego-tags label");
     labels.forEach((label) => {
@@ -845,6 +1070,14 @@ client_js = """
       if (!match) return;
       label.dataset.psqOrigin = match[1];
       label.dataset.psqPreselected = match[2];
+      const tip = decodeTip(match[3] || "");
+      if (tip) {
+        label.title = tip;
+        span.title = tip;
+      } else {
+        label.removeAttribute("title");
+        span.removeAttribute("title");
+      }
       span.textContent = text.replace(markerRe, "");
     });
   };
@@ -939,12 +1172,11 @@ def rag_pipeline_ui(
         log("Start: received prompt")
         prompt_in = (user_prompt or "").strip()
         if not prompt_in:
-            return _build_ui_payload(
-                console_text="Error: empty prompt",
-                legacy_prompt_text="",
-                row_defs=[],
-                selected_tags=[],
-            )
+            return _build_ui_payload(
+                console_text="Error: empty prompt",
+                row_defs=[],
+                selected_tags=[],
+            )
             
         log("Input:")
         log(prompt_in)
@@ -1001,10 +1233,10 @@ def rag_pipeline_ui(
 
 
         log("Step 2: Prompt Squirrel retrieval (hidden)")
-        try:
-            t0 = time.perf_counter()
-            retrieval_context_tags = list(dict.fromkeys((structural_tags or []) + (probe_tags or [])))
-            rewrite_phrases = [p.strip() for p in (rewrite_for_retrieval or "").split(",") if p.strip()]
+        try:
+            t0 = time.perf_counter()
+            retrieval_context_tags = list(dict.fromkeys((structural_tags or []) + (probe_tags or [])))
+            rewrite_phrases = [p.strip() for p in (rewrite_for_retrieval or "").split(",") if p.strip()]
             retrieval_result = psq_candidates_from_rewrite_phrases(
                 rewrite_phrases=rewrite_phrases,
                 allow_nsfw_tags=allow_nsfw_tags,
@@ -1021,10 +1253,10 @@ def rag_pipeline_ui(
             if selection_candidate_cap > 0 and len(candidates) > selection_candidate_cap:
                 candidates = candidates[:selection_candidate_cap]
                 log(f"Selection candidate cap applied: {selection_candidate_cap}")
-            dt = time.perf_counter()-t0
-            _record_timing("retrieval", dt)
-            log(f"Retrieval: {dt:.2f}s")
-            log(f"Retrieved {len(candidates)} candidate tags")
+            dt = time.perf_counter()-t0
+            _record_timing("retrieval", dt)
+            log(f"Retrieval: {dt:.2f}s")
+            log(f"Retrieved {len(candidates)} candidate tags")
             if verbose_retrieval:
                 log(f"Total unique candidates: {len(candidates)}")
                 limit = None if verbose_retrieval_all else max(1, int(verbose_retrieval_limit))
@@ -1061,11 +1293,19 @@ def rag_pipeline_ui(
                               f"  {tag}{alias_part} | fasttext={fasttext_str} context={context_str} "
                               f"combined={combined_str} count={count}"
                           )
-                    if limit is not None and len(rows) > limit:
-                        log(f"  ... ({len(rows) - limit} more)")
-        except Exception as e:
-            log(f"Retrieval fallback: {type(e).__name__}: {e}")
-            candidates = []
+                    if limit is not None and len(rows) > limit:
+                        log(f"  ... ({len(rows) - limit} more)")
+        except Exception as e:
+            log(f"Retrieval fallback: {type(e).__name__}: {e}")
+            candidates = []
+
+        retrieved_candidate_tags = list(
+            dict.fromkeys(
+                _norm_tag_for_lookup(c.tag)
+                for c in (candidates or [])
+                if getattr(c, "tag", None)
+            )
+        )
 
         log("Step 3: LLM index selection (uses rewrite + structural/probe context)")
         selection_query = _build_selection_query(
@@ -1204,6 +1444,7 @@ def rag_pipeline_ui(
         toggle_rows = _build_toggle_rows(
             seed_terms=seed_terms,
             selected_tags=active_selected_tags,
+            retrieved_candidate_tags=retrieved_candidate_tags,
             tag_selection_origins=tag_selection_origins,
             implied_parent_map=implied_parent_map,
             top_groups=max(1, int(display_top_groups)),
@@ -1213,131 +1454,115 @@ def rag_pipeline_ui(
         dt = time.perf_counter()-t0
         _record_timing("group_display", dt)
         log(f"Ranked group display: {dt:.2f}s ({len(toggle_rows)} rows)")
-        log(
-            _build_display_audit_line(
-                toggle_rows,
-                active_selected_tags=active_selected_tags,
-                direct_selected_tags=direct_selected_tags,
-                implied_selected_tags=implied_selected_tags,
-            )
-        )
-
-        total_dt = time.perf_counter()-t_total0
-        _emit_timing_summary(total_dt)
+        log(
+            _build_display_audit_line(
+                toggle_rows,
+                active_selected_tags=active_selected_tags,
+                direct_selected_tags=direct_selected_tags,
+                implied_selected_tags=implied_selected_tags,
+            )
+        )
+
+        total_dt = time.perf_counter()-t_total0
+        _emit_timing_summary(total_dt)
         _append_timing_jsonl(total_dt)
         log("Done: final prompt ready")
-        return _build_ui_payload(
-            console_text="\n".join(logs),
-            legacy_prompt_text=final_prompt,
-            row_defs=toggle_rows,
-            selected_tags=active_selected_tags,
-        )
+        return _build_ui_payload(
+            console_text="\n".join(logs),
+            row_defs=toggle_rows,
+            selected_tags=active_selected_tags,
+        )
 
     except Exception as e:
         log(f"Error: {type(e).__name__}: {e}")
-        return _build_ui_payload(
-            console_text="\n".join(logs),
-            legacy_prompt_text="",
-            row_defs=[],
-            selected_tags=[],
-        )
+        return _build_ui_payload(
+            console_text="\n".join(logs),
+            row_defs=[],
+            selected_tags=[],
+        )
 
     
 
 with gr.Blocks(css=css, js=client_js) as app:
-    with gr.Row():
-        with gr.Column(scale=3, elem_classes=["prompt-col"]):
-            image_tags = gr.Textbox(
-                label="Enter Prompt",
-                placeholder="e.g. fox, outside, detailed background, .",
-                lines=1
-            )
-        with gr.Column(scale=1):
-            _mascot_pil = _load_mascot_image()
-            if _mascot_pil is not None:
-                mascot_img = gr.Image(
-                    value=_mascot_pil,
-                    show_label=False,
-                    interactive=False,
-                    height=220,
-                    elem_id="mascot"
-                )
-            else:
-                mascot_img = gr.Markdown("`(mascot image unavailable)`")
-            submit_button = gr.Button("Run", variant="primary")
-
-    gr.Markdown(
-        """
-### Prompt Squirrel RAG (pipeline version)
-
-Type a rough prompt. This tool rewrites it and aligns it to an e621-style tag vocabulary using Prompt Squirrel internally,
-then returns a cleaned, model-friendly prompt.
-        """.strip()
-    )
-
-    console = gr.Textbox(
-        label="Console",
-        lines=10,
-        interactive=False,
-        placeholder="Progress logs will appear here."
-    )
-
-    suggested_prompt = gr.Textbox(
-        label="Suggested Prompt (From Toggled Tags)",
-        lines=3,
-        interactive=False,
-        show_copy_button=True,
-        placeholder="Comma-separated tags selected in the rows below."
-    )
-
-    with gr.Accordion("Legacy Pipeline Prompt (for reference)", open=False):
-        legacy_final_prompt = gr.Textbox(
-            label="Legacy Final Prompt",
-            lines=3,
-            interactive=False,
-            show_copy_button=True,
-        )
+    with gr.Row():
+        with gr.Column(scale=3, elem_classes=["prompt-col"]):
+            gr.Markdown(
+                'Describe your image under "Enter Prompt" and click "Run".  '
+                'Prompt Squirrel will translate it into image board tags.',
+                elem_classes=["top-instruction"],
+            )
+            with gr.Group(elem_classes=["prompt-card"]):
+                image_tags = gr.Textbox(
+                    label="Enter Prompt",
+                    placeholder="e.g. fox, outside, detailed background, .",
+                    lines=1,
+                    elem_classes=["enter-prompt-box"],
+                )
+            with gr.Group(elem_classes=["prompt-card", "suggested-prompt-card"]):
+                suggested_prompt = gr.Textbox(
+                    label="Suggested Prompt",
+                    lines=2,
+                    interactive=False,
+                    show_copy_button=True,
+                    placeholder="Comma-separated tags selected in the rows below.",
+                    elem_classes=["suggested-prompt-box"],
+                )
+        with gr.Column(scale=1):
+            _mascot_pil = _load_mascot_image()
+            if _mascot_pil is not None:
+                mascot_img = gr.Image(
+                    value=_mascot_pil,
+                    show_label=False,
+                    interactive=False,
+                    height=240,
+                    elem_id="mascot"
+                )
+            else:
+                mascot_img = gr.Markdown("`(mascot image unavailable)`")
+            submit_button = gr.Button("Run", variant="primary")
+            gr.Markdown("Typical runtime: up to ~20 seconds.", elem_classes=["run-hint"])
 
     selected_tags_state = gr.State([])
     row_defs_state = gr.State([])
     row_values_state = gr.State([])
 
-    gr.Markdown("### Toggle Tag Rows")
+    toggle_instruction = gr.Markdown(
+        "Click tag buttons to add or remove tags from the suggested prompt.",
+        elem_classes=["row-instruction"],
+        visible=False,
+    )
+    row_headers: List[gr.Markdown] = []
+    row_checkboxes: List[gr.CheckboxGroup] = []
+    for _ in range(display_max_rows_default):
+        with gr.Row():
+            with gr.Column(scale=2, min_width=170):
+                row_headers.append(gr.Markdown(value="", visible=False, elem_classes=["row-heading"]))
+            with gr.Column(scale=10):
+                row_checkboxes.append(
+                    gr.CheckboxGroup(
+                        choices=[],
+                        value=[],
+                        visible=False,
+                        interactive=True,
+                        container=False,
+                        elem_classes=["lego-tags"],
+                    )
+                )
+
     gr.HTML(
         """
         <div class="source-legend">
-          <span class="chip"><span class="swatch rewrite"></span>Rewrite phrase</span>
-          <span class="chip"><span class="swatch selection"></span>General selection</span>
-          <span class="chip"><span class="swatch probe"></span>Probe query</span>
-          <span class="chip"><span class="swatch structural"></span>Structural query</span>
-          <span class="chip"><span class="swatch implied"></span>Implied</span>
-          <span class="chip"><span class="swatch user"></span>User-toggled</span>
-          <span class="chip"><span class="swatch unselected"></span>Unselected</span>
+          <span class="legend-title">Legend:</span>
+          <span class="chip rewrite">Rewrite phrase</span>
+          <span class="chip selection">General selection</span>
+          <span class="chip probe">Probe query</span>
+          <span class="chip structural">Structural query</span>
+          <span class="chip implied">Implied</span>
+          <span class="chip user">User-toggled</span>
+          <span class="chip unselected">Unselected</span>
         </div>
         """
     )
-    gr.Markdown(
-        "Rows are ranked by expected tag count (E). Within each row: structural -> probe -> selected, "
-        "implied tags follow their triggering selected tag when possible, then unselected tags in confidence order."
-    )
-    row_headers: List[gr.Markdown] = []
-    row_checkboxes: List[gr.CheckboxGroup] = []
-    for _ in range(display_max_rows_default):
-        row_headers.append(gr.Markdown(value="", visible=False))
-        row_checkboxes.append(
-            gr.CheckboxGroup(
-                choices=[],
-                value=[],
-                visible=False,
-                interactive=True,
-                container=False,
-                elem_classes=["lego-tags"],
-            )
-        )
-
-    gr.Markdown(
-        "Toggling a tag in any row toggles it everywhere else that tag appears."
-    )
 
     with gr.Accordion("Display Settings", open=False):
         with gr.Row():
@@ -1353,43 +1578,63 @@ then returns a cleaned, model-friendly prompt.
                 label="Top Tags Shown Per Row",
                 minimum=1,
             )
-            display_rank_top_k = gr.Number(
-                value=display_rank_top_k_default,
-                precision=0,
-                label="Top Tags Used for Row Ranking",
-                minimum=1,
-            )
-
-    run_outputs = [
-        console,
-        legacy_final_prompt,
-        suggested_prompt,
-        selected_tags_state,
-        row_defs_state,
-        row_values_state,
+            display_rank_top_k = gr.Number(
+                value=display_rank_top_k_default,
+                precision=0,
+                label="Top Tags Used for Row Ranking",
+                minimum=1,
+            )
+
+    with gr.Accordion("Console", open=False):
+        console = gr.Textbox(
+            label="Console",
+            lines=10,
+            interactive=False,
+            placeholder="Progress logs will appear here."
+        )
+
+    run_outputs = [
+        console,
+        toggle_instruction,
+        suggested_prompt,
+        selected_tags_state,
+        row_defs_state,
+        row_values_state,
         *row_headers,
         *row_checkboxes,
     ]
 
-    submit_button.click(
-        rag_pipeline_ui,
-        inputs=[image_tags, display_top_groups, display_top_tags_per_group, display_rank_top_k],
-        outputs=run_outputs
-    )
-
-    image_tags.submit(
-        rag_pipeline_ui,
-        inputs=[image_tags, display_top_groups, display_top_tags_per_group, display_rank_top_k],
-        outputs=run_outputs
-    )
+    submit_button.click(
+        _prepare_run_ui,
+        inputs=[],
+        outputs=run_outputs,
+        queue=False,
+        show_progress="hidden",
+    ).then(
+        rag_pipeline_ui,
+        inputs=[image_tags, display_top_groups, display_top_tags_per_group, display_rank_top_k],
+        outputs=run_outputs,
+    )
+
+    image_tags.submit(
+        _prepare_run_ui,
+        inputs=[],
+        outputs=run_outputs,
+        queue=False,
+        show_progress="hidden",
+    ).then(
+        rag_pipeline_ui,
+        inputs=[image_tags, display_top_groups, display_top_tags_per_group, display_rank_top_k],
+        outputs=run_outputs,
+    )
 
     for idx, row_cb in enumerate(row_checkboxes):
-        row_cb.change(
+        row_cb.select(
             fn=lambda changed_values, selected_state, row_defs, row_values, i=idx: _on_toggle_row(
                 i,
                 changed_values,
-                selected_state,
-                row_defs,
+                selected_state,
+                row_defs,
                 row_values,
                 display_max_rows_default,
             ),

data/runtime_metrics/ui_pipeline_timings.jsonl

@@ -14,3 +14,47 @@
 {"timestamp_utc": "2026-03-07T03:01:39Z", "stages_s": {"preprocess": 1.6900012269616127e-05, "rewrite": 1.713694000034593, "structural": 5.799985956400633e-06, "probe": 0.049874700023792684, "retrieval": 0.35970670002279803, "selection": 0.9267913000076078, "implication_expansion": 3.909994848072529e-05, "prompt_composition": 3.7299992982298136e-05, "group_display": 0.026757099956739694}, "total_s": 3.089661000005435, "config": {"timeout_rewrite_s": 45.0, "timeout_struct_s": 45.0, "timeout_probe_s": 45.0, "timeout_select_s": 45.0}}
 {"timestamp_utc": "2026-03-07T03:09:53Z", "stages_s": {"preprocess": 0.00012510002125054598, "rewrite": 2.249713899975177, "structural": 0.5107482000021264, "probe": 3.300025127828121e-06, "retrieval": 2.3757353999535553, "selection": 2.9089593999669887, "implication_expansion": 0.2682994999922812, "prompt_composition": 3.070000093430281e-05, "group_display": 0.07982710003852844}, "total_s": 8.42714020004496, "config": {"timeout_rewrite_s": 45.0, "timeout_struct_s": 45.0, "timeout_probe_s": 45.0, "timeout_select_s": 45.0}}
 {"timestamp_utc": "2026-03-07T03:37:54Z", "stages_s": {"preprocess": 0.00011760002234950662, "rewrite": 1.968222199997399, "structural": 1.1845426999498159, "probe": 2.214354399999138, "retrieval": 2.452574900002219, "selection": 0.8585481999907643, "implication_expansion": 0.27041040000040084, "prompt_composition": 3.319996176287532e-05, "group_display": 0.07736879994627088}, "total_s": 9.059251800004859, "config": {"timeout_rewrite_s": 45.0, "timeout_struct_s": 45.0, "timeout_probe_s": 45.0, "timeout_select_s": 45.0}}
+{"timestamp_utc": "2026-03-07T14:46:02Z", "stages_s": {"preprocess": 0.00026489997981116176, "rewrite": 1.9126524000312202, "structural": 4.675470399961341, "probe": 2.1218294000136666, "retrieval": 12.559957500023302, "selection": 1.0550536999944597, "implication_expansion": 0.37385119998361915, "prompt_composition": 2.589996438473463e-05, "group_display": 0.1274346000282094}, "total_s": 22.92461790004745, "config": {"timeout_rewrite_s": 45.0, "timeout_struct_s": 45.0, "timeout_probe_s": 45.0, "timeout_select_s": 45.0}}
+{"timestamp_utc": "2026-03-07T15:03:07Z", "stages_s": {"preprocess": 0.0002395000192336738, "rewrite": 5.185094600019511, "structural": 0.7354515999904834, "probe": 3.3999676816165447e-06, "retrieval": 7.362056100042537, "selection": 4.4499055000487715, "implication_expansion": 0.40566330001456663, "prompt_composition": 5.230004899203777e-05, "group_display": 0.13085790001787245}, "total_s": 18.33150090003619, "config": {"timeout_rewrite_s": 45.0, "timeout_struct_s": 45.0, "timeout_probe_s": 45.0, "timeout_select_s": 45.0}}
+{"timestamp_utc": "2026-03-07T15:24:07Z", "stages_s": {"preprocess": 0.00018759997328743339, "rewrite": 12.691507300012745, "structural": 6.099988240748644e-06, "probe": 3.3999676816165447e-06, "retrieval": 2.92752389999805, "selection": 3.720078700047452, "implication_expansion": 0.3129685999592766, "prompt_composition": 2.6300025638192892e-05, "group_display": 0.09746540000196546}, "total_s": 19.792910400021356, "config": {"timeout_rewrite_s": 45.0, "timeout_struct_s": 45.0, "timeout_probe_s": 45.0, "timeout_select_s": 45.0}}
+{"timestamp_utc": "2026-03-07T15:25:37Z", "stages_s": {"preprocess": 1.71000137925148e-05, "rewrite": 7.169413700001314, "structural": 4.699977580457926e-06, "probe": 2.300017513334751e-06, "retrieval": 0.5495104999863543, "selection": 1.0878360999631695, "implication_expansion": 2.940004924312234e-05, "prompt_composition": 2.9200047720223665e-05, "group_display": 0.03320260002510622}, "total_s": 8.852556700003333, "config": {"timeout_rewrite_s": 45.0, "timeout_struct_s": 45.0, "timeout_probe_s": 45.0, "timeout_select_s": 45.0}}
+{"timestamp_utc": "2026-03-07T15:34:51Z", "stages_s": {"preprocess": 8.460000390186906e-05, "rewrite": 20.807663900020998, "structural": 5.00003807246685e-06, "probe": 2.600019797682762e-06, "retrieval": 2.5111192999756895, "selection": 15.921769299951848, "implication_expansion": 0.27366950002033263, "prompt_composition": 3.250001464039087e-05, "group_display": 0.08310980000533164}, "total_s": 39.63195960002486, "config": {"timeout_rewrite_s": 45.0, "timeout_struct_s": 45.0, "timeout_probe_s": 45.0, "timeout_select_s": 45.0}}
+{"timestamp_utc": "2026-03-07T15:35:52Z", "stages_s": {"preprocess": 2.47000134550035e-05, "rewrite": 1.7436033999547362, "structural": 4.400033503770828e-06, "probe": 2.600019797682762e-06, "retrieval": 0.5218271000194363, "selection": 4.346306200022809, "implication_expansion": 3.0399998649954796e-05, "prompt_composition": 2.5800021830946207e-05, "group_display": 0.03283849998842925}, "total_s": 6.659919600002468, "config": {"timeout_rewrite_s": 45.0, "timeout_struct_s": 45.0, "timeout_probe_s": 45.0, "timeout_select_s": 45.0}}
+{"timestamp_utc": "2026-03-07T15:41:27Z", "stages_s": {"preprocess": 9.679998038336635e-05, "rewrite": 3.4287550000008196, "structural": 0.9089086999883875, "probe": 1.9048012000275776, "retrieval": 2.3640123999794014, "selection": 5.8771228000405245, "implication_expansion": 0.2728748000226915, "prompt_composition": 2.9499991796910763e-05, "group_display": 0.08227899996563792}, "total_s": 14.873475200030953, "config": {"timeout_rewrite_s": 45.0, "timeout_struct_s": 45.0, "timeout_probe_s": 45.0, "timeout_select_s": 45.0}}
+{"timestamp_utc": "2026-03-07T15:41:53Z", "stages_s": {"preprocess": 1.7300015315413475e-05, "rewrite": 1.8870358999702148, "structural": 5.099980626255274e-06, "probe": 0.31994039996061474, "retrieval": 0.407905500032939, "selection": 1.6604228000505827, "implication_expansion": 3.070000093430281e-05, "prompt_composition": 2.5900022592395544e-05, "group_display": 0.027770799992140383}, "total_s": 4.315684599976521, "config": {"timeout_rewrite_s": 45.0, "timeout_struct_s": 45.0, "timeout_probe_s": 45.0, "timeout_select_s": 45.0}}
+{"timestamp_utc": "2026-03-07T15:49:25Z", "stages_s": {"preprocess": 0.0001020999625325203, "rewrite": 14.022166899987496, "structural": 4.699977580457926e-06, "probe": 2.4999608285725117e-06, "retrieval": 3.3183021000004373, "selection": 11.949675500043668, "implication_expansion": 0.32175860001007095, "prompt_composition": 4.040001658722758e-05, "group_display": 0.08752110000932589}, "total_s": 29.732599700044375, "config": {"timeout_rewrite_s": 45.0, "timeout_struct_s": 45.0, "timeout_probe_s": 45.0, "timeout_select_s": 45.0}}
+{"timestamp_utc": "2026-03-07T15:50:22Z", "stages_s": {"preprocess": 1.7600017599761486e-05, "rewrite": 1.7429223000071943, "structural": 3.700028173625469e-06, "probe": 0.005793399992398918, "retrieval": 0.4191273999749683, "selection": 1.2126603999640793, "implication_expansion": 3.060000017285347e-05, "prompt_composition": 2.659996971487999e-05, "group_display": 0.027827800018712878}, "total_s": 3.4225500999600627, "config": {"timeout_rewrite_s": 45.0, "timeout_struct_s": 45.0, "timeout_probe_s": 45.0, "timeout_select_s": 45.0}}
+{"timestamp_utc": "2026-03-07T16:00:38Z", "stages_s": {"preprocess": 0.00010899995686486363, "rewrite": 8.79864729999099, "structural": 3.5999692045152187e-06, "probe": 2.100015990436077e-06, "retrieval": 2.8054729999857955, "selection": 1.2764754999661818, "implication_expansion": 0.3174371999921277, "prompt_composition": 2.6399968191981316e-05, "group_display": 0.08990299998549744}, "total_s": 13.324789500038605, "config": {"timeout_rewrite_s": 45.0, "timeout_struct_s": 45.0, "timeout_probe_s": 45.0, "timeout_select_s": 45.0}}
+{"timestamp_utc": "2026-03-07T16:02:59Z", "stages_s": {"preprocess": 2.5200017262250185e-05, "rewrite": 1.991041800007224, "structural": 5.099980626255274e-06, "probe": 17.524970299971756, "retrieval": 0.4314843000029214, "selection": 1.7358130000066012, "implication_expansion": 3.0499999411404133e-05, "prompt_composition": 2.8199981898069382e-05, "group_display": 0.027282700000796467}, "total_s": 21.72257919999538, "config": {"timeout_rewrite_s": 45.0, "timeout_struct_s": 45.0, "timeout_probe_s": 45.0, "timeout_select_s": 45.0}}
+{"timestamp_utc": "2026-03-07T16:04:30Z", "stages_s": {"preprocess": 2.559996210038662e-05, "rewrite": 2.176026900007855, "structural": 5.400041118264198e-06, "probe": 0.19614840002031997, "retrieval": 0.4781980999978259, "selection": 1.6205251999781467, "implication_expansion": 3.789999755099416e-05, "prompt_composition": 3.749999450519681e-05, "group_display": 0.03069300000788644}, "total_s": 4.5152849000296555, "config": {"timeout_rewrite_s": 45.0, "timeout_struct_s": 45.0, "timeout_probe_s": 45.0, "timeout_select_s": 45.0}}
+{"timestamp_utc": "2026-03-07T16:10:49Z", "stages_s": {"preprocess": 9.689998114481568e-05, "rewrite": 2.410708999959752, "structural": 0.8642730999854393, "probe": 0.4224375000339933, "retrieval": 2.802454399992712, "selection": 2.783213499991689, "implication_expansion": 0.31982660002540797, "prompt_composition": 2.7100031729787588e-05, "group_display": 0.09723489999305457}, "total_s": 9.736253200040665, "config": {"timeout_rewrite_s": 45.0, "timeout_struct_s": 45.0, "timeout_probe_s": 45.0, "timeout_select_s": 45.0}}
+{"timestamp_utc": "2026-03-07T16:11:31Z", "stages_s": {"preprocess": 1.71000137925148e-05, "rewrite": 1.7746342000318691, "structural": 0.2111163000226952, "probe": 4.599976819008589e-06, "retrieval": 0.5392439999850467, "selection": 2.572300500003621, "implication_expansion": 3.130000550299883e-05, "prompt_composition": 3.15000070258975e-05, "group_display": 0.03570860001491383}, "total_s": 5.145930600003339, "config": {"timeout_rewrite_s": 45.0, "timeout_struct_s": 45.0, "timeout_probe_s": 45.0, "timeout_select_s": 45.0}}
+{"timestamp_utc": "2026-03-07T16:17:15Z", "stages_s": {"preprocess": 9.919999865815043e-05, "rewrite": 2.202809799986426, "structural": 1.5271345999790356, "probe": 4.593681500002276, "retrieval": 2.751306999998633, "selection": 8.643455000012182, "implication_expansion": 0.35227009997470304, "prompt_composition": 3.160000778734684e-05, "group_display": 0.11034630000358447}, "total_s": 20.237214900029358, "config": {"timeout_rewrite_s": 45.0, "timeout_struct_s": 45.0, "timeout_probe_s": 45.0, "timeout_select_s": 45.0}}
+{"timestamp_utc": "2026-03-07T16:17:43Z", "stages_s": {"preprocess": 2.7300033252686262e-05, "rewrite": 1.0030765000265092, "structural": 2.59619329997804, "probe": 1.0295192999765277, "retrieval": 0.43122639995999634, "selection": 19.92651720001595, "implication_expansion": 2.989999484270811e-05, "prompt_composition": 3.0199997127056122e-05, "group_display": 0.028948699997272342}, "total_s": 25.028504199988674, "config": {"timeout_rewrite_s": 45.0, "timeout_struct_s": 45.0, "timeout_probe_s": 45.0, "timeout_select_s": 45.0}}
+{"timestamp_utc": "2026-03-07T16:30:10Z", "stages_s": {"preprocess": 8.229998638853431e-05, "rewrite": 6.530854899960104, "structural": 0.5428495000232942, "probe": 3.5999692045152187e-06, "retrieval": 2.2112261000438593, "selection": 37.76459150004666, "implication_expansion": 0.26712879998376593, "prompt_composition": 3.42000275850296e-05, "group_display": 0.07745989999966696}, "total_s": 47.426328199973796, "config": {"timeout_rewrite_s": 45.0, "timeout_struct_s": 45.0, "timeout_probe_s": 45.0, "timeout_select_s": 45.0}}
+{"timestamp_utc": "2026-03-07T16:30:23Z", "stages_s": {"preprocess": 1.71000137925148e-05, "rewrite": 0.929964899958577, "structural": 0.0898478000308387, "probe": 1.6804130000527948, "retrieval": 0.41350249998504296, "selection": 9.11415430001216, "implication_expansion": 3.300001844763756e-05, "prompt_composition": 3.2299954909831285e-05, "group_display": 0.027016500011086464}, "total_s": 12.266514900024049, "config": {"timeout_rewrite_s": 45.0, "timeout_struct_s": 45.0, "timeout_probe_s": 45.0, "timeout_select_s": 45.0}}
+{"timestamp_utc": "2026-03-07T16:30:38Z", "stages_s": {"preprocess": 3.239995567128062e-05, "rewrite": 7.538256199972238, "structural": 5.300040356814861e-06, "probe": 2.300017513334751e-06, "retrieval": 0.40969980001682416, "selection": 2.6479469999903813, "implication_expansion": 2.829998265951872e-05, "prompt_composition": 2.4800014216452837e-05, "group_display": 0.027821999974548817}, "total_s": 10.63382319995435, "config": {"timeout_rewrite_s": 45.0, "timeout_struct_s": 45.0, "timeout_probe_s": 45.0, "timeout_select_s": 45.0}}
+{"timestamp_utc": "2026-03-07T16:39:56Z", "stages_s": {"preprocess": 9.499996667727828e-05, "rewrite": 2.1227241000160575, "structural": 0.8582198000513017, "probe": 4.121821600012481, "retrieval": 3.0876275000046007, "selection": 1.8579011999536306, "implication_expansion": 0.3362734999973327, "prompt_composition": 4.47999918833375e-05, "group_display": 0.09885080001549795}, "total_s": 12.54350390000036, "config": {"timeout_rewrite_s": 45.0, "timeout_struct_s": 45.0, "timeout_probe_s": 45.0, "timeout_select_s": 45.0}}
+{"timestamp_utc": "2026-03-07T16:42:46Z", "stages_s": {"preprocess": 1.650000922381878e-05, "rewrite": 1.8503554000053555, "structural": 4.999979864805937e-06, "probe": 0.1099030000041239, "retrieval": 0.4038925000349991, "selection": 3.962773200008087, "implication_expansion": 4.18000272475183e-05, "prompt_composition": 3.519997699186206e-05, "group_display": 0.0322487999801524}, "total_s": 6.370800500037149, "config": {"timeout_rewrite_s": 45.0, "timeout_struct_s": 45.0, "timeout_probe_s": 45.0, "timeout_select_s": 45.0}}
+{"timestamp_utc": "2026-03-07T16:45:56Z", "stages_s": {"preprocess": 3.549997927621007e-05, "rewrite": 2.710496100015007, "structural": 0.1680390000110492, "probe": 4.699977580457926e-06, "retrieval": 1.825858500029426, "selection": 3.9926271999720484, "implication_expansion": 3.15000070258975e-05, "prompt_composition": 5.5100012104958296e-05, "group_display": 0.03422769997268915}, "total_s": 8.741921000008006, "config": {"timeout_rewrite_s": 45.0, "timeout_struct_s": 45.0, "timeout_probe_s": 45.0, "timeout_select_s": 45.0}}
+{"timestamp_utc": "2026-03-07T16:46:30Z", "stages_s": {"preprocess": 3.660004585981369e-05, "rewrite": 3.5675273000379093, "structural": 1.2902518999762833, "probe": 4.299974534660578e-06, "retrieval": 0.6466077999793924, "selection": 0.9123659000033513, "implication_expansion": 2.010003663599491e-05, "prompt_composition": 2.239999594166875e-05, "group_display": 0.031055399973411113}, "total_s": 6.457136899989564, "config": {"timeout_rewrite_s": 45.0, "timeout_struct_s": 45.0, "timeout_probe_s": 45.0, "timeout_select_s": 45.0}}
+{"timestamp_utc": "2026-03-07T16:52:11Z", "stages_s": {"preprocess": 9.279994992539287e-05, "rewrite": 2.4995197000098415, "structural": 0.650494700006675, "probe": 0.13954110001213849, "retrieval": 2.8580662000458688, "selection": 4.111845000006724, "implication_expansion": 0.2978370999917388, "prompt_composition": 2.719997428357601e-05, "group_display": 0.08794649998890236}, "total_s": 10.679650100006256, "config": {"timeout_rewrite_s": 45.0, "timeout_struct_s": 45.0, "timeout_probe_s": 45.0, "timeout_select_s": 45.0}}
+{"timestamp_utc": "2026-03-07T16:54:21Z", "stages_s": {"preprocess": 2.6500027161091566e-05, "rewrite": 3.775604799971916, "structural": 4.799978341907263e-06, "probe": 2.7999631129205227e-06, "retrieval": 0.660035600012634, "selection": 2.061849199992139, "implication_expansion": 2.629996743053198e-05, "prompt_composition": 2.95999925583601e-05, "group_display": 0.03182149998610839}, "total_s": 6.542931500007398, "config": {"timeout_rewrite_s": 45.0, "timeout_struct_s": 45.0, "timeout_probe_s": 45.0, "timeout_select_s": 45.0}}
+{"timestamp_utc": "2026-03-07T16:56:48Z", "stages_s": {"preprocess": 1.1800031643360853e-05, "rewrite": 1.6858424000092782, "structural": 0.3115612000110559, "probe": 3.600027412176132e-06, "retrieval": 0.18196690001059324, "selection": 0.8783706999965943, "implication_expansion": 1.3899989426136017e-05, "prompt_composition": 1.1400028597563505e-05, "group_display": 0.026653499982785434}, "total_s": 3.09556500002509, "config": {"timeout_rewrite_s": 45.0, "timeout_struct_s": 45.0, "timeout_probe_s": 45.0, "timeout_select_s": 45.0}}
+{"timestamp_utc": "2026-03-07T17:00:41Z", "stages_s": {"preprocess": 7.049995474517345e-05, "rewrite": 2.107173199998215, "structural": 0.7115459000342526, "probe": 0.7592908999649808, "retrieval": 2.065408500027843, "selection": 0.881550399994012, "implication_expansion": 0.26592110004276037, "prompt_composition": 1.4399993233382702e-05, "group_display": 0.07751559995813295}, "total_s": 6.901116500026546, "config": {"timeout_rewrite_s": 45.0, "timeout_struct_s": 45.0, "timeout_probe_s": 45.0, "timeout_select_s": 45.0}}
+{"timestamp_utc": "2026-03-07T17:29:51Z", "stages_s": {"preprocess": 8.860003435984254e-05, "rewrite": 2.1579742000321858, "structural": 0.4430788999889046, "probe": 1.0411412000539713, "retrieval": 2.0500706000020728, "selection": 1.186861299967859, "implication_expansion": 0.26447719999123365, "prompt_composition": 1.5300000086426735e-05, "group_display": 0.08299089997308329}, "total_s": 7.258497399976477, "config": {"timeout_rewrite_s": 45.0, "timeout_struct_s": 45.0, "timeout_probe_s": 45.0, "timeout_select_s": 45.0}}
+{"timestamp_utc": "2026-03-07T17:35:31Z", "stages_s": {"preprocess": 1.009996049106121e-05, "rewrite": 2.5867222999804653, "structural": 1.5706295000272803, "probe": 0.5591535000130534, "retrieval": 0.09234020003350452, "selection": 17.545875800016802, "implication_expansion": 3.190001007169485e-05, "prompt_composition": 1.3300043065100908e-05, "group_display": 0.0324972000089474}, "total_s": 22.399744599999394, "config": {"timeout_rewrite_s": 45.0, "timeout_struct_s": 45.0, "timeout_probe_s": 45.0, "timeout_select_s": 45.0}}
+{"timestamp_utc": "2026-03-07T17:38:38Z", "stages_s": {"preprocess": 7.449998520314693e-05, "rewrite": 2.332632600038778, "structural": 0.7940433000330813, "probe": 0.8119671999593265, "retrieval": 2.234404999995604, "selection": 0.7451644000248052, "implication_expansion": 0.27489820000482723, "prompt_composition": 1.6200006939470768e-05, "group_display": 0.07717659999616444}, "total_s": 7.304320800001733, "config": {"timeout_rewrite_s": 45.0, "timeout_struct_s": 45.0, "timeout_probe_s": 45.0, "timeout_select_s": 45.0}}
+{"timestamp_utc": "2026-03-07T17:42:01Z", "stages_s": {"preprocess": 8.510000770911574e-05, "rewrite": 1.867464300012216, "structural": 1.647069400001783, "probe": 4.299974534660578e-06, "retrieval": 2.395426100003533, "selection": 1.200732600002084, "implication_expansion": 0.33177699998486787, "prompt_composition": 2.2199994418770075e-05, "group_display": 0.09814279997954145}, "total_s": 7.579292399983387, "config": {"timeout_rewrite_s": 45.0, "timeout_struct_s": 45.0, "timeout_probe_s": 45.0, "timeout_select_s": 45.0}}
+{"timestamp_utc": "2026-03-07T17:48:44Z", "stages_s": {"preprocess": 8.590001380071044e-05, "rewrite": 2.0132066000369377, "structural": 1.060188300034497, "probe": 0.4243395999656059, "retrieval": 2.132219100021757, "selection": 1.1945027000037953, "implication_expansion": 0.2687906000064686, "prompt_composition": 1.9000028260052204e-05, "group_display": 0.07895809999899939}, "total_s": 7.206461600027978, "config": {"timeout_rewrite_s": 45.0, "timeout_struct_s": 45.0, "timeout_probe_s": 45.0, "timeout_select_s": 45.0}}
+{"timestamp_utc": "2026-03-07T17:54:46Z", "stages_s": {"preprocess": 0.00010950001887977123, "rewrite": 1.3990224999724887, "structural": 2.04190930002369, "probe": 1.069002999982331, "retrieval": 2.1156343999900855, "selection": 1.9406172999879345, "implication_expansion": 0.2861632999847643, "prompt_composition": 1.6000005416572094e-05, "group_display": 0.09219529997790232}, "total_s": 8.978390700009186, "config": {"timeout_rewrite_s": 45.0, "timeout_struct_s": 45.0, "timeout_probe_s": 45.0, "timeout_select_s": 45.0}}
+{"timestamp_utc": "2026-03-07T17:54:54Z", "stages_s": {"preprocess": 2.6899971999228e-05, "rewrite": 1.5525400000042282, "structural": 3.999972250312567e-06, "probe": 2.1999585442245007e-06, "retrieval": 0.10463370004436001, "selection": 1.4364217999973334, "implication_expansion": 1.999997766688466e-05, "prompt_composition": 1.4999997802078724e-05, "group_display": 0.032557499944232404}, "total_s": 3.138574599986896, "config": {"timeout_rewrite_s": 45.0, "timeout_struct_s": 45.0, "timeout_probe_s": 45.0, "timeout_select_s": 45.0}}
+{"timestamp_utc": "2026-03-07T17:58:13Z", "stages_s": {"preprocess": 0.00011359999189153314, "rewrite": 2.2580789999919944, "structural": 1.3124472000054084, "probe": 3.5999692045152187e-06, "retrieval": 2.127778899972327, "selection": 1.0589646000298671, "implication_expansion": 0.29374579997966066, "prompt_composition": 1.4899997040629387e-05, "group_display": 0.08566429995698854}, "total_s": 7.171340400003828, "config": {"timeout_rewrite_s": 45.0, "timeout_struct_s": 45.0, "timeout_probe_s": 45.0, "timeout_select_s": 45.0}}
+{"timestamp_utc": "2026-03-07T18:01:07Z", "stages_s": {"preprocess": 7.969996659085155e-05, "rewrite": 1.745156999968458, "structural": 2.802765399974305, "probe": 0.9890876000281423, "retrieval": 2.023351099982392, "selection": 0.5886470000259578, "implication_expansion": 0.2654718999983743, "prompt_composition": 1.8899969290941954e-05, "group_display": 0.07822809997014701}, "total_s": 8.527449000044726, "config": {"timeout_rewrite_s": 45.0, "timeout_struct_s": 45.0, "timeout_probe_s": 45.0, "timeout_select_s": 45.0}}
+{"timestamp_utc": "2026-03-07T18:03:23Z", "stages_s": {"preprocess": 8.140003774315119e-05, "rewrite": 1.4975244000088423, "structural": 1.186394400021527, "probe": 0.6484065999975428, "retrieval": 2.452991199970711, "selection": 4.813816999958362, "implication_expansion": 0.1623875999939628, "prompt_composition": 4.579999949783087e-05, "group_display": 0.0797012000111863}, "total_s": 10.879897099977825, "config": {"timeout_rewrite_s": 45.0, "timeout_struct_s": 45.0, "timeout_probe_s": 45.0, "timeout_select_s": 45.0}}
+{"timestamp_utc": "2026-03-07T18:06:46Z", "stages_s": {"preprocess": 9.800016414374113e-06, "rewrite": 1.3867014999850653, "structural": 0.39007520000450313, "probe": 5.211147500027437, "retrieval": 0.17631519999122247, "selection": 6.730482600047253, "implication_expansion": 1.7700018361210823e-05, "prompt_composition": 1.2499978765845299e-05, "group_display": 0.031001800030935556}, "total_s": 13.936744500009809, "config": {"timeout_rewrite_s": 45.0, "timeout_struct_s": 45.0, "timeout_probe_s": 45.0, "timeout_select_s": 45.0}}
+{"timestamp_utc": "2026-03-07T18:07:41Z", "stages_s": {"preprocess": 1.0100018698722124e-05, "rewrite": 1.0803023999906145, "structural": 0.15050079999491572, "probe": 4.7244069000007585, "retrieval": 0.1667132000438869, "selection": 0.9119019000208937, "implication_expansion": 1.8200022168457508e-05, "prompt_composition": 1.4199991710484028e-05, "group_display": 0.027934999961871654}, "total_s": 7.071552800014615, "config": {"timeout_rewrite_s": 45.0, "timeout_struct_s": 45.0, "timeout_probe_s": 45.0, "timeout_select_s": 45.0}}
+{"timestamp_utc": "2026-03-07T18:09:40Z", "stages_s": {"preprocess": 1.5700003132224083e-05, "rewrite": 1.418132099963259, "structural": 1.423503800004255, "probe": 4.57354850001866, "retrieval": 0.10137270000996068, "selection": 1.4030677999835461, "implication_expansion": 2.789997961372137e-05, "prompt_composition": 1.8000020645558834e-05, "group_display": 0.03112399997189641}, "total_s": 8.961162999970838, "config": {"timeout_rewrite_s": 45.0, "timeout_struct_s": 45.0, "timeout_probe_s": 45.0, "timeout_select_s": 45.0}}
+{"timestamp_utc": "2026-03-07T18:11:33Z", "stages_s": {"preprocess": 8.610001532360911e-05, "rewrite": 1.9479332999908365, "structural": 1.865794699988328, "probe": 0.06017700000666082, "retrieval": 2.0440989999915473, "selection": 2.4227961000287905, "implication_expansion": 0.28120840003248304, "prompt_composition": 2.069998299703002e-05, "group_display": 0.0792113000061363}, "total_s": 8.734649899997748, "config": {"timeout_rewrite_s": 45.0, "timeout_struct_s": 45.0, "timeout_probe_s": 45.0, "timeout_select_s": 45.0}}
+{"timestamp_utc": "2026-03-07T18:17:24Z", "stages_s": {"preprocess": 0.00012939999578520656, "rewrite": 3.181579700030852, "structural": 0.45629230001941323, "probe": 4.243250800005626, "retrieval": 2.128536299976986, "selection": 4.2171271000406705, "implication_expansion": 0.29443830001400784, "prompt_composition": 2.110004425048828e-05, "group_display": 0.0817057000240311}, "total_s": 14.642313299991656, "config": {"timeout_rewrite_s": 45.0, "timeout_struct_s": 45.0, "timeout_probe_s": 45.0, "timeout_select_s": 45.0}}
+{"timestamp_utc": "2026-03-07T18:20:15Z", "stages_s": {"preprocess": 9.400013368576765e-06, "rewrite": 0.9211662000161596, "structural": 0.49053100001765415, "probe": 0.4501308999606408, "retrieval": 0.08053859998472035, "selection": 1.9059181000338867, "implication_expansion": 3.290001768618822e-05, "prompt_composition": 1.8200022168457508e-05, "group_display": 0.0299964000005275}, "total_s": 3.888701299962122, "config": {"timeout_rewrite_s": 45.0, "timeout_struct_s": 45.0, "timeout_probe_s": 45.0, "timeout_select_s": 45.0}}

psq_rag/parsing/prompt_grammar.py

@@ -1,60 +1,23 @@
-import re
-from lark import Lark, Token
-
-
-#Parser
-grammar=r"""
-!start: (prompt | /[][():]/+)*
-prompt: (emphasized | plain | comma | WHITESPACE)*
-!emphasized: "(" prompt ")"
-        | "(" prompt ":" [WHITESPACE] NUMBER [WHITESPACE] ")"
-comma: ","
-WHITESPACE: /\s+/
-plain: /([^,\\\[\]():|]|\\.)+/
-%import common.SIGNED_NUMBER -> NUMBER
-"""
-
-# Initialize the parser
-parser = Lark(grammar, start='start')
-
-# Function to extract tags
-def extract_tags(tree):
-    tags_with_positions = []
-    def _traverse(node):
-        if isinstance(node, Token) and node.type == '__ANON_1':
-            tag_position = node.start_pos
-            tag_text = node.value
-            tags_with_positions.append((tag_text, tag_position, "tag"))
-        elif not isinstance(node, Token):
-            for child in node.children:
-                _traverse(child)
-    _traverse(tree)
-    return tags_with_positions
-    
-
-
-def build_tag_offsets_dicts(new_image_tags_with_positions):
-    # Structure the data for HighlightedText
-    tag_data = []
-    for tag_text, start_pos, nodetype in new_image_tags_with_positions:
-        # Modify the tag
-        modified_tag = tag_text.replace('_', ' ').replace('\\(', '(').replace('\\)', ')').strip()
-        artist_matrix_tag = tag_text.replace('_', ' ').replace('\\(', '\(').replace('\\)', '\)').strip()
-        tf_idf_matrix_tag = re.sub(r'\\([()])', r'\1', re.sub(r' ', '_', tag_text.strip().removeprefix('by ').removeprefix('by_')))
-        # Calculate the end position based on the original tag length
-        end_pos = start_pos + len(tag_text)
-        # Append the structured data for each tag
-        tag_data.append({
-            "original_tag": tag_text,
-            "start_pos": start_pos,
-            "end_pos": end_pos,
-            "modified_tag": modified_tag,
-            "artist_matrix_tag": artist_matrix_tag,
-            "tf_idf_matrix_tag": tf_idf_matrix_tag, 
-            "node_type": nodetype
-        })
-    return tag_data
-    
-    
-if __name__ == "__main__":
-    print("prompt_grammar.py imports ok")
+from __future__ import annotations
+
+from lark import Lark
+
+
+# Minimal prompt grammar kept for import compatibility and simple parsing use.
+grammar = r"""
+!start: (prompt | /[][():]/+)*
+prompt: (emphasized | plain | comma | WHITESPACE)*
+!emphasized: "(" prompt ")"
+        | "(" prompt ":" [WHITESPACE] NUMBER [WHITESPACE] ")"
+comma: ","
+WHITESPACE: /\s+/
+plain: /([^,\\\[\]():|]|\\.)+/
+%import common.SIGNED_NUMBER -> NUMBER
+"""
+
+parser = Lark(grammar, start="start")
+
+
+def parse_prompt(text: str):
+    return parser.parse(text or "")
+

psq_rag/retrieval/psq_retrieval.py

@@ -8,7 +8,6 @@ import pathlib
 import re
 from collections import Counter, OrderedDict
 from dataclasses import dataclass
-from itertools import islice
 from typing import Any, Dict, Iterable, List, Optional, Sequence, Set, Tuple, Union
 
 import numpy as np
@@ -42,14 +41,6 @@ def _norm_tag_for_lookup(s: str) -> str:
     return s.replace(' ', '_').replace('\\(', '(').replace('\\)', ')')
 
 
-special_tags = ["score:0", "score:1", "score:2", "score:3", "score:4", "score:5", "score:6", "score:7", "score:8", "score:9", "rating:s", "rating:q", "rating:e"]
-def remove_special_tags(original_string):
-    tags = [tag.strip() for tag in original_string.split(",")]
-    remaining_tags = [tag for tag in tags if tag not in special_tags]
-    removed_tags = [tag for tag in tags if tag in special_tags]
-    return ", ".join(remaining_tags), removed_tags
-    
-
 def construct_pseudo_vector(pseudo_doc_terms, idf, term_to_column_index):
     cols, data = [], []
     for term, w in pseudo_doc_terms.items():
@@ -121,23 +112,6 @@ def get_tfidf_reduced_similar_tags(pseudo_doc_terms, allow_nsfw_tags):
     return transformed_sorted_tag_similarity_dict
     
     
-def psq_candidates_from_terms(terms: Sequence[str], *, allow_nsfw_tags: bool, k: int = 300):
-    cand_dict = get_tfidf_reduced_similar_tags(dict(Counter(terms)), allow_nsfw_tags)
-    candidates = list(islice(cand_dict.items(), k))
-    tag_counts = get_tag_counts()
-    return [
-        Candidate(
-            tag=tag,
-            score_combined=float(score),
-            score_fasttext=None,
-            score_context=None,
-            count=tag_counts.get(tag),
-            sources=[],
-        )
-        for tag, score in candidates
-    ]
-
-
 def psq_candidates_from_rewrite_phrases(
     rewrite_phrases: Sequence[str],
     *,

psq_rag/retrieval/state.py

@@ -18,7 +18,6 @@ TFIDF_PATH = pathlib.Path("tf_idf_files_420.joblib")
 NSFW_CSV_PATH = pathlib.Path("word_rating_probabilities.csv")
 NSFW_THRESHOLD = 0.95
 
-HNSW_ART_PATH = pathlib.Path("tfidf_hnsw_artists.bin")
 HNSW_TAG_PATH = pathlib.Path("tfidf_hnsw_tags.bin")
 FASTTEXT_MODEL_PATH = pathlib.Path("e621FastTextModel010Replacement_small.bin")
 TAG_ALIASES_PATH = pathlib.Path("fluffyrock_3m.csv")
@@ -37,9 +36,7 @@ _tag_implications: Optional[Dict[str, List[str]]] = None
 
 
 _hnsw_tag_index: Optional["hnswlib.Index"] = None
-_hnsw_artist_index: Optional["hnswlib.Index"] = None
 _hnsw_tag_count: int = 0
-_hnsw_artist_count: int = 0
 
 # Tag type names inferred from e621 wiki documentation.
 # Numeric IDs come from fluffyrock_3m.csv column 1; mapping is heuristic but
@@ -167,10 +164,6 @@ def get_artist_set() -> Set[str]:
     return _artist_set
 
 
-def is_artist(name: str) -> bool:
-    return name in get_artist_set()
-
-
 def get_fasttext_model() -> Any:
     global _fasttext_model
     if _fasttext_model is not None:
@@ -380,18 +373,6 @@ def get_tfidf_tag_vectors() -> Dict[str, Any]:
     }
     return _tfidf_tag_vectors
 
-
-def retrieval_assets_status() -> Dict[str, bool]:
-    return {
-        "tfidf": TFIDF_PATH.is_file(),
-        "nsfw_csv": NSFW_CSV_PATH.is_file(),
-        "fasttext_model": FASTTEXT_MODEL_PATH.is_file(),
-        "tag_aliases_csv": TAG_ALIASES_PATH.is_file(),
-        "hnsw_tags": HNSW_TAG_PATH.is_file(),
-        "hnsw_artists": HNSW_ART_PATH.is_file(),
-    }
-
-
 def _build_or_load_index(path: pathlib.Path, rows: list[int], rm: np.ndarray, dim: int) -> "hnswlib.Index":
     idx = hnswlib.Index(space="cosine", dim=dim)
     need_build = True
@@ -425,51 +406,26 @@ def _build_or_load_index(path: pathlib.Path, rows: list[int], rm: np.ndarray, di
     return idx
 
 
-def _ensure_hnsw_indexes(need_artists: bool) -> None:
-    global _hnsw_tag_index, _hnsw_artist_index, _hnsw_tag_count, _hnsw_artist_count
+def _ensure_hnsw_indexes() -> None:
+    global _hnsw_tag_index, _hnsw_tag_count
 
     if hnswlib is None:
         return
 
-    if _hnsw_tag_index is not None and (not need_artists or _hnsw_artist_index is not None):
+    if _hnsw_tag_index is not None:
         return
 
     components = get_tfidf_components()
     reduced_matrix = components["reduced_matrix"]
-    row_to_tag = components["row_to_tag"]
     rm = _l2_normalize_rows(reduced_matrix).astype(np.float32)
     n_items, dim = rm.shape
 
-    artist_set = get_artist_set() if need_artists else set()
-    artist_rows: list[int] = []
-    tag_rows: list[int] = []
-
-    for i in range(n_items):
-        tag = row_to_tag.get(i, "")
-        base = tag[3:] if tag.startswith("by_") else tag
-
-        if tag in {"by_unknown_artist", "by_conditional_dnp"}:
-            tag_rows.append(i)
-            continue
-
-        if artist_set and is_artist(base):
-            artist_rows.append(i)
-        else:
-            tag_rows.append(i)
+    tag_rows = list(range(n_items))
 
     _hnsw_tag_index = _build_or_load_index(HNSW_TAG_PATH, tag_rows, rm, dim)
     _hnsw_tag_count = len(tag_rows)
 
-    if need_artists:
-        _hnsw_artist_index = _build_or_load_index(HNSW_ART_PATH, artist_rows, rm, dim)
-        _hnsw_artist_count = len(artist_rows)
-
 
 def get_hnsw_tag_index() -> Tuple[Optional["hnswlib.Index"], int]:
-    _ensure_hnsw_indexes(need_artists=False)
+    _ensure_hnsw_indexes()
     return _hnsw_tag_index, _hnsw_tag_count
-
-
-def get_hnsw_artist_index() -> Tuple[Optional["hnswlib.Index"], int]:
-    _ensure_hnsw_indexes(need_artists=True)
-    return _hnsw_artist_index, _hnsw_artist_count

psq_rag/tagging/categorized_suggestions.py

@@ -205,25 +205,3 @@ def generate_categorized_suggestions(
         categories=categories,
     )
 
-
-
-def get_category_suggestions_dict(
-    categorized: CategorizedTagSuggestions
-) -> Dict[str, List[str]]:
-    """
-    Get simple dict of category -> suggested tags (without scores).
-
-    Args:
-        categorized: The categorized suggestions
-
-    Returns:
-        Dict mapping category_name -> [tag1, tag2, ...]
-    """
-    result = {}
-
-    for cat_name, cat_sugg in categorized.by_category.items():
-        result[cat_name] = [tag for tag, _ in cat_sugg.suggestions]
-
-    result['other'] = [tag for tag, _ in categorized.other_suggestions]
-
-    return result

psq_rag/ui/group_ranked_display.py

@@ -304,42 +304,3 @@ def rank_groups_from_tfidf(
 
     rows_out.sort(key=lambda r: r.expected_count, reverse=True)
     return rows_out[: max(1, int(top_groups))]
-
-
-def _fmt_tag_cell(tag: str, p: float) -> str:
-    safe_tag = tag.replace("|", "\\|")
-    return f"`{safe_tag}` (p={p:.2f}, E={p:.2f})"
-
-
-def render_group_rankings_markdown(
-    seed_terms: Sequence[str],
-    *,
-    top_groups: int,
-    top_tags_per_group: int,
-    group_rank_top_k: int,
-) -> str:
-    rows = rank_groups_from_tfidf(
-        seed_terms,
-        top_groups=top_groups,
-        top_tags_per_group=top_tags_per_group,
-        group_rank_top_k=group_rank_top_k,
-    )
-    if not rows:
-        return "No ranked group display available (insufficient TF-IDF context)."
-
-    k = max(1, int(top_tags_per_group))
-    headers = ["Group/Category", f"Expected Tags (top {max(1, int(group_rank_top_k))})"]
-    headers.extend([f"Tag {i}" for i in range(1, k + 1)])
-    table = [
-        "| " + " | ".join(headers) + " |",
-        "| " + " | ".join(["---"] * len(headers)) + " |",
-    ]
-
-    for row in rows:
-        cells = [row.group_name, f"{row.expected_count:.2f}"]
-        tag_cells = [_fmt_tag_cell(tag, p) for tag, p in row.tags]
-        if len(tag_cells) < k:
-            tag_cells.extend([""] * (k - len(tag_cells)))
-        cells.extend(tag_cells)
-        table.append("| " + " | ".join(cells) + " |")
-    return "\n".join(table)