deploy app, storage, readme

app.py

@@ -11,8 +11,7 @@ import pandas as pd
 from huggingface_hub import hf_hub_download
 
 from storage import VoteStorage
-from results_tab import add_results_tab, build_results_data, load_more_results, on_gallery_select
-from stats_view import format_post_row
+from explorer import add_results_tab, build_results_data, load_more_results, on_gallery_select
 
 LOCAL_DATA_DIR = 'data'
 DEBUG_MODE = os.getenv("DEBUG", "0").lower() in ("1", "true", "yes", "on")
@@ -39,10 +38,11 @@ _pool_group_dfs = {g: gdf for g, gdf in _pool_df.groupby("group")}
 _stats_lock = threading.Lock()
 _stats_last_loaded_at = 0.0
 _stats_by_key: dict[str, tuple[int, int]] = {}
+_classifier_score_by_key: dict[str, float] = {}
 
 
 def _reload_stats_if_due(force: bool = False):
-    global _stats_last_loaded_at, _stats_by_key
+    global _stats_last_loaded_at, _stats_by_key, _classifier_score_by_key
     now = time.time()
     if not force and (now - _stats_last_loaded_at) < STATS_RELOAD_S:
         return
@@ -56,11 +56,15 @@ def _reload_stats_if_due(force: bool = False):
             repo_type="dataset",
             token=RATINGS_APP_TOKEN,
         )
-        _stats_df = pd.read_parquet(stats_path, columns=["item_key", "wins", "losses"])  # [n, 3]
+        _stats_df = pd.read_parquet(stats_path, columns=["item_key", "wins", "losses", "classifier_score"])  # [n, 4]
         _stats_by_key = {
             str(r.item_key): (int(r.wins), int(r.losses))
             for r in _stats_df.itertuples(index=False)
         }
+        _classifier_score_by_key = {
+            str(r.item_key): float(r.classifier_score)
+            for r in _stats_df.itertuples(index=False)
+        }
         _stats_last_loaded_at = now
 
 
@@ -105,6 +109,13 @@ def _commit_oldest_pending(state: dict):
     oldest = pending.pop(0)
     threading.Thread(target=VOTE_STORAGE.append_vote_row, args=(oldest.copy(), oldest.get("winner")), daemon=True).start()
 
+
+def _format_rating_post_row(post_id: int, wins: int, losses: int, label: str | None = None) -> str:
+    total_votes = wins + losses
+    url = f"https://e621.net/posts/{post_id}"
+    row = f"{url} | Times rated: {total_votes}"
+    return f"{label}: {row}" if label else row
+
 def _render_current(state: dict, submit_status: str = "") -> tuple:
     _reload_stats_if_due()
     wins_a, losses_a = _stats_by_key.get(str(state["key_a"]), (0, 0))
@@ -113,8 +124,8 @@ def _render_current(state: dict, submit_status: str = "") -> tuple:
     title_b = "Image B"
     img_a_html = f"<div class=\"rating-card\"><div class=\"rating-card-title\"><strong>{html.escape(title_a)}</strong></div><div class=\"rating-image-frame\"><img src=\"{html.escape(state['url_a'])}\" class=\"rating-image\" loading=\"eager\" referrerpolicy=\"no-referrer\"></div></div>"
     img_b_html = f"<div class=\"rating-card\"><div class=\"rating-card-title\"><strong>{html.escape(title_b)}</strong></div><div class=\"rating-image-frame\"><img src=\"{html.escape(state['url_b'])}\" class=\"rating-image\" loading=\"eager\" referrerpolicy=\"no-referrer\"></div></div>"
-    link_a = format_post_row(state["id_a"], wins_a, losses_a, label="Image A")
-    link_b = format_post_row(state["id_b"], wins_b, losses_b, label="Image B")
+    link_a = _format_rating_post_row(state["id_a"], wins_a, losses_a, label="Image A")
+    link_b = _format_rating_post_row(state["id_b"], wins_b, losses_b, label="Image B")
     can_go_back = bool(state.get("can_go_back"))
     back_md = "[Undo Rating (Ctrl+z)](#back)" if can_go_back else "<span class='subtle-back-link-disabled'>Undo Rating (Ctrl+z)</span>"
     details = f"<span class='subtle-note'>Group: {state['group']}</span>"
@@ -146,13 +157,14 @@ def _load_results(rating_pref: str):
     rating_pref = _normalize_rating_pref(rating_pref)
     _reload_stats_if_due()
     groups = _select_groups(DATASETS[DEFAULT_DATASET], rating_pref)
-    summary, gallery_items, groups_state, page_meta, next_offset, btn_update = build_results_data(
+    summary, score_distribution_plot, distribution_data, gallery_items, groups_state, page_meta, next_offset, btn_update = build_results_data(
         _pool_df,
         groups,
         rating_pref,
         _stats_by_key,
+        _classifier_score_by_key,
     )
-    return summary, gallery_items, btn_update, "Click an image to reveal its ID and link.", groups_state, page_meta, next_offset
+    return summary, score_distribution_plot, distribution_data, gallery_items, btn_update, "Click an image to reveal its ID and link.", groups_state, page_meta, next_offset
 
 # -- Gradio callbacks -------------------------------------------------------
 
@@ -469,6 +481,8 @@ with gr.Blocks(
 
         (
             results_summary_md,
+            results_score_distribution_plot,
+            results_distribution_state,
             results_gallery,
             results_load_more_btn,
             selected_image_md,
@@ -480,6 +494,8 @@ with gr.Blocks(
     outputs = [img_a, img_b, link_a, link_b, back_link, details_md, submit_status_md, state]
     results_outputs = [
         results_summary_md,
+        results_score_distribution_plot,
+        results_distribution_state,
         results_gallery,
         results_load_more_btn,
         selected_image_md,
@@ -499,13 +515,19 @@ with gr.Blocks(
     demo.load(fn=_initial_load, inputs=[state, rating_pref_store, submit_key_store], outputs=[rating_dd, submit_key_tb, *outputs], queue=False, show_progress="hidden")
     demo.load(fn=_load_results, inputs=[rating_pref_store], outputs=results_outputs, queue=False, show_progress="hidden")
     results_load_more_btn.click(
-        fn=lambda g, o: load_more_results(_pool_df, g, _stats_by_key, o),
+        fn=lambda g, o: load_more_results(_pool_df, g, _stats_by_key, _classifier_score_by_key, o),
         inputs=[results_groups_state, results_page_offset_state],
         outputs=[results_gallery, results_page_meta_state, results_page_offset_state, results_load_more_btn],
         queue=False,
         show_progress="hidden",
     )
-    results_gallery.select(fn=on_gallery_select, inputs=[results_page_meta_state], outputs=[selected_image_md], queue=False, show_progress="hidden")
+    results_gallery.select(
+        fn=on_gallery_select,
+        inputs=[results_page_meta_state, results_distribution_state],
+        outputs=[selected_image_md, results_score_distribution_plot],
+        queue=False,
+        show_progress="hidden",
+    )
 
 if __name__ == "__main__":
     demo.launch()

explorer.py

@@ -0,0 +1,188 @@
+import gradio as gr
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+from matplotlib.figure import Figure
+
+PAGE_SIZE = 30
+
+
+def _row_image_url(row) -> str | None:
+    sample_url = row.get("sample_url")
+    if isinstance(sample_url, str) and sample_url:
+        return sample_url
+    image_url = row.get("image_url")
+    if isinstance(image_url, str) and image_url:
+        return image_url
+    return None
+
+
+def _gallery_items(meta: list[dict[str, str | int]]) -> list[tuple[str, str]]:
+    return [(str(item["url"]), f"Score: {float(item['classifier_score']):.4f}") for item in meta]
+
+
+def _distribution_data(
+    pool_df: pd.DataFrame,
+    groups: list[str],
+    classifier_score_by_key: dict[str, float],
+) -> dict[str, list[float] | int]:
+    subset = pool_df[pool_df["group"].isin(set(groups))]["md5"].astype(str)
+    scores = [classifier_score_by_key.get(md5) for md5 in subset]
+    valid_scores = [float(score) for score in scores if score is not None]
+    if not valid_scores:
+        return {"bin_edges": [], "counts": [], "total": 0}
+    counts, bin_edges = np.histogram(valid_scores, bins=40)
+    return {
+        "bin_edges": bin_edges.astype(float).tolist(),
+        "counts": counts.astype(float).tolist(),
+        "total": int(len(valid_scores)),
+    }
+
+
+def _classifier_score_distribution_plot(
+    distribution_data: dict[str, list[float] | int],
+    selected_score: float | None = None,
+) -> Figure:
+    fig, ax = plt.subplots(figsize=(6, 2.2))
+    fig.patch.set_facecolor("#0f1117")
+    ax.set_facecolor("#151922")
+    bin_edges = np.asarray(distribution_data.get("bin_edges", []), dtype=float)
+    counts = np.asarray(distribution_data.get("counts", []), dtype=float)
+    if counts.size > 0 and bin_edges.size == counts.size + 1:
+        widths = np.diff(bin_edges)
+        ax.bar(bin_edges[:-1], counts, width=widths, align="edge", color="#3b82f6", alpha=0.9, edgecolor="#93c5fd", linewidth=0.35)
+        ax.set_ylabel("Count", color="#e5e7eb")
+        if selected_score is not None:
+            ax.axvline(float(selected_score), color="#f97316", linewidth=2.0)
+    else:
+        ax.text(0.5, 0.5, "No classifier scores available.", ha="center", va="center", transform=ax.transAxes, color="#e5e7eb")
+        ax.set_yticks([])
+    ax.set_title("Classifier Score Distribution", color="#f3f4f6")
+    ax.set_xlabel("Classifier score", color="#e5e7eb")
+    ax.tick_params(colors="#d1d5db")
+    for spine in ax.spines.values():
+        spine.set_color("#4b5563")
+    ax.grid(axis="y", color="#374151", alpha=0.4)
+    fig.tight_layout()
+    return fig
+
+
+def _approx_percentile(score: float, distribution_data: dict[str, list[float] | int]) -> float | None:
+    bin_edges = np.asarray(distribution_data.get("bin_edges", []), dtype=float)
+    counts = np.asarray(distribution_data.get("counts", []), dtype=float)
+    total = float(distribution_data.get("total", 0))
+    if total <= 0 or counts.size == 0 or bin_edges.size != counts.size + 1:
+        return None
+    if score <= float(bin_edges[0]):
+        return 0.0
+    if score >= float(bin_edges[-1]):
+        return 100.0
+    idx = int(np.searchsorted(bin_edges, score, side="right") - 1)
+    idx = max(0, min(idx, counts.size - 1))
+    below = float(counts[:idx].sum())
+    left = float(bin_edges[idx])
+    right = float(bin_edges[idx + 1])
+    width = right - left
+    frac = 0.0 if width <= 0 else (float(score) - left) / width
+    frac = max(0.0, min(1.0, frac))
+    return max(0.0, min(100.0, 100.0 * (below + frac * float(counts[idx])) / total))
+
+
+def _build_page_meta(
+    pool_df: pd.DataFrame,
+    groups: list[str],
+    stats_by_key: dict[str, tuple[int, int]],
+    classifier_score_by_key: dict[str, float],
+    offset: int,
+) -> tuple[list[dict[str, str | int]], int, bool, int]:
+    subset = pool_df[pool_df["group"].isin(set(groups))][["id", "md5", "sample_url", "image_url"]]
+    has_sample = subset["sample_url"].notna() & (subset["sample_url"] != "")
+    has_image = subset["image_url"].notna() & (subset["image_url"] != "")
+    filtered = subset[has_sample | has_image]
+    page_df = filtered.iloc[offset:offset + PAGE_SIZE]
+    page_meta: list[dict[str, str | int]] = []
+    for row in page_df.to_dict("records"):
+        url = _row_image_url(row)
+        assert url is not None
+        post_id = int(row["id"])
+        md5 = str(row["md5"])
+        wins, losses = stats_by_key.get(md5, (0, 0))
+        classifier_score = float(classifier_score_by_key.get(md5, 0.0))
+        page_meta.append({"id": post_id, "md5": md5, "url": url, "wins": wins, "losses": losses, "classifier_score": classifier_score})
+    next_offset = offset + len(page_meta)
+    has_more = next_offset < len(filtered)
+    return page_meta, next_offset, has_more, len(filtered)
+
+
+def build_results_data(
+    pool_df: pd.DataFrame,
+    groups: list[str],
+    category_label: str,
+    stats_by_key: dict[str, tuple[int, int]],
+    classifier_score_by_key: dict[str, float],
+) -> tuple[str, Figure, dict[str, list[float] | int], list[tuple[str, str]], list[str], list[dict[str, str | int]], int, dict]:
+    page_meta, next_offset, has_more, total = _build_page_meta(pool_df, groups, stats_by_key, classifier_score_by_key, offset=0)
+    summary = f"Showing {total} images for category: {category_label}."
+    distribution_data = _distribution_data(pool_df, groups, classifier_score_by_key)
+    score_distribution_plot = _classifier_score_distribution_plot(distribution_data)
+    return summary, score_distribution_plot, distribution_data, _gallery_items(page_meta), groups, page_meta, next_offset, gr.update(visible=has_more)
+
+
+def load_more_results(
+    pool_df: pd.DataFrame,
+    groups: list[str],
+    stats_by_key: dict[str, tuple[int, int]],
+    classifier_score_by_key: dict[str, float],
+    offset: int,
+):
+    page_meta, next_offset, has_more, _total = _build_page_meta(pool_df, groups, stats_by_key, classifier_score_by_key, offset=int(offset))
+    return _gallery_items(page_meta), page_meta, next_offset, gr.update(visible=has_more)
+
+
+def on_gallery_select(
+    evt: gr.SelectData,
+    meta: list[dict[str, str | int]],
+    distribution_data: dict[str, list[float] | int],
+) -> tuple[str, Figure]:
+    index = evt.index[0] if isinstance(evt.index, tuple) else evt.index
+    if not isinstance(index, int) or index < 0 or index >= len(meta):
+        return "No image selected.", _classifier_score_distribution_plot(distribution_data)
+    selected = meta[index]
+    post_id = int(selected["id"])
+    md5 = str(selected["md5"])
+    times_rated = int(selected["wins"]) + int(selected["losses"])
+    classifier_score = float(selected["classifier_score"])
+    percentile = _approx_percentile(classifier_score, distribution_data)
+    percentile_text = "n/a" if percentile is None else f"{percentile:.1f}%"
+    info = f"MD5: {md5} | ID: {post_id} | Times rated: {times_rated} | Score: {classifier_score:.4f} | Approx percentile: {percentile_text}\nhttps://e621.net/posts/{post_id}"
+    return info, _classifier_score_distribution_plot(distribution_data, selected_score=classifier_score)
+
+
+def add_results_tab(pool_df: pd.DataFrame):
+    with gr.Tab("Explorer"):
+        results_summary_md = gr.Markdown()
+        results_gallery = gr.Gallery(
+            label="Category Mosaic",
+            columns=[6],
+            object_fit="contain",
+            preview=True,
+            height="auto",
+        )
+        results_load_more_btn = gr.Button("Load more (ArrowDown)", elem_id="btn-results-load-more")
+        selected_image_md = gr.Markdown("Click an image to reveal its ID and link.")
+        results_score_distribution_plot = gr.Plot(label="Classifier score distribution")
+        results_distribution_state = gr.State({"bin_edges": [], "counts": [], "total": 0})
+        results_groups_state = gr.State([])
+        results_page_meta_state = gr.State([])
+        results_page_offset_state = gr.State(0)
+    return (
+        results_summary_md,
+        results_score_distribution_plot,
+        results_distribution_state,
+        results_gallery,
+        results_load_more_btn,
+        selected_image_md,
+        results_groups_state,
+        results_page_meta_state,
+        results_page_offset_state,
+    )