import React, { useEffect, useMemo, useState } from "react" import { api } from "./api" const DEFAULT_SOURCES = [ { id: "safebooru", label: "Safebooru", sfw_policy: "rating:safe", max_content_tags: null }, { id: "danbooru", label: "Danbooru", sfw_policy: "rating:g", max_content_tags: 2 }, ] function pct(value) { if (value == null || Number.isNaN(Number(value))) return "n/a" return `${(Number(value) * 100).toFixed(1)}%` } export default function ClassifierDebugPage() { const [sources, setSources] = useState(DEFAULT_SOURCES) const [source, setSource] = useState("safebooru") const [pullTags, setPullTags] = useState(["1girl", "solo"]) const [count, setCount] = useState(10) const [tagQuery, setTagQuery] = useState("") const [tagOptions, setTagOptions] = useState([]) const [result, setResult] = useState(null) const [loading, setLoading] = useState(false) const [error, setError] = useState("") const [settingsSummary, setSettingsSummary] = useState({ tagger_model: "wd_swinv2_v3", confidence_threshold: 0.6, selected_tags: [], experimental_style_detector_enabled: false, }) const [realismCount, setRealismCount] = useState(12) const [realismCompare, setRealismCompare] = useState(true) const [realismLoading, setRealismLoading] = useState(false) const [realismError, setRealismError] = useState("") const [realismResult, setRealismResult] = useState(null) const [styleCount, setStyleCount] = useState(12) const [styleLoading, setStyleLoading] = useState(false) const [styleError, setStyleError] = useState("") const [styleResult, setStyleResult] = useState(null) const [tagRecallThreshold, setTagRecallThreshold] = useState(0.35) const [tagRecallTopK, setTagRecallTopK] = useState(20) const [tagRecallLoading, setTagRecallLoading] = useState(false) const [tagRecallError, setTagRecallError] = useState("") const [tagRecallResult, setTagRecallResult] = useState(null) const [tagFpThreshold, setTagFpThreshold] = useState(0.6) const [tagFpMinWeight, setTagFpMinWeight] = useState(0.85) const [tagFpAlsoWd, setTagFpAlsoWd] = useState(true) const [tagFpLoading, setTagFpLoading] = useState(false) const [tagFpError, setTagFpError] = useState("") const [tagFpResult, setTagFpResult] = useState(null) const sourceMeta = useMemo( () => sources.find((s) => s.id === source) || sources[0], [sources, source] ) useEffect(() => { api .getSfwDebugSources() .then((data) => { if (Array.isArray(data?.sources) && data.sources.length) { setSources(data.sources) } }) .catch(() => {}) api .getSettings() .then((data) => { const thr = Number(data.confidence_threshold ?? 0.6) setSettingsSummary({ tagger_model: data.tagger_model || "wd_swinv2_v3", confidence_threshold: thr, selected_tags: Array.isArray(data.selected_tags) ? data.selected_tags : [], experimental_style_detector_enabled: Boolean( data.experimental_style_detector_enabled ), }) setTagFpThreshold(thr) }) .catch((err) => setError(err.message)) }, []) useEffect(() => { let cancelled = false const timer = setTimeout(() => { api .getTags(tagQuery, 20) .then((data) => { if (!cancelled) setTagOptions(data.items || []) }) .catch(() => { if (!cancelled) setTagOptions([]) }) }, 150) return () => { cancelled = true clearTimeout(timer) } }, [tagQuery]) const handleAddPullTag = (value) => { if (!value) return const maxTags = sourceMeta?.max_content_tags setPullTags((prev) => { if (prev.includes(value)) return prev if (maxTags != null && prev.length >= maxTags) { setError(`${sourceMeta.label} allows at most ${maxTags} content tag(s).`) return prev } setError("") return [...prev, value] }) } const handleFetchEvaluate = async () => { if (pullTags.length === 0) { setError("Add at least one tag to pull for the SFW debug eval.") return } setLoading(true) setError("") try { const next = await api.runSfwDebugEval({ source, tags: pullTags, count, }) setResult(next) const refreshed = await api.getSettings() setSettingsSummary({ tagger_model: refreshed.tagger_model || next.tagger_model, confidence_threshold: Number( refreshed.confidence_threshold ?? next.confidence_threshold ?? 0.6 ), selected_tags: Array.isArray(refreshed.selected_tags) ? refreshed.selected_tags : next.destination_tags || [], experimental_style_detector_enabled: Boolean( refreshed.experimental_style_detector_enabled ), }) } catch (err) { setError(`SFW debug eval failed: ${err.message}`) } finally { setLoading(false) } } const handleRealismEval = async () => { setRealismLoading(true) setRealismError("") try { const next = await api.runRealismDebugEval({ count_per_class: realismCount, compare_models: realismCompare, tagger_model: realismCompare ? null : settingsSummary.tagger_model, }) setRealismResult(next) } catch (err) { setRealismError(`Realism eval failed: ${err.message}`) } finally { setRealismLoading(false) } } const handleStyleEval = async () => { setStyleLoading(true) setStyleError("") try { const next = await api.runStyleDebugEval({ count_per_class: styleCount, tagger_model: settingsSummary.tagger_model, }) setStyleResult(next) } catch (err) { setStyleError(`Style detector eval failed: ${err.message}`) } finally { setStyleLoading(false) } } const handleTagRecallEval = async () => { setTagRecallLoading(true) setTagRecallError("") try { const next = await api.runTagRecallEval({ models: ["ml_danbooru", "wd_swinv2_v3"], threshold: tagRecallThreshold, top_k: tagRecallTopK, refresh_cache: false, include_items: true, }) setTagRecallResult(next) } catch (err) { setTagRecallError(`Tag recall eval failed: ${err.message}`) } finally { setTagRecallLoading(false) } } const handleTagFpEval = async () => { setTagFpLoading(true) setTagFpError("") try { const next = await api.runTagFpEval({ models: ["ml_danbooru", "wd_swinv2_v3"], tag_threshold: tagFpThreshold, route_threshold: tagFpThreshold, min_weight: tagFpMinWeight, also_wd_threshold: tagFpAlsoWd, refresh_cache: false, }) setTagFpResult(next) } catch (err) { setTagFpError(`Tag FP eval failed: ${err.message}`) } finally { setTagFpLoading(false) } } const multi = realismResult?.multi_model const overall = multi?.overall_conclusion const metrics = realismResult?.metrics || {} const conclusion = realismResult?.conclusion || {} const styleOverall = styleResult?.overall_conclusion || {} const styleReports = styleResult?.reports || [] const tagRecallReports = tagRecallResult?.reports || [] const tagRecallComparison = tagRecallResult?.comparison || {} const tagFpReports = tagFpResult?.reports || [] const tagFpAlt = tagFpResult?.at_wd_general_threshold return (
Curated tag recall / false-positive benchmarks, SFW pull-tag recall, and real-life vs anime separation. Uses saved settings from the main page — no migrate.
ml_danbooru vs wd_swinv2_v3. Metrics are
recall@threshold and recall@top‑K on each sample’s desired tags (dense WD scores).
Pin/download images once via{" "}
scripts/fetch_tag_recall_suite.py --rebuild.
Pairwise @threshold: {pair.model_a} wins {pair.a_wins}, {pair.model_b} wins{" "} {pair.b_wins}, ties {pair.ties} (n={pair.compared})
))}| Model | Micro @thr | Micro @topK | Macro @thr | Macro @topK | Opp | N |
|---|---|---|---|---|---|---|
| {row.tagger_model} | {pct(s.micro_recall_at_threshold)} | {pct(s.micro_recall_at_top_k)} | {pct(s.macro_recall_at_threshold)} | {pct(s.macro_recall_at_top_k)} | {s.opportunities ?? 0} | {row.count_evaluated} |
| Tag | {tagRecallReports.map((r) => ({r.tagger_model} @thr | ))} {tagRecallReports.map((r) => ({r.tagger_model} @topK | ))}
|---|---|---|
| {tag} | {tagRecallReports.map((r) => ({pct(r.summary?.per_tag?.[tag]?.recall_at_threshold)} | ))} {tagRecallReports.map((r) => ({pct(r.summary?.per_tag?.[tag]?.recall_at_top_k)} | ))}
| Preview | Bucket | Post | {tagRecallReports.map((r) => ({r.tagger_model} | ))}
|---|---|---|---|
|
{preview ? (
|
{item.bucket_id} | {item.source}/{item.post_id} | {tagRecallReports.map((r) => { const match = (r.items || []).find( (x) => x.source === item.source && x.post_id === item.post_id ) const parts = (match?.tag_results || []).map((tr) => { const marks = [ tr.hit_at_threshold ? "T" : "·", tr.hit_at_top_k ? "K" : "·", ].join("") const score = tr.score == null ? "—" : Number(tr.score).toFixed(2) return `${tr.tag}:${score}[${marks}]` }) return (
{parts.join(" · ") || "—"}
|
)
})}
Selected:{" "} {(settingsSummary.selected_tags || []).length ? settingsSummary.selected_tags.join(", ") : "(none — pick folders on THR3SHR settings)"}
| Model | Micro FPR | Macro FPR | FP / neg | Route FPR | Route FP | N |
|---|---|---|---|---|---|---|
| {row.tagger_model} | {pct(tf.micro_fpr)} | {pct(tf.macro_fpr)} | {tf.false_positives ?? 0}/{tf.negatives ?? 0} | {pct(rf.fpr)} | {rf.false_positives ?? 0}/{rf.evaluated ?? 0} | {row.count_evaluated} |
| Model | Micro FPR | Route FPR |
|---|---|---|
| {row.tagger_model} | {pct(row.tag_fp?.micro_fpr)} | {pct(row.folder_route_fp?.fpr)} |
| Tag | {tagFpReports.map((r) => ({r.tagger_model} FPR | ))} {tagFpReports.map((r) => ({r.tagger_model} fp/neg | ))}
|---|---|---|
| {tag} | {tagFpReports.map((r) => ({pct(rowsByModel[r.tagger_model]?.fpr)} | ))} {tagFpReports.map((r) => { const row = rowsByModel[r.tagger_model] return ({row ? `${row.false_positives}/${row.negatives}` : "0"} | ) })}
| Preview | Bucket | Truth roots | {tagFpReports.map((r) => ({r.tagger_model} | ))}|
|---|---|---|---|---|
|
{preview ? (
|
{item.bucket_id} | {(item.truth || []).join(", ") || "—"} | {tagFpReports.map((r) => { const match = (r.sample_route_fps || []).find( (x) => x.source === item.source && x.post_id === item.post_id ) if (!match) { return (— | ) } const score = match.score == null ? "" : ` (${Number(match.score).toFixed(2)})` return ({match.predicted || "—"} {score} | ) })}
deepghs/anime_real_cls via imgutils) vs WD{" "}
real_life taxonomy. Same photo/anime corpus as below. Enable the
experimental style gate on the main THR3SHR settings (next to GIF/video) to use
CAFormer in production runs.
WD baseline uses settings model: {settingsSummary.tagger_model} {" · "} Experimental style gate:{" "} {settingsSummary.experimental_style_detector_enabled ? "ON" : "OFF"} {" "} (toggle on THR3SHR settings)
{styleOverall.summary}
{styleOverall.note}
| Detector | Decision | Typical P | Typical R | Typical F1 | Anime FP | Uncertain | N |
|---|---|---|---|---|---|---|---|
| {row.detector_id} | {row.conclusion?.decision} | {pct(t.precision)} | {pct(t.recall)} | {pct(t.f1)} | {pct(t.anime_false_positive_rate)} | {row.conclusion?.uncertain_count ?? 0} | {row.count_evaluated} |
| Preview | Truth | {styleReports.map((r) => ({r.detector_id} | ))}|
|---|---|---|---|
|
{preview ? (
|
{item.label}
{item.bucket}
|
{styleReports.map((r) => {
const cell = (r.items || [])[idx]
if (!cell) return — | return (
{cell.predicted_bucket}
{pct(cell.confidence)} {cell.correct ? "ok" : "miss"}
|
)
})}
real_life taxonomy
routing.
Current settings model: {settingsSummary.tagger_model} {!realismCompare ? " (used when compare is off)" : " (ignored while comparing)"}
{(overall && overall.summary) || conclusion.summary}
{(overall && overall.video_and_gif) || conclusion.video_note}
| Model | Decision | P | R | F1 | Anime FP | N |
|---|---|---|---|---|---|---|
| {row.tagger_model} | {row.conclusion?.decision} | {pct(row.metrics?.precision)} | {pct(row.metrics?.recall)} | {pct(row.metrics?.f1)} | {pct(row.metrics?.anime_false_positive_rate)} | {row.count_evaluated} |
| Preview | Truth | Predicted | Evidence | Folder |
|---|---|---|---|---|
|
{previewUrl ? (
|
{item.bucket}
{item.label} · {item.query}
|
{item.predicted_bucket}{" "} {item.correct ? ( ok ) : ( miss )} | {evText} | {item.primary_folder ? `${item.primary_folder} (${Number(item.primary_score || 0).toFixed(3)})` : "—"} |
Model: {settingsSummary.tagger_model} · Threshold:{" "} {Number(settingsSummary.confidence_threshold).toFixed(2)} · Destination tags:{" "} {settingsSummary.selected_tags.length ? settingsSummary.selected_tags.join(", ") : "(none — classify preview will need review)"} {sourceMeta?.max_content_tags != null ? ` · ${sourceMeta.label} max content tags: ${sourceMeta.max_content_tags}` : ""}
{result.query}
Evaluated: {result.count_evaluated}/{result.count_requested}
| Preview | Post | Pull scores | Primary | Review |
|---|---|---|---|---|
|
{previewUrl ? (
|
{item.post_id}
{(item.known_tags || []).join(", ")}
|
{pullText} | {item.primary_tag ? `${item.primary_tag} (${Number(item.primary_score).toFixed(3)})` : "—"} | {item.needs_review ? item.review_reason || "needs review" : "ok"} |