import React, { useEffect, useMemo, useState } from "react" import { api } from "./api" const DEFAULT_SOURCES = [ { id: "safebooru", label: "Safebooru", sfw_policy: "rating:safe", max_content_tags: null }, { id: "danbooru", label: "Danbooru", sfw_policy: "rating:g", max_content_tags: 2 }, ] function pct(value) { if (value == null || Number.isNaN(Number(value))) return "n/a" return `${(Number(value) * 100).toFixed(1)}%` } export default function ClassifierDebugPage() { const [sources, setSources] = useState(DEFAULT_SOURCES) const [source, setSource] = useState("safebooru") const [pullTags, setPullTags] = useState(["1girl", "solo"]) const [count, setCount] = useState(10) const [tagQuery, setTagQuery] = useState("") const [tagOptions, setTagOptions] = useState([]) const [result, setResult] = useState(null) const [loading, setLoading] = useState(false) const [error, setError] = useState("") const [settingsSummary, setSettingsSummary] = useState({ tagger_model: "wd_swinv2_v3", confidence_threshold: 0.6, selected_tags: [], experimental_style_detector_enabled: false, }) const [realismCount, setRealismCount] = useState(12) const [realismCompare, setRealismCompare] = useState(true) const [realismLoading, setRealismLoading] = useState(false) const [realismError, setRealismError] = useState("") const [realismResult, setRealismResult] = useState(null) const [styleCount, setStyleCount] = useState(12) const [styleLoading, setStyleLoading] = useState(false) const [styleError, setStyleError] = useState("") const [styleResult, setStyleResult] = useState(null) const [tagRecallThreshold, setTagRecallThreshold] = useState(0.35) const [tagRecallTopK, setTagRecallTopK] = useState(20) const [tagRecallLoading, setTagRecallLoading] = useState(false) const [tagRecallError, setTagRecallError] = useState("") const [tagRecallResult, setTagRecallResult] = useState(null) const [tagFpThreshold, setTagFpThreshold] = useState(0.6) const [tagFpMinWeight, setTagFpMinWeight] = useState(0.85) const [tagFpAlsoWd, setTagFpAlsoWd] = useState(true) const [tagFpLoading, setTagFpLoading] = useState(false) const [tagFpError, setTagFpError] = useState("") const [tagFpResult, setTagFpResult] = useState(null) const sourceMeta = useMemo( () => sources.find((s) => s.id === source) || sources[0], [sources, source] ) useEffect(() => { api .getSfwDebugSources() .then((data) => { if (Array.isArray(data?.sources) && data.sources.length) { setSources(data.sources) } }) .catch(() => {}) api .getSettings() .then((data) => { const thr = Number(data.confidence_threshold ?? 0.6) setSettingsSummary({ tagger_model: data.tagger_model || "wd_swinv2_v3", confidence_threshold: thr, selected_tags: Array.isArray(data.selected_tags) ? data.selected_tags : [], experimental_style_detector_enabled: Boolean( data.experimental_style_detector_enabled ), }) setTagFpThreshold(thr) }) .catch((err) => setError(err.message)) }, []) useEffect(() => { let cancelled = false const timer = setTimeout(() => { api .getTags(tagQuery, 20) .then((data) => { if (!cancelled) setTagOptions(data.items || []) }) .catch(() => { if (!cancelled) setTagOptions([]) }) }, 150) return () => { cancelled = true clearTimeout(timer) } }, [tagQuery]) const handleAddPullTag = (value) => { if (!value) return const maxTags = sourceMeta?.max_content_tags setPullTags((prev) => { if (prev.includes(value)) return prev if (maxTags != null && prev.length >= maxTags) { setError(`${sourceMeta.label} allows at most ${maxTags} content tag(s).`) return prev } setError("") return [...prev, value] }) } const handleFetchEvaluate = async () => { if (pullTags.length === 0) { setError("Add at least one tag to pull for the SFW debug eval.") return } setLoading(true) setError("") try { const next = await api.runSfwDebugEval({ source, tags: pullTags, count, }) setResult(next) const refreshed = await api.getSettings() setSettingsSummary({ tagger_model: refreshed.tagger_model || next.tagger_model, confidence_threshold: Number( refreshed.confidence_threshold ?? next.confidence_threshold ?? 0.6 ), selected_tags: Array.isArray(refreshed.selected_tags) ? refreshed.selected_tags : next.destination_tags || [], experimental_style_detector_enabled: Boolean( refreshed.experimental_style_detector_enabled ), }) } catch (err) { setError(`SFW debug eval failed: ${err.message}`) } finally { setLoading(false) } } const handleRealismEval = async () => { setRealismLoading(true) setRealismError("") try { const next = await api.runRealismDebugEval({ count_per_class: realismCount, compare_models: realismCompare, tagger_model: realismCompare ? null : settingsSummary.tagger_model, }) setRealismResult(next) } catch (err) { setRealismError(`Realism eval failed: ${err.message}`) } finally { setRealismLoading(false) } } const handleStyleEval = async () => { setStyleLoading(true) setStyleError("") try { const next = await api.runStyleDebugEval({ count_per_class: styleCount, tagger_model: settingsSummary.tagger_model, }) setStyleResult(next) } catch (err) { setStyleError(`Style detector eval failed: ${err.message}`) } finally { setStyleLoading(false) } } const handleTagRecallEval = async () => { setTagRecallLoading(true) setTagRecallError("") try { const next = await api.runTagRecallEval({ models: ["ml_danbooru", "wd_swinv2_v3"], threshold: tagRecallThreshold, top_k: tagRecallTopK, refresh_cache: false, include_items: true, }) setTagRecallResult(next) } catch (err) { setTagRecallError(`Tag recall eval failed: ${err.message}`) } finally { setTagRecallLoading(false) } } const handleTagFpEval = async () => { setTagFpLoading(true) setTagFpError("") try { const next = await api.runTagFpEval({ models: ["ml_danbooru", "wd_swinv2_v3"], tag_threshold: tagFpThreshold, route_threshold: tagFpThreshold, min_weight: tagFpMinWeight, also_wd_threshold: tagFpAlsoWd, refresh_cache: false, }) setTagFpResult(next) } catch (err) { setTagFpError(`Tag FP eval failed: ${err.message}`) } finally { setTagFpLoading(false) } } const multi = realismResult?.multi_model const overall = multi?.overall_conclusion const metrics = realismResult?.metrics || {} const conclusion = realismResult?.conclusion || {} const styleOverall = styleResult?.overall_conclusion || {} const styleReports = styleResult?.reports || [] const tagRecallReports = tagRecallResult?.reports || [] const tagRecallComparison = tagRecallResult?.comparison || {} const tagFpReports = tagFpResult?.reports || [] const tagFpAlt = tagFpResult?.at_wd_general_threshold return (
THR3SHR Debug

THR3SHR debug

Curated tag recall / false-positive benchmarks, SFW pull-tag recall, and real-life vs anime separation. Uses saved settings from the main page — no migrate.

{(error || realismError || styleError || tagRecallError || tagFpError) && (
{error || realismError || styleError || tagRecallError || tagFpError}
)}

Tag recall benchmark

Fixed curated suite: ml_danbooru vs wd_swinv2_v3. Metrics are recall@threshold and recall@top‑K on each sample’s desired tags (dense WD scores). Pin/download images once via{" "} scripts/fetch_tag_recall_suite.py --rebuild.
{tagRecallResult && (

Tag recall results

Best: {tagRecallComparison.best_model || "—"} Samples: {tagRecallResult.count_samples_cached ?? 0} thr={tagRecallResult.threshold} · topK={tagRecallResult.top_k}
{(tagRecallComparison.pairwise || []).map((pair) => (

Pairwise @threshold: {pair.model_a} wins {pair.a_wins}, {pair.model_b} wins{" "} {pair.b_wins}, ties {pair.ties} (n={pair.compared})

))}
{tagRecallReports.map((row) => { const s = row.summary || {} return ( ) })}
Model Micro @thr Micro @topK Macro @thr Macro @topK Opp N
{row.tagger_model} {pct(s.micro_recall_at_threshold)} {pct(s.micro_recall_at_top_k)} {pct(s.macro_recall_at_threshold)} {pct(s.macro_recall_at_top_k)} {s.opportunities ?? 0} {row.count_evaluated}
{tagRecallReports[0] && ( <>

Per-tag (side by side)

{tagRecallReports.map((r) => ( ))} {tagRecallReports.map((r) => ( ))} {Array.from( new Set( tagRecallReports.flatMap((r) => Object.keys(r.summary?.per_tag || {}) ) ) ) .sort() .map((tag) => ( {tagRecallReports.map((r) => ( ))} {tagRecallReports.map((r) => ( ))} ))}
Tag {r.tagger_model} @thr {r.tagger_model} @topK
{tag} {pct(r.summary?.per_tag?.[tag]?.recall_at_threshold)} {pct(r.summary?.per_tag?.[tag]?.recall_at_top_k)}
)} {tagRecallReports[0]?.items?.length > 0 && ( <>

Samples

{tagRecallReports.map((r) => ( ))} {(tagRecallReports[0].items || []).map((item) => { const preview = api.getTagRecallPreviewUrl( item.source, item.file_name ) return ( {tagRecallReports.map((r) => { const match = (r.items || []).find( (x) => x.source === item.source && x.post_id === item.post_id ) const parts = (match?.tag_results || []).map((tr) => { const marks = [ tr.hit_at_threshold ? "T" : "·", tr.hit_at_top_k ? "K" : "·", ].join("") const score = tr.score == null ? "—" : Number(tr.score).toFixed(2) return `${tr.tag}:${score}[${marks}]` }) return ( ) })} ) })}
Preview Bucket Post {r.tagger_model}
{preview ? ( ) : ( "—" )} {item.bucket_id} {item.source}/{item.post_id} {parts.join(" · ") || "—"}
)}
)}

Tag false-positive benchmark

Same curated suite. Probes taxonomy evidence tags (weight ≥ min) for your{" "} selected folders from Settings. FP = score ≥ threshold when the Danbooru post does not list that tag. Also reports folder-routing FPs.

Selected:{" "} {(settingsSummary.selected_tags || []).length ? settingsSummary.selected_tags.join(", ") : "(none — pick folders on THR3SHR settings)"}

{tagFpResult && (

False-positive results

Samples: {tagFpResult.count_samples ?? 0} Probe tags: {tagFpResult.probe_tag_count ?? 0} thr={tagFpResult.tag_threshold} · minW={tagFpResult.min_weight}
{tagFpReports.map((row) => { const tf = row.tag_fp || {} const rf = row.folder_route_fp || {} return ( ) })}
Model Micro FPR Macro FPR FP / neg Route FPR Route FP N
{row.tagger_model} {pct(tf.micro_fpr)} {pct(tf.macro_fpr)} {tf.false_positives ?? 0}/{tf.negatives ?? 0} {pct(rf.fpr)} {rf.false_positives ?? 0}/{rf.evaluated ?? 0} {row.count_evaluated}
{tagFpAlt?.reports?.length > 0 && ( <>

Also at WD general thr={tagFpAlt.tag_threshold}

{tagFpAlt.reports.map((row) => ( ))}
Model Micro FPR Route FPR
{row.tagger_model} {pct(row.tag_fp?.micro_fpr)} {pct(row.folder_route_fp?.fpr)}
)}

Top false-positive tags (side by side)

{tagFpReports.map((r) => ( ))} {tagFpReports.map((r) => ( ))} {Array.from( new Set( tagFpReports.flatMap((r) => (r.top_false_positive_tags || []).map((t) => t.tag) ) ) ) .slice(0, 25) .map((tag) => { const rowsByModel = Object.fromEntries( tagFpReports.map((r) => [ r.tagger_model, (r.top_false_positive_tags || []).find((t) => t.tag === tag), ]) ) return ( {tagFpReports.map((r) => ( ))} {tagFpReports.map((r) => { const row = rowsByModel[r.tagger_model] return ( ) })} ) })}
Tag {r.tagger_model} FPR {r.tagger_model} fp/neg
{tag} {pct(rowsByModel[r.tagger_model]?.fpr)} {row ? `${row.false_positives}/${row.negatives}` : "0"}
{tagFpReports.some((r) => (r.sample_route_fps || []).length > 0) && ( <>

Folder route false positives

{tagFpReports.map((r) => ( ))} {Array.from( new Map( tagFpReports .flatMap((r) => r.sample_route_fps || []) .map((i) => [`${i.source}:${i.post_id}`, i]) ).values() ).map((item) => { const preview = api.getTagRecallPreviewUrl( item.source, item.file_name ) return ( {tagFpReports.map((r) => { const match = (r.sample_route_fps || []).find( (x) => x.source === item.source && x.post_id === item.post_id ) if (!match) { return ( ) } const score = match.score == null ? "" : ` (${Number(match.score).toFixed(2)})` return ( ) })} ) })}
Preview Bucket Truth roots {r.tagger_model}
{preview ? ( ) : ( "—" )} {item.bucket_id} {(item.truth || []).join(", ") || "—"} {match.predicted || "—"} {score}
)}
)}

Style detectors (real vs anime)

Debug-only compare: dedicated ONNX classifiers ( deepghs/anime_real_cls via imgutils) vs WD{" "} real_life taxonomy. Same photo/anime corpus as below. Enable the experimental style gate on the main THR3SHR settings (next to GIF/video) to use CAFormer in production runs.

WD baseline uses settings model: {settingsSummary.tagger_model} {" · "} Experimental style gate:{" "} {settingsSummary.experimental_style_detector_enabled ? "ON" : "OFF"} {" "} (toggle on THR3SHR settings)

{styleResult && (

Style detector results

Decision: {styleOverall.decision || "—"} Best: {styleOverall.best_detector || "—"} Paths: {styleResult.count_paths} (photos {styleResult.count_photos_fetched}, anime {styleResult.count_anime_fetched})

{styleOverall.summary}

{styleOverall.note}

Per detector

{styleReports.map((row) => { const t = row.conclusion?.typical_metrics || {} return ( ) })}
Detector Decision Typical P Typical R Typical F1 Anime FP Uncertain N
{row.detector_id} {row.conclusion?.decision} {pct(t.precision)} {pct(t.recall)} {pct(t.f1)} {pct(t.anime_false_positive_rate)} {row.conclusion?.uncertain_count ?? 0} {row.count_evaluated}
{styleReports[0]?.items?.length > 0 && ( <>

Samples ({styleResult.best_detector || styleReports[0].detector_id})

{styleReports.map((r) => ( ))} {(styleReports[0].items || []).map((item, idx) => { const preview = api.getRealismDebugPreviewUrl( item.source, item.file_name ) return ( {styleReports.map((r) => { const cell = (r.items || [])[idx] if (!cell) return return ( ) })} ) })}
Preview Truth {r.detector_id}
{preview ? ( {item.title ) : ( "—" )} {item.label}
{item.bucket}
{cell.predicted_bucket}
{pct(cell.confidence)} {cell.correct ? "ok" : "miss"}
)}
)}

Real-life vs anime

Remote people portraits (RandomUser CDN) vs Safebooru anime, including realistic / photorealistic / 3d edge cases. Scores through production real_life taxonomy routing.

Current settings model: {settingsSummary.tagger_model} {!realismCompare ? " (used when compare is off)" : " (ignored while comparing)"}

{realismResult && (

Realism results

Decision:{" "} {(overall && overall.decision) || conclusion.decision || "—"} Evaluated: {realismResult.count_evaluated} (photos fetched{" "} {realismResult.count_photos_fetched}, anime {realismResult.count_anime_fetched}) Model shown: {realismResult.tagger_model} {overall?.best_model ? Best model: {overall.best_model} : null}

{(overall && overall.summary) || conclusion.summary}

{(overall && overall.video_and_gif) || conclusion.video_note}

Typical photo vs anime (GO gate)

{[ ["Precision", (conclusion.typical_metrics || metrics).precision], ["Recall", (conclusion.typical_metrics || metrics).recall], ["F1", (conclusion.typical_metrics || metrics).f1], [ "Anime FP", (conclusion.typical_metrics || metrics).anime_false_positive_rate, ], ].map(([label, value]) => (
{label} {pct(value)}
))}
Edge quarantine {conclusion.edge_quarantine_count ?? "—"}/{conclusion.edge_sample_count ?? "—"}

Including photoreal/3d edges

Precision {pct(metrics.precision)}
Recall {pct(metrics.recall)}
Anime FP rate {pct(metrics.anime_false_positive_rate)}
{multi?.reports?.length ? ( <>

Per-model

{multi.reports.map((row) => ( ))}
Model Decision P R F1 Anime FP N
{row.tagger_model} {row.conclusion?.decision} {pct(row.metrics?.precision)} {pct(row.metrics?.recall)} {pct(row.metrics?.f1)} {pct(row.metrics?.anime_false_positive_rate)} {row.count_evaluated}
) : null} {realismResult.by_label && Object.keys(realismResult.by_label).length > 0 ? ( <>

Edge / label breakdown

{Object.entries(realismResult.by_label).map(([label, row]) => (
{label} acc {pct(row.accuracy)} (n={row.count})
))}
) : null} {realismResult.errors?.length > 0 && (
{realismResult.errors.join(" · ")}
)}

Samples

{(realismResult.items || []).map((item) => { const previewUrl = api.getRealismDebugPreviewUrl(item.source, item.file_name) const ev = item.evidence_scores || {} const evText = ["realistic", "photorealistic", "photo_(medium)", "3d"] .map((tag) => `${tag}:${Number(ev[tag] || 0).toFixed(2)}`) .join(" ") return ( ) })}
Preview Truth Predicted Evidence Folder
{previewUrl ? ( {item.title ) : ( n/a )} {item.bucket}
{item.label} · {item.query}
{item.predicted_bucket}{" "} {item.correct ? ( ok ) : ( miss )} {evText} {item.primary_folder ? `${item.primary_folder} (${Number(item.primary_score || 0).toFixed(3)})` : "—"}
)}

SFW tag recall

Safebooru / Danbooru only. Change tagger and destination tags on the main THR3SHR page, then return here.

Model: {settingsSummary.tagger_model} · Threshold:{" "} {Number(settingsSummary.confidence_threshold).toFixed(2)} · Destination tags:{" "} {settingsSummary.selected_tags.length ? settingsSummary.selected_tags.join(", ") : "(none — classify preview will need review)"} {sourceMeta?.max_content_tags != null ? ` · ${sourceMeta.label} max content tags: ${sourceMeta.max_content_tags}` : ""}

{tagOptions.map((tag) => ( ))}
{pullTags.length === 0 ? ( No pull tags yet ) : ( pullTags.map((tag) => ( {tag} )) )}
{result && (

SFW results

Query: {result.query} Evaluated: {result.count_evaluated}/{result.count_requested}
{result.errors?.length > 0 && (
{result.errors.join(" · ")}
)}

Recall @ {Number(result.confidence_threshold).toFixed(2)}

{(result.recall || []).map((row) => (
{row.tag} {row.hit_rate == null ? "n/a" : `${(row.hit_rate * 100).toFixed(0)}% (${row.hits_at_threshold}/${row.present_in_posts})`}
))}

Classify preview

{(result.items || []).map((item) => { const previewUrl = api.getSfwDebugPreviewUrl(item.source, item.file_name) const pullText = Object.entries(item.pull_tag_scores || {}) .map(([tag, score]) => `${tag}:${score == null ? "—" : Number(score).toFixed(2)}` ) .join(" ") return ( ) })}
Preview Post Pull scores Primary Review
{previewUrl ? ( {`Post ) : ( n/a )} {item.post_id}
{(item.known_tags || []).join(", ")}
{pullText} {item.primary_tag ? `${item.primary_tag} (${Number(item.primary_score).toFixed(3)})` : "—"} {item.needs_review ? item.review_reason || "needs review" : "ok"}
)}
) }