Spaces:
Sleeping
Sleeping
Commit ·
c32e3b8
1
Parent(s): f931767
Integrate Deep-Fake-Detector-v2 visual teacher and exam logging.
Browse filesPre-cache ViT teacher at Docker build, fuse teacher scores into EXM bands, append JSONL exam records for distillation.
Co-authored-by: Cursor <cursoragent@cursor.com>
- Dockerfile +3 -0
- backend/app/config.py +10 -0
- backend/app/jobs.py +1 -0
- backend/app/main.py +13 -1
- backend/app/pipeline/context.py +2 -0
- backend/app/pipeline/detection.py +92 -41
- backend/app/pipeline/exam_log.py +77 -0
- backend/app/pipeline/findings.py +19 -0
- backend/app/pipeline/open_audio.py +25 -0
- backend/app/pipeline/open_visual.py +59 -0
- backend/app/pipeline/orchestrator.py +42 -3
- backend/requirements.txt +2 -0
Dockerfile
CHANGED
|
@@ -11,6 +11,9 @@ WORKDIR /app
|
|
| 11 |
COPY backend/requirements.txt .
|
| 12 |
RUN pip install --no-cache-dir -r requirements.txt
|
| 13 |
|
|
|
|
|
|
|
|
|
|
| 14 |
COPY backend/app ./app
|
| 15 |
COPY models ./models
|
| 16 |
|
|
|
|
| 11 |
COPY backend/requirements.txt .
|
| 12 |
RUN pip install --no-cache-dir -r requirements.txt
|
| 13 |
|
| 14 |
+
ENV HF_HOME=/app/.cache/huggingface
|
| 15 |
+
RUN python -c "from transformers import ViTForImageClassification, ViTImageProcessor; mid='prithivMLmods/Deep-Fake-Detector-v2-Model'; ViTImageProcessor.from_pretrained(mid); ViTForImageClassification.from_pretrained(mid)"
|
| 16 |
+
|
| 17 |
COPY backend/app ./app
|
| 18 |
COPY models ./models
|
| 19 |
|
backend/app/config.py
CHANGED
|
@@ -11,6 +11,16 @@ class Settings(BaseSettings):
|
|
| 11 |
models_dir: str = "/app/models"
|
| 12 |
visual_model_file: str = "IDF_Solid_Final.pth"
|
| 13 |
audio_model_file: str = "IDF_Audio_Scientific_Solid.pth"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 14 |
max_frames_sample: int = 48
|
| 15 |
sample_fps: float = 2.0
|
| 16 |
dense_sample_fps: float = 5.0
|
|
|
|
| 11 |
models_dir: str = "/app/models"
|
| 12 |
visual_model_file: str = "IDF_Solid_Final.pth"
|
| 13 |
audio_model_file: str = "IDF_Audio_Scientific_Solid.pth"
|
| 14 |
+
enable_open_visual_teacher: bool = True
|
| 15 |
+
open_visual_model_id: str = "prithivMLmods/Deep-Fake-Detector-v2-Model"
|
| 16 |
+
enable_open_audio_teacher: bool = False
|
| 17 |
+
open_audio_model_id: str = ""
|
| 18 |
+
exam_log_enabled: bool = True
|
| 19 |
+
exam_log_dir: str = "/app/data/exam_logs"
|
| 20 |
+
teacher_pseudo_fake_threshold: float = 0.85
|
| 21 |
+
teacher_pseudo_real_threshold: float = 0.15
|
| 22 |
+
teacher_elevated_threshold: float = 0.65
|
| 23 |
+
teacher_suspicious_threshold: float = 0.55
|
| 24 |
max_frames_sample: int = 48
|
| 25 |
sample_fps: float = 2.0
|
| 26 |
dense_sample_fps: float = 5.0
|
backend/app/jobs.py
CHANGED
|
@@ -21,6 +21,7 @@ STAGES = [
|
|
| 21 |
"audio_extract",
|
| 22 |
"frame_extract",
|
| 23 |
"visual_model",
|
|
|
|
| 24 |
"audio_model",
|
| 25 |
"ela_analysis",
|
| 26 |
"temporal_analysis",
|
|
|
|
| 21 |
"audio_extract",
|
| 22 |
"frame_extract",
|
| 23 |
"visual_model",
|
| 24 |
+
"open_visual_teacher",
|
| 25 |
"audio_model",
|
| 26 |
"ela_analysis",
|
| 27 |
"temporal_analysis",
|
backend/app/main.py
CHANGED
|
@@ -12,6 +12,7 @@ from pydantic import BaseModel
|
|
| 12 |
|
| 13 |
from app.config import settings
|
| 14 |
from app.jobs import JobStatus, create_job, get_job, job_to_dict
|
|
|
|
| 15 |
from app.pipeline.orchestrator import run_pipeline
|
| 16 |
|
| 17 |
app = FastAPI(
|
|
@@ -54,9 +55,20 @@ def root():
|
|
| 54 |
@app.get("/health")
|
| 55 |
def health():
|
| 56 |
ready = models_ready()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 57 |
return {
|
| 58 |
-
"status": "healthy" if
|
| 59 |
"models": ready,
|
|
|
|
| 60 |
"limits": {
|
| 61 |
"max_upload_mb": settings.max_upload_mb,
|
| 62 |
"max_video_seconds": settings.max_video_seconds,
|
|
|
|
| 12 |
|
| 13 |
from app.config import settings
|
| 14 |
from app.jobs import JobStatus, create_job, get_job, job_to_dict
|
| 15 |
+
from app.pipeline import open_audio, open_visual
|
| 16 |
from app.pipeline.orchestrator import run_pipeline
|
| 17 |
|
| 18 |
app = FastAPI(
|
|
|
|
| 55 |
@app.get("/health")
|
| 56 |
def health():
|
| 57 |
ready = models_ready()
|
| 58 |
+
model_flags = {k: v for k, v in ready.items() if isinstance(v, bool)}
|
| 59 |
+
teachers = {
|
| 60 |
+
"open_visual_teacher": open_visual.teacher_available(),
|
| 61 |
+
"open_audio_teacher": open_audio.teacher_available(),
|
| 62 |
+
"open_visual_model_id": settings.open_visual_model_id if settings.enable_open_visual_teacher else None,
|
| 63 |
+
"exam_log_enabled": settings.exam_log_enabled,
|
| 64 |
+
}
|
| 65 |
+
degraded = not all(model_flags.values())
|
| 66 |
+
if settings.enable_open_visual_teacher and not teachers["open_visual_teacher"]:
|
| 67 |
+
degraded = True
|
| 68 |
return {
|
| 69 |
+
"status": "healthy" if not degraded else "degraded",
|
| 70 |
"models": ready,
|
| 71 |
+
"teachers": teachers,
|
| 72 |
"limits": {
|
| 73 |
"max_upload_mb": settings.max_upload_mb,
|
| 74 |
"max_video_seconds": settings.max_video_seconds,
|
backend/app/pipeline/context.py
CHANGED
|
@@ -30,7 +30,9 @@ class PipelineContext:
|
|
| 30 |
frames: list[FrameSample] = field(default_factory=list)
|
| 31 |
waveform: list[float] = field(default_factory=list)
|
| 32 |
visual_scores: list[float] = field(default_factory=list)
|
|
|
|
| 33 |
audio_score: float | None = None
|
|
|
|
| 34 |
visual_embedding: list[float] | None = None
|
| 35 |
audio_embedding: list[float] | None = None
|
| 36 |
visual_source_frame: str | None = None
|
|
|
|
| 30 |
frames: list[FrameSample] = field(default_factory=list)
|
| 31 |
waveform: list[float] = field(default_factory=list)
|
| 32 |
visual_scores: list[float] = field(default_factory=list)
|
| 33 |
+
teacher_visual_scores: list[float] = field(default_factory=list)
|
| 34 |
audio_score: float | None = None
|
| 35 |
+
teacher_audio_score: float | None = None
|
| 36 |
visual_embedding: list[float] | None = None
|
| 37 |
audio_embedding: list[float] | None = None
|
| 38 |
visual_source_frame: str | None = None
|
backend/app/pipeline/detection.py
CHANGED
|
@@ -22,12 +22,16 @@ def _clip_stats(scores: list[float]) -> dict[str, float]:
|
|
| 22 |
|
| 23 |
def compute_clip_assessment(ctx: PipelineContext) -> dict[str, Any]:
|
| 24 |
"""
|
| 25 |
-
Fuse
|
| 26 |
Does NOT render a legal verdict — indicates examination outcome band for expert review.
|
| 27 |
"""
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 31 |
|
| 32 |
temporal = ctx.temporal or {}
|
| 33 |
mean_drift = float(temporal.get("mean_drift") or 0.0)
|
|
@@ -57,6 +61,23 @@ def compute_clip_assessment(ctx: PipelineContext) -> dict[str, Any]:
|
|
| 57 |
corroborating = 0
|
| 58 |
reasons: list[str] = []
|
| 59 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 60 |
if mean_drift >= 0.35 or max_drift >= 0.55:
|
| 61 |
corroborating += 1
|
| 62 |
reasons.append(
|
|
@@ -84,65 +105,91 @@ def compute_clip_assessment(ctx: PipelineContext) -> dict[str, Any]:
|
|
| 84 |
corroborating += 2
|
| 85 |
reasons.append("Generative-tool metadata indicator matched")
|
| 86 |
|
| 87 |
-
|
| 88 |
-
|
|
|
|
| 89 |
|
| 90 |
-
if
|
| 91 |
band = "ELEVATED"
|
| 92 |
headline = (
|
| 93 |
-
f"
|
|
|
|
| 94 |
)
|
| 95 |
-
elif
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 96 |
band = "SUSPICIOUS"
|
| 97 |
headline = (
|
| 98 |
-
f"
|
| 99 |
"expert review recommended"
|
| 100 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 101 |
elif corroborating >= 3:
|
| 102 |
band = "SUSPICIOUS"
|
| 103 |
headline = (
|
| 104 |
-
f"Neural scores in baseline band (peak {peak:.2f}) but {corroborating} independent indicators "
|
| 105 |
-
"align — synthetic or heavily processed media cannot be ruled out"
|
| 106 |
)
|
| 107 |
elif corroborating >= 1 and (mean_drift >= 0.25 or edit_count >= 1):
|
| 108 |
band = "INCONCLUSIVE"
|
| 109 |
headline = (
|
| 110 |
-
f"Mixed signals — peak
|
| 111 |
-
"
|
|
|
|
| 112 |
)
|
| 113 |
-
elif peak <= settings.visual_borderline and corroborating == 0:
|
| 114 |
band = "BASELINE"
|
| 115 |
headline = (
|
| 116 |
-
f"No elevated
|
| 117 |
-
f"(peak {peak:.2f})"
|
| 118 |
)
|
| 119 |
else:
|
| 120 |
band = "INCONCLUSIVE"
|
| 121 |
-
headline =
|
| 122 |
|
|
|
|
| 123 |
fusion_score = min(
|
| 124 |
1.0,
|
| 125 |
-
(
|
| 126 |
-
+ (corroborating * 0.
|
| 127 |
-
+ (min(mean_drift, 1.0) * 0.
|
| 128 |
-
+ (
|
| 129 |
)
|
| 130 |
|
| 131 |
-
court_summary = _court_narrative(
|
|
|
|
|
|
|
| 132 |
|
| 133 |
return {
|
| 134 |
"band": band,
|
| 135 |
"headline": headline,
|
| 136 |
"fusion_score": round(fusion_score, 3),
|
| 137 |
-
"
|
|
|
|
|
|
|
| 138 |
"frames_analyzed": len(ctx.visual_scores),
|
| 139 |
"corroborating_count": corroborating,
|
| 140 |
"corroborating_reasons": reasons,
|
| 141 |
"court_summary": court_summary,
|
| 142 |
"neural_note": (
|
| 143 |
-
"
|
| 144 |
-
if
|
| 145 |
-
else
|
|
|
|
|
|
|
|
|
|
|
|
|
| 146 |
),
|
| 147 |
}
|
| 148 |
|
|
@@ -154,36 +201,40 @@ def _court_narrative(
|
|
| 154 |
n_frames: int,
|
| 155 |
reasons: list[str],
|
| 156 |
ctx: PipelineContext,
|
|
|
|
|
|
|
| 157 |
) -> str:
|
| 158 |
prov = ctx.provenance or {}
|
| 159 |
origin = prov.get("origin_class", "unknown")
|
| 160 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 161 |
intro = (
|
| 162 |
-
f"The examination analyzed {n_frames} full-frame sample(s) with dense regional scanning.
|
| 163 |
-
f"
|
|
|
|
| 164 |
)
|
| 165 |
|
| 166 |
if band == "ELEVATED":
|
| 167 |
body = (
|
| 168 |
-
"The
|
| 169 |
"This supports further expert review; it is not a standalone legal conclusion."
|
| 170 |
)
|
| 171 |
elif band == "SUSPICIOUS":
|
| 172 |
body = (
|
| 173 |
-
"
|
| 174 |
-
"
|
| 175 |
-
"consistent with synthetic generation, re-encoding, or non-camera-origin media. "
|
| 176 |
-
"This profile is commonly seen in AI-generated or heavily processed exhibits and requires expert review."
|
| 177 |
)
|
| 178 |
elif band == "INCONCLUSIVE":
|
| 179 |
-
body =
|
| 180 |
-
"Signals are mixed or partial. The exhibit cannot be confidently classified from automated analysis alone."
|
| 181 |
-
)
|
| 182 |
else:
|
| 183 |
body = (
|
| 184 |
-
"No strong automated indicators
|
| 185 |
-
"
|
| 186 |
-
"regions may differ."
|
| 187 |
)
|
| 188 |
|
| 189 |
prov_line = ""
|
|
@@ -196,6 +247,6 @@ def _court_narrative(
|
|
| 196 |
|
| 197 |
reason_block = ""
|
| 198 |
if reasons:
|
| 199 |
-
reason_block = " Key observations: " + "; ".join(reasons[:
|
| 200 |
|
| 201 |
return intro + body + prov_line + reason_block
|
|
|
|
| 22 |
|
| 23 |
def compute_clip_assessment(ctx: PipelineContext) -> dict[str, Any]:
|
| 24 |
"""
|
| 25 |
+
Fuse open teacher, student IDF, temporal, provenance, reference, and edit signals.
|
| 26 |
Does NOT render a legal verdict — indicates examination outcome band for expert review.
|
| 27 |
"""
|
| 28 |
+
student_stats = _clip_stats(ctx.visual_scores)
|
| 29 |
+
teacher_stats = _clip_stats(ctx.teacher_visual_scores) if ctx.teacher_visual_scores else None
|
| 30 |
+
|
| 31 |
+
peak = student_stats["peak"]
|
| 32 |
+
mean = student_stats["mean"]
|
| 33 |
+
teacher_peak = teacher_stats["peak"] if teacher_stats else None
|
| 34 |
+
teacher_mean = teacher_stats["mean"] if teacher_stats else None
|
| 35 |
|
| 36 |
temporal = ctx.temporal or {}
|
| 37 |
mean_drift = float(temporal.get("mean_drift") or 0.0)
|
|
|
|
| 61 |
corroborating = 0
|
| 62 |
reasons: list[str] = []
|
| 63 |
|
| 64 |
+
if teacher_stats:
|
| 65 |
+
reasons.insert(
|
| 66 |
+
0,
|
| 67 |
+
f"Open teacher peak {teacher_peak:.2f}, mean {teacher_mean:.2f} "
|
| 68 |
+
f"({settings.open_visual_model_id})",
|
| 69 |
+
)
|
| 70 |
+
if teacher_peak >= settings.teacher_elevated_threshold:
|
| 71 |
+
corroborating += 2
|
| 72 |
+
reasons.append(
|
| 73 |
+
f"Open deepfake detector elevated on sampled frames (peak {teacher_peak:.2f})"
|
| 74 |
+
)
|
| 75 |
+
elif teacher_peak >= settings.teacher_suspicious_threshold:
|
| 76 |
+
corroborating += 1
|
| 77 |
+
reasons.append(
|
| 78 |
+
f"Open deepfake detector borderline/suspicious (peak {teacher_peak:.2f})"
|
| 79 |
+
)
|
| 80 |
+
|
| 81 |
if mean_drift >= 0.35 or max_drift >= 0.55:
|
| 82 |
corroborating += 1
|
| 83 |
reasons.append(
|
|
|
|
| 105 |
corroborating += 2
|
| 106 |
reasons.append("Generative-tool metadata indicator matched")
|
| 107 |
|
| 108 |
+
student_elevated = peak >= settings.visual_threshold
|
| 109 |
+
teacher_elevated = teacher_peak is not None and teacher_peak >= settings.teacher_elevated_threshold
|
| 110 |
+
teacher_suspicious = teacher_peak is not None and teacher_peak >= settings.teacher_suspicious_threshold
|
| 111 |
|
| 112 |
+
if teacher_elevated:
|
| 113 |
band = "ELEVATED"
|
| 114 |
headline = (
|
| 115 |
+
f"Open deepfake detector elevated — teacher peak {teacher_peak:.2f} "
|
| 116 |
+
f"across {len(ctx.teacher_visual_scores)} frame(s)"
|
| 117 |
)
|
| 118 |
+
elif student_elevated:
|
| 119 |
+
band = "ELEVATED"
|
| 120 |
+
headline = (
|
| 121 |
+
f"Student visual engine elevated — peak score {peak:.2f} across {len(ctx.visual_scores)} frame(s)"
|
| 122 |
+
)
|
| 123 |
+
elif teacher_suspicious and corroborating >= 1:
|
| 124 |
band = "SUSPICIOUS"
|
| 125 |
headline = (
|
| 126 |
+
f"Open teacher suspicious (peak {teacher_peak:.2f}) with corroborating signals — "
|
| 127 |
"expert review recommended"
|
| 128 |
)
|
| 129 |
+
elif teacher_suspicious:
|
| 130 |
+
band = "SUSPICIOUS"
|
| 131 |
+
headline = (
|
| 132 |
+
f"Open deepfake detector suspicious — teacher peak {teacher_peak:.2f}; "
|
| 133 |
+
"student model remained near baseline"
|
| 134 |
+
)
|
| 135 |
+
elif peak >= settings.visual_borderline and corroborating >= 2:
|
| 136 |
+
band = "SUSPICIOUS"
|
| 137 |
+
headline = (
|
| 138 |
+
f"Borderline student response (peak {peak:.2f}) with {corroborating} corroborating signal(s)"
|
| 139 |
+
)
|
| 140 |
elif corroborating >= 3:
|
| 141 |
band = "SUSPICIOUS"
|
| 142 |
headline = (
|
| 143 |
+
f"Neural scores in baseline band (peak {peak:.2f}) but {corroborating} independent indicators align"
|
|
|
|
| 144 |
)
|
| 145 |
elif corroborating >= 1 and (mean_drift >= 0.25 or edit_count >= 1):
|
| 146 |
band = "INCONCLUSIVE"
|
| 147 |
headline = (
|
| 148 |
+
f"Mixed signals — peak student {peak:.2f}"
|
| 149 |
+
+ (f", teacher {teacher_peak:.2f}" if teacher_peak is not None else "")
|
| 150 |
+
+ f", {corroborating} secondary indicator(s)"
|
| 151 |
)
|
| 152 |
+
elif peak <= settings.visual_borderline and corroborating == 0 and not teacher_suspicious:
|
| 153 |
band = "BASELINE"
|
| 154 |
headline = (
|
| 155 |
+
f"No elevated indicators in {len(ctx.visual_scores)} frame sample(s) (peak {peak:.2f})"
|
|
|
|
| 156 |
)
|
| 157 |
else:
|
| 158 |
band = "INCONCLUSIVE"
|
| 159 |
+
headline = "Examination inconclusive — review teacher and secondary findings"
|
| 160 |
|
| 161 |
+
fusion_peak = teacher_peak if teacher_peak is not None else peak
|
| 162 |
fusion_score = min(
|
| 163 |
1.0,
|
| 164 |
+
(fusion_peak * 0.50)
|
| 165 |
+
+ (corroborating * 0.10)
|
| 166 |
+
+ (min(mean_drift, 1.0) * 0.20)
|
| 167 |
+
+ (student_stats["elevated_ratio"] * 0.10),
|
| 168 |
)
|
| 169 |
|
| 170 |
+
court_summary = _court_narrative(
|
| 171 |
+
band, peak, mean, len(ctx.visual_scores), reasons, ctx, teacher_peak, teacher_mean
|
| 172 |
+
)
|
| 173 |
|
| 174 |
return {
|
| 175 |
"band": band,
|
| 176 |
"headline": headline,
|
| 177 |
"fusion_score": round(fusion_score, 3),
|
| 178 |
+
"student_stats": student_stats,
|
| 179 |
+
"teacher_stats": teacher_stats,
|
| 180 |
+
"frame_stats": student_stats,
|
| 181 |
"frames_analyzed": len(ctx.visual_scores),
|
| 182 |
"corroborating_count": corroborating,
|
| 183 |
"corroborating_reasons": reasons,
|
| 184 |
"court_summary": court_summary,
|
| 185 |
"neural_note": (
|
| 186 |
+
f"Open teacher peak {teacher_peak:.2f}; student peak {peak:.2f}"
|
| 187 |
+
if teacher_peak is not None
|
| 188 |
+
else (
|
| 189 |
+
"Primary visual model did not strongly elevate"
|
| 190 |
+
if peak < settings.visual_threshold
|
| 191 |
+
else "Primary visual model elevated on at least one frame"
|
| 192 |
+
)
|
| 193 |
),
|
| 194 |
}
|
| 195 |
|
|
|
|
| 201 |
n_frames: int,
|
| 202 |
reasons: list[str],
|
| 203 |
ctx: PipelineContext,
|
| 204 |
+
teacher_peak: float | None,
|
| 205 |
+
teacher_mean: float | None,
|
| 206 |
) -> str:
|
| 207 |
prov = ctx.provenance or {}
|
| 208 |
origin = prov.get("origin_class", "unknown")
|
| 209 |
|
| 210 |
+
teacher_line = ""
|
| 211 |
+
if teacher_peak is not None:
|
| 212 |
+
teacher_line = (
|
| 213 |
+
f" Open deepfake teacher scores mean {teacher_mean:.2f}, peak {teacher_peak:.2f}."
|
| 214 |
+
)
|
| 215 |
+
|
| 216 |
intro = (
|
| 217 |
+
f"The examination analyzed {n_frames} full-frame sample(s) with dense regional scanning."
|
| 218 |
+
f" Student model scores mean {mean:.2f}, peak {peak:.2f}."
|
| 219 |
+
f"{teacher_line}"
|
| 220 |
)
|
| 221 |
|
| 222 |
if band == "ELEVATED":
|
| 223 |
body = (
|
| 224 |
+
"The detection engines flagged elevated synthetic or manipulation indicators. "
|
| 225 |
"This supports further expert review; it is not a standalone legal conclusion."
|
| 226 |
)
|
| 227 |
elif band == "SUSPICIOUS":
|
| 228 |
body = (
|
| 229 |
+
"One or more engines or independent indicators align with synthetic generation, "
|
| 230 |
+
"re-encoding, or non-camera-origin media. Expert review is recommended."
|
|
|
|
|
|
|
| 231 |
)
|
| 232 |
elif band == "INCONCLUSIVE":
|
| 233 |
+
body = "Signals are mixed or partial. The exhibit cannot be confidently classified from automated analysis alone."
|
|
|
|
|
|
|
| 234 |
else:
|
| 235 |
body = (
|
| 236 |
+
"No strong automated indicators were observed in sampled frames. "
|
| 237 |
+
"This does not prove authenticity."
|
|
|
|
| 238 |
)
|
| 239 |
|
| 240 |
prov_line = ""
|
|
|
|
| 247 |
|
| 248 |
reason_block = ""
|
| 249 |
if reasons:
|
| 250 |
+
reason_block = " Key observations: " + "; ".join(reasons[:5]) + "."
|
| 251 |
|
| 252 |
return intro + body + prov_line + reason_block
|
backend/app/pipeline/exam_log.py
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
from datetime import datetime, timezone
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
from typing import Any
|
| 7 |
+
|
| 8 |
+
from app.config import settings
|
| 9 |
+
from app.pipeline.context import PipelineContext
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def _log_path() -> Path:
|
| 13 |
+
root = Path(settings.exam_log_dir)
|
| 14 |
+
root.mkdir(parents=True, exist_ok=True)
|
| 15 |
+
day = datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
| 16 |
+
return root / f"exams_{day}.jsonl"
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def append_exam_record(
|
| 20 |
+
*,
|
| 21 |
+
ctx: PipelineContext,
|
| 22 |
+
warnings: list[str],
|
| 23 |
+
clip_assessment: dict[str, Any],
|
| 24 |
+
) -> dict[str, Any] | None:
|
| 25 |
+
"""
|
| 26 |
+
Append one JSONL record for distillation / benchmarking.
|
| 27 |
+
Stores scores only — no raw media bytes.
|
| 28 |
+
"""
|
| 29 |
+
if not settings.exam_log_enabled:
|
| 30 |
+
return None
|
| 31 |
+
|
| 32 |
+
frames: list[dict[str, Any]] = []
|
| 33 |
+
for i, frame in enumerate(ctx.frames):
|
| 34 |
+
frame_id = f"F{i + 1:03d}"
|
| 35 |
+
student = ctx.visual_scores[i] if i < len(ctx.visual_scores) else None
|
| 36 |
+
teacher = ctx.teacher_visual_scores[i] if i < len(ctx.teacher_visual_scores) else None
|
| 37 |
+
frames.append(
|
| 38 |
+
{
|
| 39 |
+
"id": frame_id,
|
| 40 |
+
"time_sec": round(float(frame.timestamp_sec), 3),
|
| 41 |
+
"student_score": round(float(student), 4) if student is not None else None,
|
| 42 |
+
"teacher_score": round(float(teacher), 4) if teacher is not None else None,
|
| 43 |
+
}
|
| 44 |
+
)
|
| 45 |
+
|
| 46 |
+
record: dict[str, Any] = {
|
| 47 |
+
"logged_at": datetime.now(timezone.utc).isoformat(),
|
| 48 |
+
"job_id": ctx.job_id,
|
| 49 |
+
"sha256": ctx.sha256,
|
| 50 |
+
"filename": ctx.filename,
|
| 51 |
+
"media_kind": ctx.media_kind,
|
| 52 |
+
"size_bytes": ctx.size_bytes,
|
| 53 |
+
"duration_sec": ctx.duration_sec,
|
| 54 |
+
"frames": frames,
|
| 55 |
+
"audio": {
|
| 56 |
+
"student_score": ctx.audio_score,
|
| 57 |
+
"teacher_score": ctx.teacher_audio_score,
|
| 58 |
+
},
|
| 59 |
+
"clip_assessment": {
|
| 60 |
+
"band": clip_assessment.get("band"),
|
| 61 |
+
"fusion_score": clip_assessment.get("fusion_score"),
|
| 62 |
+
"teacher_peak": clip_assessment.get("teacher_stats", {}).get("peak"),
|
| 63 |
+
"student_peak": clip_assessment.get("student_stats", {}).get("peak"),
|
| 64 |
+
},
|
| 65 |
+
"provenance": ctx.provenance,
|
| 66 |
+
"warnings": warnings,
|
| 67 |
+
"teacher_models": {
|
| 68 |
+
"visual": settings.open_visual_model_id if settings.enable_open_visual_teacher else None,
|
| 69 |
+
"audio": settings.open_audio_model_id if settings.enable_open_audio_teacher else None,
|
| 70 |
+
},
|
| 71 |
+
}
|
| 72 |
+
|
| 73 |
+
path = _log_path()
|
| 74 |
+
with path.open("a", encoding="utf-8") as fh:
|
| 75 |
+
fh.write(json.dumps(record, separators=(",", ":")) + "\n")
|
| 76 |
+
|
| 77 |
+
return {"path": str(path), "record_count_hint": "append-only jsonl"}
|
backend/app/pipeline/findings.py
CHANGED
|
@@ -2,6 +2,7 @@ from __future__ import annotations
|
|
| 2 |
|
| 3 |
from datetime import datetime, timezone
|
| 4 |
|
|
|
|
| 5 |
from app.pipeline.context import PipelineContext
|
| 6 |
from app.pipeline.observations import describe_audio_score, describe_dna_composite, describe_ela_score
|
| 7 |
|
|
@@ -54,6 +55,24 @@ def build_findings(ctx: PipelineContext, ela_scores: list[float], warnings: list
|
|
| 54 |
)
|
| 55 |
|
| 56 |
clip_mean = sum(ctx.visual_scores) / len(ctx.visual_scores) if ctx.visual_scores else 0.0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 57 |
if ctx.visual_scores:
|
| 58 |
indexed = sorted(enumerate(ctx.visual_scores), key=lambda x: x[1], reverse=True)
|
| 59 |
for rank, (idx, score) in enumerate(indexed[:6], start=1):
|
|
|
|
| 2 |
|
| 3 |
from datetime import datetime, timezone
|
| 4 |
|
| 5 |
+
from app.config import settings
|
| 6 |
from app.pipeline.context import PipelineContext
|
| 7 |
from app.pipeline.observations import describe_audio_score, describe_dna_composite, describe_ela_score
|
| 8 |
|
|
|
|
| 55 |
)
|
| 56 |
|
| 57 |
clip_mean = sum(ctx.visual_scores) / len(ctx.visual_scores) if ctx.visual_scores else 0.0
|
| 58 |
+
if ctx.teacher_visual_scores:
|
| 59 |
+
tpeak = max(ctx.teacher_visual_scores)
|
| 60 |
+
tmean = sum(ctx.teacher_visual_scores) / len(ctx.teacher_visual_scores)
|
| 61 |
+
indexed_t = sorted(enumerate(ctx.teacher_visual_scores), key=lambda x: x[1], reverse=True)
|
| 62 |
+
t_obs = [
|
| 63 |
+
f"Open teacher peak {tpeak:.2f}, mean {tmean:.2f}",
|
| 64 |
+
f"Model: {settings.open_visual_model_id}",
|
| 65 |
+
f"Sampled {len(ctx.teacher_visual_scores)} frame(s)",
|
| 66 |
+
]
|
| 67 |
+
for rank, (idx, tscore) in enumerate(indexed_t[:3], start=1):
|
| 68 |
+
frame_id = f"F{idx + 1:03d}"
|
| 69 |
+
student = ctx.visual_scores[idx] if idx < len(ctx.visual_scores) else None
|
| 70 |
+
pair = f"Frame {frame_id}: teacher {tscore:.2f}"
|
| 71 |
+
if student is not None:
|
| 72 |
+
pair += f", student {student:.2f}"
|
| 73 |
+
t_obs.append(pair)
|
| 74 |
+
findings.append(_finding("TCH-001", t_obs))
|
| 75 |
+
|
| 76 |
if ctx.visual_scores:
|
| 77 |
indexed = sorted(enumerate(ctx.visual_scores), key=lambda x: x[1], reverse=True)
|
| 78 |
for rank, (idx, score) in enumerate(indexed[:6], start=1):
|
backend/app/pipeline/open_audio.py
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from functools import lru_cache
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
|
| 6 |
+
from app.config import settings
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
@lru_cache(maxsize=1)
|
| 10 |
+
def _teacher_ready() -> bool:
|
| 11 |
+
"""Phase 2: wire an open anti-spoof / synthetic-speech teacher (e.g. ASVspoof/AASIST)."""
|
| 12 |
+
if not settings.enable_open_audio_teacher or not settings.open_audio_model_id:
|
| 13 |
+
return False
|
| 14 |
+
return False
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def teacher_available() -> bool:
|
| 18 |
+
return _teacher_ready()
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def score_audio(path: Path) -> float | None:
|
| 22 |
+
"""Return P(synthetic/spoof) from open audio teacher when enabled."""
|
| 23 |
+
if not teacher_available():
|
| 24 |
+
return None
|
| 25 |
+
return None
|
backend/app/pipeline/open_visual.py
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from functools import lru_cache
|
| 4 |
+
from typing import Any
|
| 5 |
+
|
| 6 |
+
import cv2
|
| 7 |
+
import numpy as np
|
| 8 |
+
import torch
|
| 9 |
+
from PIL import Image
|
| 10 |
+
|
| 11 |
+
from app.config import settings
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def _fake_class_index(id2label: dict[int, str]) -> int:
|
| 15 |
+
for idx, label in id2label.items():
|
| 16 |
+
lower = label.lower()
|
| 17 |
+
if any(k in lower for k in ("fake", "deepfake", "ai", "synthetic", "generated")):
|
| 18 |
+
return int(idx)
|
| 19 |
+
return max(id2label.keys())
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
@lru_cache(maxsize=1)
|
| 23 |
+
def _load_teacher() -> tuple[Any, Any, int] | None:
|
| 24 |
+
if not settings.enable_open_visual_teacher:
|
| 25 |
+
return None
|
| 26 |
+
try:
|
| 27 |
+
from transformers import ViTForImageClassification, ViTImageProcessor
|
| 28 |
+
|
| 29 |
+
processor = ViTImageProcessor.from_pretrained(settings.open_visual_model_id)
|
| 30 |
+
model = ViTForImageClassification.from_pretrained(settings.open_visual_model_id)
|
| 31 |
+
model.eval()
|
| 32 |
+
fake_idx = _fake_class_index(model.config.id2label)
|
| 33 |
+
return model, processor, fake_idx
|
| 34 |
+
except Exception:
|
| 35 |
+
return None
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def teacher_available() -> bool:
|
| 39 |
+
return _load_teacher() is not None
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def score_bgr_frame(bgr: np.ndarray) -> float | None:
|
| 43 |
+
"""Return P(fake) from the open ViT teacher, or None if unavailable."""
|
| 44 |
+
loaded = _load_teacher()
|
| 45 |
+
if loaded is None:
|
| 46 |
+
return None
|
| 47 |
+
model, processor, fake_idx = loaded
|
| 48 |
+
|
| 49 |
+
rgb = cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB)
|
| 50 |
+
pil = Image.fromarray(rgb)
|
| 51 |
+
inputs = processor(images=pil, return_tensors="pt")
|
| 52 |
+
with torch.no_grad():
|
| 53 |
+
logits = model(**inputs).logits
|
| 54 |
+
probs = torch.softmax(logits, dim=-1)
|
| 55 |
+
return float(probs[0, fake_idx].item())
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def score_bgr_batch(frames_bgr: list[np.ndarray]) -> list[float | None]:
|
| 59 |
+
return [score_bgr_frame(bgr) for bgr in frames_bgr]
|
backend/app/pipeline/orchestrator.py
CHANGED
|
@@ -38,6 +38,8 @@ from app.pipeline.media import (
|
|
| 38 |
probe_metadata,
|
| 39 |
)
|
| 40 |
from app.pipeline import models as model_runtime
|
|
|
|
|
|
|
| 41 |
|
| 42 |
|
| 43 |
ProgressCallback = Callable[[str, int], None]
|
|
@@ -76,6 +78,7 @@ def _build_flagged_frames(
|
|
| 76 |
frames_out: list[dict[str, Any]] = []
|
| 77 |
for i, frame in enumerate(ctx.frames):
|
| 78 |
score = ctx.visual_scores[i] if i < len(ctx.visual_scores) else 0.0
|
|
|
|
| 79 |
ela = ela_scores[i] if i < len(ela_scores) else None
|
| 80 |
bgr = _frame_bgr(frame)
|
| 81 |
thumb = frame_thumbnail_data_url(bgr) if bgr is not None else None
|
|
@@ -119,6 +122,7 @@ def _build_flagged_frames(
|
|
| 119 |
{
|
| 120 |
"id": frame_id,
|
| 121 |
"peak": round(float(score), 4),
|
|
|
|
| 122 |
"ela": round(float(ela), 4) if ela is not None else None,
|
| 123 |
"time": time_label,
|
| 124 |
"thumbnail_url": thumb or None,
|
|
@@ -130,7 +134,12 @@ def _build_flagged_frames(
|
|
| 130 |
"observations": package["observations"],
|
| 131 |
}
|
| 132 |
)
|
| 133 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 134 |
return frames_out[: settings.max_flagged_frames]
|
| 135 |
|
| 136 |
|
|
@@ -168,9 +177,13 @@ def _build_audio_segments(ctx: PipelineContext) -> list[dict[str, Any]]:
|
|
| 168 |
|
| 169 |
def _integrity_index(ctx: PipelineContext) -> float:
|
| 170 |
parts: list[float] = []
|
| 171 |
-
if ctx.
|
|
|
|
|
|
|
| 172 |
parts.append(max(ctx.visual_scores))
|
| 173 |
-
if ctx.
|
|
|
|
|
|
|
| 174 |
parts.append(ctx.audio_score)
|
| 175 |
if not parts:
|
| 176 |
return 0.0
|
|
@@ -257,6 +270,24 @@ def run_pipeline(
|
|
| 257 |
ctx.visual_embedding = peak_emb
|
| 258 |
ctx.visual_source_frame = peak_id
|
| 259 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 260 |
_set_stage(on_progress, "audio_model")
|
| 261 |
if ctx.audio_path and ctx.audio_path.exists():
|
| 262 |
try:
|
|
@@ -311,6 +342,12 @@ def run_pipeline(
|
|
| 311 |
ctx.findings = build_findings(ctx, ela_scores, warnings)
|
| 312 |
ctx.executive_summary = build_executive_summary(ctx)
|
| 313 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 314 |
_set_stage(on_progress, "report")
|
| 315 |
result = {
|
| 316 |
"sha256": ctx.sha256,
|
|
@@ -334,6 +371,8 @@ def run_pipeline(
|
|
| 334 |
"expert_annotations": ctx.expert_annotations,
|
| 335 |
"clip_assessment": ctx.clip_assessment,
|
| 336 |
}
|
|
|
|
|
|
|
| 337 |
result["report_package"] = build_report_package(result)
|
| 338 |
if warnings:
|
| 339 |
ctx.message = "; ".join(warnings)
|
|
|
|
| 38 |
probe_metadata,
|
| 39 |
)
|
| 40 |
from app.pipeline import models as model_runtime
|
| 41 |
+
from app.pipeline import open_visual, open_audio
|
| 42 |
+
from app.pipeline.exam_log import append_exam_record
|
| 43 |
|
| 44 |
|
| 45 |
ProgressCallback = Callable[[str, int], None]
|
|
|
|
| 78 |
frames_out: list[dict[str, Any]] = []
|
| 79 |
for i, frame in enumerate(ctx.frames):
|
| 80 |
score = ctx.visual_scores[i] if i < len(ctx.visual_scores) else 0.0
|
| 81 |
+
teacher_score = ctx.teacher_visual_scores[i] if i < len(ctx.teacher_visual_scores) else None
|
| 82 |
ela = ela_scores[i] if i < len(ela_scores) else None
|
| 83 |
bgr = _frame_bgr(frame)
|
| 84 |
thumb = frame_thumbnail_data_url(bgr) if bgr is not None else None
|
|
|
|
| 122 |
{
|
| 123 |
"id": frame_id,
|
| 124 |
"peak": round(float(score), 4),
|
| 125 |
+
"teacher_score": round(float(teacher_score), 4) if teacher_score is not None else None,
|
| 126 |
"ela": round(float(ela), 4) if ela is not None else None,
|
| 127 |
"time": time_label,
|
| 128 |
"thumbnail_url": thumb or None,
|
|
|
|
| 134 |
"observations": package["observations"],
|
| 135 |
}
|
| 136 |
)
|
| 137 |
+
def _rank_score(i: int) -> float:
|
| 138 |
+
s = ctx.visual_scores[i] if i < len(ctx.visual_scores) else 0.0
|
| 139 |
+
t = ctx.teacher_visual_scores[i] if i < len(ctx.teacher_visual_scores) else 0.0
|
| 140 |
+
return max(s, t)
|
| 141 |
+
|
| 142 |
+
frames_out.sort(key=lambda f: _rank_score(int(f["id"][1:]) - 1), reverse=True)
|
| 143 |
return frames_out[: settings.max_flagged_frames]
|
| 144 |
|
| 145 |
|
|
|
|
| 177 |
|
| 178 |
def _integrity_index(ctx: PipelineContext) -> float:
|
| 179 |
parts: list[float] = []
|
| 180 |
+
if ctx.teacher_visual_scores:
|
| 181 |
+
parts.append(max(ctx.teacher_visual_scores))
|
| 182 |
+
elif ctx.visual_scores:
|
| 183 |
parts.append(max(ctx.visual_scores))
|
| 184 |
+
if ctx.teacher_audio_score is not None:
|
| 185 |
+
parts.append(ctx.teacher_audio_score)
|
| 186 |
+
elif ctx.audio_score is not None:
|
| 187 |
parts.append(ctx.audio_score)
|
| 188 |
if not parts:
|
| 189 |
return 0.0
|
|
|
|
| 270 |
ctx.visual_embedding = peak_emb
|
| 271 |
ctx.visual_source_frame = peak_id
|
| 272 |
|
| 273 |
+
_set_stage(on_progress, "open_visual_teacher")
|
| 274 |
+
if ctx.frames and settings.enable_open_visual_teacher:
|
| 275 |
+
try:
|
| 276 |
+
if open_visual.teacher_available():
|
| 277 |
+
for frame in ctx.frames:
|
| 278 |
+
bgr = _frame_bgr(frame)
|
| 279 |
+
if bgr is None:
|
| 280 |
+
continue
|
| 281 |
+
tscore = open_visual.score_bgr_frame(bgr)
|
| 282 |
+
if tscore is not None:
|
| 283 |
+
ctx.teacher_visual_scores.append(tscore)
|
| 284 |
+
else:
|
| 285 |
+
warnings.append(
|
| 286 |
+
"Open visual teacher unavailable — check transformers install and HF model cache"
|
| 287 |
+
)
|
| 288 |
+
except Exception as exc:
|
| 289 |
+
warnings.append(f"Open visual teacher: {exc}")
|
| 290 |
+
|
| 291 |
_set_stage(on_progress, "audio_model")
|
| 292 |
if ctx.audio_path and ctx.audio_path.exists():
|
| 293 |
try:
|
|
|
|
| 342 |
ctx.findings = build_findings(ctx, ela_scores, warnings)
|
| 343 |
ctx.executive_summary = build_executive_summary(ctx)
|
| 344 |
|
| 345 |
+
exam_log_meta = append_exam_record(
|
| 346 |
+
ctx=ctx,
|
| 347 |
+
warnings=warnings,
|
| 348 |
+
clip_assessment=ctx.clip_assessment,
|
| 349 |
+
)
|
| 350 |
+
|
| 351 |
_set_stage(on_progress, "report")
|
| 352 |
result = {
|
| 353 |
"sha256": ctx.sha256,
|
|
|
|
| 371 |
"expert_annotations": ctx.expert_annotations,
|
| 372 |
"clip_assessment": ctx.clip_assessment,
|
| 373 |
}
|
| 374 |
+
if exam_log_meta:
|
| 375 |
+
result["exam_log"] = exam_log_meta
|
| 376 |
result["report_package"] = build_report_package(result)
|
| 377 |
if warnings:
|
| 378 |
ctx.message = "; ".join(warnings)
|
backend/requirements.txt
CHANGED
|
@@ -12,3 +12,5 @@ torch==2.5.1
|
|
| 12 |
torchvision==0.20.1
|
| 13 |
reportlab==4.2.5
|
| 14 |
timm==1.0.12
|
|
|
|
|
|
|
|
|
| 12 |
torchvision==0.20.1
|
| 13 |
reportlab==4.2.5
|
| 14 |
timm==1.0.12
|
| 15 |
+
transformers==4.47.1
|
| 16 |
+
huggingface-hub==0.27.0
|