from io import BytesIO import pytest from PIL import Image, PngImagePlugin from app.analysis import ( _huggingface_detector_for_model, _map_classifier_outputs_to_ai_probability, _model_inference_views, analyze_image_bytes, aggregate_verdict, DetectorSignal, extract_metadata, model_ensemble_signal, ) def _png_bytes(with_marker: bool = False) -> bytes: image = Image.new("RGB", (96, 96), (120, 80, 200)) output = BytesIO() pnginfo = PngImagePlugin.PngInfo() if with_marker: pnginfo.add_text("Software", "Stable Diffusion") pnginfo.add_text("prompt", "synthetic portrait generated by a diffusion model") image.save(output, format="PNG", pnginfo=pnginfo) return output.getvalue() def test_analysis_reports_generative_metadata_marker() -> None: result = analyze_image_bytes(_png_bytes(with_marker=True)) assert result["verdict"]["label"] == "likely_ai_generated" assert result["technical_appendix"]["metadata"]["generative_markers"] assert result["technical_appendix"]["hashes"]["sha256"] def test_analysis_is_cautious_without_strong_signals() -> None: result = analyze_image_bytes(_png_bytes(with_marker=False)) assert result["verdict"]["label"] in {"inconclusive", "likely_manipulated_or_deepfake"} assert "Image authenticity cannot be proven from pixels alone." in result["summary"]["limitations"] def test_analysis_includes_explainable_analytical_layers() -> None: result = analyze_image_bytes(_png_bytes(with_marker=False)) layers = result["analytical_layers"] assert len(layers) >= 8 assert {layer["id"] for layer in layers} >= { "source_provenance", "visual_model_consensus", "model_transform_robustness", "luminance", "chroma", "edge_geometry", "noise_residual", "compression_ela", "frequency", "tile_regions", } for layer in layers: assert layer["question"] assert layer["method"] assert layer["conclusion"] assert 0 <= layer["ai_signal"] <= 1 assert 0 <= layer["manipulation_signal"] <= 1 assert layer["evidence"] assert layer["decision_role"] in { "primary_evidence", "guard_or_supporting_evidence", "review_context_only", } assert 0 <= layer["influence"] <= 1 assert layer["counterfactual"] assert result["explainability"]["layer_ledger"]["layers"] assert result["explainability"]["decision_attribution"] assert result["explainability"]["explanation_contract"]["score_semantics"] assert result["explainability"]["decision_support"]["primary_drivers"] assert result["explainability"]["regional_evidence_map"]["grid"] == {"rows": 4, "cols": 4} tile_layer = next(layer for layer in layers if layer["id"] == "tile_regions") tile_grid = tile_layer["metrics"]["tile_grid"] assert len(tile_grid) == 16 assert {tile["row"] for tile in tile_grid} == {1, 2, 3, 4} assert {tile["col"] for tile in tile_grid} == {1, 2, 3, 4} assert {tile["severity_band"] for tile in tile_grid} <= {"low", "medium", "high"} def test_xmp_markers_are_summarized_without_raw_xml() -> None: image = Image.new("RGB", (96, 96), (120, 80, 200)) output = BytesIO() image.save(output, format="JPEG") image_bytes = output.getvalue() + ( b'' b'' b"" ) metadata = extract_metadata(Image.open(BytesIO(image_bytes)), image_bytes) software_summary = " ".join(metadata["software_values"]) assert metadata["editing_markers"] == ["photoshop"] assert "XMP metadata mentions: photoshop" in software_summary assert " None: assert _map_classifier_outputs_to_ai_probability( [{"label": "ai", "score": 0.91}, {"label": "hum", "score": 0.09}] ) == pytest.approx(0.91) assert _map_classifier_outputs_to_ai_probability( [{"label": "hum", "score": 0.82}, {"label": "ai", "score": 0.18}] ) == pytest.approx(0.18) def test_model_label_mapping_sums_multiclass_ai_labels() -> None: assert _map_classifier_outputs_to_ai_probability( [ {"label": "Real", "score": 0.23}, {"label": "Deepfake", "score": 0.46}, {"label": "Artificial", "score": 0.31}, ] ) == pytest.approx(0.77) def test_model_label_mapping_handles_human_artificial_labels() -> None: assert _map_classifier_outputs_to_ai_probability( [{"label": "human", "score": 0.99}, {"label": "artificial", "score": 0.01}] ) == pytest.approx(0.01) def test_model_label_mapping_supports_explicit_numeric_labels() -> None: profile = {"ai_labels": ["label_1"], "real_labels": ["label_0"]} assert _map_classifier_outputs_to_ai_probability( [{"label": "LABEL_1", "score": 0.83}, {"label": "LABEL_0", "score": 0.17}], profile, ) == pytest.approx(0.83) def test_broad_primary_uses_five_robustness_views() -> None: image = Image.new("RGB", (320, 240), (80, 110, 140)) views = _model_inference_views(image, {"multi_view": True}) assert [name for name, _ in views] == [ "original", "center_crop_92pct", "jpeg_quality_85", "horizontal_flip", "social_resize_75pct", ] assert views[-1][1].size == (240, 180) def test_model_ensemble_signal_summarizes_votes() -> None: signal = model_ensemble_signal( [ DetectorSignal("hf:a", "ok", "model_likely_ai_generated", 0.96, None, "medium", [], 0.55), DetectorSignal("hf:b", "ok", "model_likely_ai_generated", 0.92, None, "medium", [], 0.55), DetectorSignal("hf:c", "ok", "model_likely_ai_generated", 0.90, None, "medium", [], 0.55), ] ) assert signal.label == "ensemble_likely_ai_generated" assert signal.confidence == "high" assert signal.ai_probability == pytest.approx(0.869) def test_model_ensemble_rejects_soft_leaning_ai_consensus() -> None: signal = model_ensemble_signal( [ DetectorSignal("hf:a", "ok", "model_likely_ai_generated", 0.99, None, "medium", [], 0.55), DetectorSignal("hf:b", "ok", "model_inconclusive", 0.69, None, "medium", [], 0.55), DetectorSignal("hf:c", "ok", "model_inconclusive", 0.62, None, "medium", [], 0.55), ] ) assert signal.label == "ensemble_inconclusive" assert signal.confidence == "low" assert signal.ai_probability == pytest.approx(0.6607) def test_model_ensemble_accepts_primary_anchored_alignment() -> None: signal = model_ensemble_signal( [ DetectorSignal( "hf:buildborderless/CommunityForensics-DeepfakeDet-ViT", "ok", "model_inconclusive", 0.72, None, "low", [], 1.0, ), DetectorSignal( "hf:Ateeqq/ai-vs-human-image-detector", "ok", "model_likely_ai_generated", 0.999, None, "medium", [], 0.3, ), DetectorSignal( "hf:jacoballessio/ai-image-detect-distilled", "ok", "model_inconclusive", 0.62, None, "low", [], 0.45, ), ] ) assert signal.label == "ensemble_likely_ai_generated" assert signal.confidence == "medium" assert signal.ai_probability == pytest.approx(0.74) def test_model_ensemble_is_inconclusive_when_detectors_disagree_strongly() -> None: signal = model_ensemble_signal( [ DetectorSignal("hf:a", "ok", "model_likely_ai_generated", 0.99, None, "medium", [], 0.55), DetectorSignal("hf:b", "ok", "model_likely_human_or_real", 0.22, None, "medium", [], 0.55), ] ) assert signal.label == "ensemble_inconclusive" assert signal.confidence == "low" def test_aggregate_verdict_caps_uncorroborated_model_false_positive() -> None: detectors = [ DetectorSignal("metadata_provenance", "ok", "inconclusive", 0.5, 0.25, "low", ["No strong metadata signal."], 0.22), DetectorSignal("compression_noise_forensics", "ok", "no_strong_forensic_signal", 0.38, 0.38, "low", ["Weak forensic signal."], 0.28), DetectorSignal("hf:a", "ok", "model_likely_ai_generated", 0.99, None, "medium", ["a"], 0.55), DetectorSignal("hf:b", "ok", "model_inconclusive", 0.61, None, "medium", ["b"], 0.55), DetectorSignal("open_source_model_ensemble", "ok", "ensemble_inconclusive", 0.8, None, "low", ["disagreement"], 0.9), ] verdict = aggregate_verdict( detectors, metadata={"generative_markers": [], "has_exif": False}, c2pa={"claim": None}, forensics={"artificiality_score": 0.08, "manipulation_score": 0.38}, ) assert verdict["label"] == "inconclusive" assert verdict["confidence"] == "low" assert verdict["ai_probability"] <= 0.64 assert any("capped" in item for item in verdict["rationale"]) def test_aggregate_verdict_requires_independent_support_for_two_model_ai_claim() -> None: detectors = [ DetectorSignal("metadata_provenance", "ok", "inconclusive", 0.5, 0.25, "low", ["No strong metadata signal."], 0.22), DetectorSignal("compression_noise_forensics", "ok", "no_strong_forensic_signal", 0.34, 0.29, "low", ["Weak forensic signal."], 0.28), DetectorSignal("hf:a", "ok", "model_likely_ai_generated", 0.99, None, "medium", ["a"], 0.55), DetectorSignal("hf:b", "ok", "model_likely_ai_generated", 0.88, None, "medium", ["b"], 0.55), DetectorSignal("open_source_model_ensemble", "ok", "ensemble_likely_ai_generated", 0.935, None, "high", ["consensus"], 0.9), ] verdict = aggregate_verdict( detectors, metadata={"generative_markers": [], "has_exif": False}, c2pa={"claim": None}, forensics={"artificiality_score": 0.06, "manipulation_score": 0.29}, ) assert verdict["label"] == "inconclusive" assert verdict["confidence"] == "low" assert verdict["ai_probability"] <= 0.55 assert any("independent metadata" in item for item in verdict["rationale"]) def test_aggregate_verdict_allows_overwhelming_model_consensus_without_real_vote() -> None: detectors = [ DetectorSignal("metadata_provenance", "ok", "inconclusive", 0.5, 0.25, "low", ["No strong metadata signal."], 0.22), DetectorSignal("compression_noise_forensics", "ok", "no_strong_forensic_signal", 0.36, 0.29, "low", ["Weak forensic signal."], 0.28), DetectorSignal("hf:a", "ok", "model_likely_ai_generated", 0.97, None, "medium", ["a"], 0.55), DetectorSignal("hf:b", "ok", "model_likely_ai_generated", 0.94, None, "medium", ["b"], 0.55), DetectorSignal("hf:c", "ok", "model_likely_ai_generated", 0.92, None, "medium", ["c"], 0.55), DetectorSignal("open_source_model_ensemble", "ok", "ensemble_likely_ai_generated", 0.9433, None, "high", ["consensus"], 0.9), ] verdict = aggregate_verdict( detectors, metadata={"generative_markers": [], "has_exif": False, "format": "PNG"}, c2pa={"claim": None}, forensics={"artificiality_score": 0.06, "manipulation_score": 0.29}, ) assert verdict["label"] == "likely_ai_generated" assert verdict["confidence"] == "medium" assert verdict["ai_probability"] == pytest.approx(0.72) def test_aggregate_verdict_allows_primary_anchored_consensus() -> None: detectors = [ DetectorSignal("metadata_provenance", "ok", "inconclusive", 0.5, 0.25, "low", ["none"], 0.22), DetectorSignal("compression_noise_forensics", "ok", "no_strong_forensic_signal", 0.36, 0.25, "low", ["weak"], 0.28), DetectorSignal("hf:buildborderless/CommunityForensics-DeepfakeDet-ViT", "ok", "model_inconclusive", 0.72, None, "low", ["primary"], 1.0), DetectorSignal("hf:Ateeqq/ai-vs-human-image-detector", "ok", "model_likely_ai_generated", 0.999, None, "medium", ["counter"], 0.3), DetectorSignal("hf:jacoballessio/ai-image-detect-distilled", "ok", "model_inconclusive", 0.62, None, "low", ["counter"], 0.45), DetectorSignal("open_source_model_ensemble", "ok", "ensemble_likely_ai_generated", 0.74, None, "medium", ["aligned"], 0.9), ] verdict = aggregate_verdict( detectors, metadata={"generative_markers": [], "has_exif": False, "format": "PNG"}, c2pa={"claim": None}, forensics={"artificiality_score": 0.12, "manipulation_score": 0.25, "quality": {"risk_score": 0.1}}, ) assert verdict["label"] == "likely_ai_generated" assert verdict["ai_probability"] == pytest.approx(0.72) assert any("primary-anchored" in item for item in verdict["rationale"]) def test_aggregate_verdict_caps_camera_like_real_photo_even_with_model_consensus() -> None: detectors = [ DetectorSignal("metadata_provenance", "ok", "camera_metadata_present", 0.35, 0.25, "low", ["EXIF"], 0.22), DetectorSignal("compression_noise_forensics", "ok", "no_strong_forensic_signal", 0.36, 0.29, "low", ["Weak forensic signal."], 0.28), DetectorSignal("hf:a", "ok", "model_likely_ai_generated", 0.97, None, "medium", ["a"], 0.55), DetectorSignal("hf:b", "ok", "model_likely_ai_generated", 0.94, None, "medium", ["b"], 0.55), DetectorSignal("hf:c", "ok", "model_likely_ai_generated", 0.92, None, "medium", ["c"], 0.55), DetectorSignal("open_source_model_ensemble", "ok", "ensemble_likely_ai_generated", 0.9433, None, "high", ["consensus"], 0.9), ] verdict = aggregate_verdict( detectors, metadata={"generative_markers": [], "has_exif": True, "format": "JPEG"}, c2pa={"claim": None}, forensics={ "artificiality_score": 0.06, "manipulation_score": 0.29, "noise": {"tile_variance_mean": 240.0, "low_noise_hint": 0.0}, "entropy": 6.5, }, ) assert verdict["label"] != "likely_ai_generated" assert verdict["ai_probability"] <= 0.48 assert any("real-photo false-positive guard" in item for item in verdict["rationale"]) def test_aggregate_verdict_uses_portrait_specialist_to_prevent_ai_false_positive() -> None: detectors = [ DetectorSignal("metadata_provenance", "ok", "inconclusive", 0.5, 0.25, "low", ["No strong metadata signal."], 0.22), DetectorSignal("compression_noise_forensics", "ok", "no_strong_forensic_signal", 0.36, 0.29, "low", ["Weak forensic signal."], 0.28), DetectorSignal("hf:Ateeqq/ai-vs-human-image-detector", "ok", "model_likely_ai_generated", 0.99, None, "medium", ["a"], 0.72), DetectorSignal("hf:dima806/ai_vs_real_image_detection", "ok", "model_likely_ai_generated", 0.87, None, "medium", ["b"], 0.35), DetectorSignal("hf:jacoballessio/ai-image-detect-distilled", "ok", "model_likely_human_or_real", 0.26, None, "medium", ["c"], 0.70), DetectorSignal("hf:SadraCoding/SDXL-Deepfake-Detector", "ok", "model_likely_human_or_real", 0.0, None, "medium", ["portrait"], 0.65), DetectorSignal("open_source_model_ensemble", "ok", "ensemble_inconclusive", 0.5, None, "low", ["tie"], 0.9), ] verdict = aggregate_verdict( detectors, metadata={"generative_markers": [], "has_exif": False}, c2pa={"claim": None}, forensics={ "artificiality_score": 0.06, "manipulation_score": 0.38, "quality": {"risk_score": 0.28}, }, ) assert verdict["label"] == "inconclusive" assert verdict["confidence"] == "low" assert verdict["ai_probability"] <= 0.52 def test_portrait_specialist_is_skipped_when_portrait_gate_is_low() -> None: signal = _huggingface_detector_for_model( Image.new("RGB", (256, 256), (30, 40, 50)), "SadraCoding/SDXL-Deepfake-Detector", {"portrait_score": 0.42, "evidence": ["low portrait score"]}, ) assert signal.status == "unavailable" assert signal.label == "portrait_gate_not_met" assert signal.ai_probability is None