feat(phase0): add CriticEvaluator with rule-based gate logic

viral_script_engine/evaluation/critic_evaluator.py

@@ -0,0 +1,94 @@
+from typing import List, Tuple
+
+from pydantic import BaseModel
+
+from viral_script_engine.agents.critic import CritiqueOutput
+
+
+class EvaluationResult(BaseModel):
+    script_id: str = ""
+    claim_count: int
+    specificity_score: float
+    falsifiability_score: float
+    timestamp_coverage: float
+    critique_class_diversity: float
+    passes_gate: bool
+
+
+class BatchEvaluationResult(BaseModel):
+    pass_count: int
+    pass_rate: float
+    passes_overall_gate: bool
+    per_script_results: List[EvaluationResult]
+    failing_scripts: List[str]
+
+
+class CriticEvaluator:
+    def evaluate(self, output: CritiqueOutput, script_text: str, script_id: str = "") -> EvaluationResult:
+        claims = output.claims
+        claim_count = len(claims)
+
+        if claim_count == 0:
+            return EvaluationResult(
+                script_id=script_id,
+                claim_count=0,
+                specificity_score=0.0,
+                falsifiability_score=0.0,
+                timestamp_coverage=0.0,
+                critique_class_diversity=0.0,
+                passes_gate=False,
+            )
+
+        specificity_score = sum(
+            1 for c in claims if c.evidence and c.evidence.strip() in script_text
+        ) / claim_count
+
+        falsifiability_score = sum(1 for c in claims if c.is_falsifiable) / claim_count
+
+        timestamp_coverage = sum(
+            1 for c in claims if c.timestamp_range and c.timestamp_range.strip() != "N/A"
+        ) / claim_count
+
+        unique_classes = {c.critique_class for c in claims}
+        critique_class_diversity = len(unique_classes) / 6
+
+        passes_gate = (
+            claim_count >= 3
+            and specificity_score >= 0.6
+            and falsifiability_score >= 0.7
+        )
+
+        return EvaluationResult(
+            script_id=script_id,
+            claim_count=claim_count,
+            specificity_score=round(specificity_score, 3),
+            falsifiability_score=round(falsifiability_score, 3),
+            timestamp_coverage=round(timestamp_coverage, 3),
+            critique_class_diversity=round(critique_class_diversity, 3),
+            passes_gate=passes_gate,
+        )
+
+    def batch_evaluate(
+        self,
+        results: List[Tuple[CritiqueOutput, str]],
+        script_ids: List[str] = None,
+    ) -> BatchEvaluationResult:
+        if script_ids is None:
+            script_ids = ["" for _ in results]
+
+        per_script = [
+            self.evaluate(output, script_text, sid)
+            for (output, script_text), sid in zip(results, script_ids)
+        ]
+
+        pass_count = sum(1 for r in per_script if r.passes_gate)
+        pass_rate = pass_count / len(per_script) if per_script else 0.0
+        failing_scripts = [r.script_id for r in per_script if not r.passes_gate]
+
+        return BatchEvaluationResult(
+            pass_count=pass_count,
+            pass_rate=round(pass_rate, 3),
+            passes_overall_gate=pass_rate >= 0.8,
+            per_script_results=per_script,
+            failing_scripts=failing_scripts,
+        )

viral_script_engine/tests/test_critic.py

@@ -3,6 +3,7 @@ import pytest
 from unittest.mock import MagicMock, patch
 
 from viral_script_engine.agents.critic import CritiqueClaim, CritiqueOutput
+from viral_script_engine.evaluation.critic_evaluator import CriticEvaluator, EvaluationResult
 
 
 # ── Task 2: Model parsing tests ───────────────────────────────────────────────
@@ -40,3 +41,49 @@ def test_critique_output_parses_valid_json():
     }
     output = CritiqueOutput(**data)
     assert len(output.claims) == 1
+
+
+# ── Task 4: CriticEvaluator tests ────────────────────────────────────────────
+
+SCRIPT_TEXT = "Let me tell you a secret about money. First, save twenty percent. Second, invest in index funds. Finally, avoid lifestyle inflation. That is all."
+
+
+def _make_output(claims_data, overall="medium"):
+    claims = [CritiqueClaim(**c) for c in claims_data]
+    return CritiqueOutput(claims=claims, overall_severity=overall, raw_response="{}")
+
+
+def test_evaluator_passes_good_critique():
+    output = _make_output([
+        {"claim_id": "C1", "critique_class": "hook_weakness", "claim_text": "x", "timestamp_range": "0:00-0:03", "evidence": "Let me tell you a secret", "is_falsifiable": True, "severity": "high"},
+        {"claim_id": "C2", "critique_class": "cta_buried", "claim_text": "x", "timestamp_range": "0:10-0:15", "evidence": "That is all", "is_falsifiable": True, "severity": "medium"},
+        {"claim_id": "C3", "critique_class": "pacing_issue", "claim_text": "x", "timestamp_range": "N/A", "evidence": "save twenty percent", "is_falsifiable": True, "severity": "low"},
+    ])
+    evaluator = CriticEvaluator()
+    result = evaluator.evaluate(output, SCRIPT_TEXT)
+    assert result.passes_gate is True
+    assert result.claim_count == 3
+    assert result.specificity_score >= 0.6
+    assert result.falsifiability_score >= 0.7
+
+
+def test_evaluator_fails_too_few_claims():
+    output = _make_output([
+        {"claim_id": "C1", "critique_class": "hook_weakness", "claim_text": "x", "timestamp_range": "0:00-0:03", "evidence": "Let me tell you", "is_falsifiable": True, "severity": "high"},
+        {"claim_id": "C2", "critique_class": "cta_buried", "claim_text": "x", "timestamp_range": "0:10", "evidence": "invest", "is_falsifiable": True, "severity": "medium"},
+    ])
+    evaluator = CriticEvaluator()
+    result = evaluator.evaluate(output, SCRIPT_TEXT)
+    assert result.passes_gate is False
+    assert result.claim_count == 2
+
+
+def test_evaluator_fails_low_specificity():
+    output = _make_output([
+        {"claim_id": "C1", "critique_class": "hook_weakness", "claim_text": "x", "timestamp_range": "0:00-0:03", "evidence": "this quote is not in the script", "is_falsifiable": True, "severity": "high"},
+        {"claim_id": "C2", "critique_class": "cta_buried", "claim_text": "x", "timestamp_range": "0:10", "evidence": "also not in script", "is_falsifiable": False, "severity": "medium"},
+        {"claim_id": "C3", "critique_class": "pacing_issue", "claim_text": "x", "timestamp_range": "N/A", "evidence": "still not in script", "is_falsifiable": False, "severity": "low"},
+    ])
+    evaluator = CriticEvaluator()
+    result = evaluator.evaluate(output, SCRIPT_TEXT)
+    assert result.passes_gate is False