tinymind-native-colab-handoff/bundle/evaluation/mythos_report_analyzer.py

"""Analyze public Claude Mythos reports into evidence-bound lessons.

The analyzer treats Mythos articles/reports as comparison material, not as
ground-truth training labels. It extracts reported metrics, uncertainties, risk
signals, and transferable engineering lessons while blocking any promotion of
Mythos scores as TinyMind scores.
"""

from __future__ import annotations

from datetime import datetime, timezone
import json
from pathlib import Path
import re
from typing import Any


DEFAULT_MYTHOS_REPORT_SOURCES = [
    {
        "source_id": "pcgamer_cloudflare_mythos",
        "title": "Cloudflare says Claude Mythos reasoning looks like senior researcher work",
        "url": "https://www.pcgamer.com/software/security/web-infrastructure-company-cloudflare-says-claude-mythos-reasoning-looks-like-the-work-of-a-senior-researcher/",
        "published": "2026-05-21",
        "text": "Public article describing partner impressions of Claude Mythos security reasoning; useful as a qualitative comparison boundary, not an official TinyMind score.",
    },
    {
        "source_id": "techradar_mythos_vulnerability_claims",
        "title": "Anthropic claims Mythos found many critical vulnerabilities",
        "url": "https://www.techradar.com/pro/security/after-one-month-most-partners-have-each-found-hundreds-of-critical-or-high-severity-vulnerabilities-anthropic-claims-mythos-has-found-over-ten-thousand-major-security-vulnerabilities-across-the-most-systemically-important-software-in-the-world",
        "published": "2026-05-25",
        "text": "Public article reporting large-scale vulnerability discovery claims. Treat as reported claim requiring independent verification before comparison.",
    },
    {
        "source_id": "arxiv_mythos_bug_rediscovery",
        "title": "Benchmarking Mythos-Linked Bug Rediscovery",
        "url": "https://arxiv.org/abs/2605.17416",
        "published": "2026-05-19",
        "text": "Research paper about Mythos-linked bug rediscovery. Extract methodology constraints and avoid contaminating training with target identifiers.",
    },
    {
        "source_id": "reported_mythos_benchmark_digest",
        "title": "Reported Claude Mythos benchmark digest",
        "url": "https://alhertech.com/en/claude-mythos/benchmarks",
        "published": "2026-05-12",
        "text": (
            "A non-official benchmark digest reports Claude Mythos Preview at 93.9% on SWE-bench Verified, "
            "94.6% on GPQA Diamond, 97.6% on USAMO, 82.0% on Terminal-Bench 2.0, and 100% pass@1 on Cybench. "
            "Treat these as reported comparison-boundary claims requiring exact-source evidence before any public superiority claim."
        ),
    },
]


METRIC_PATTERNS = {
    "swe_bench_verified": re.compile(r"(?P<score>\d+(?:\.\d+)?)\s*%\s+on\s+SWE[- ]bench\s+Verified", re.I),
    "gpqa_diamond": re.compile(r"(?P<score>\d+(?:\.\d+)?)\s*%\s+on\s+GPQA\s+Diamond", re.I),
    "usamo": re.compile(r"(?P<score>\d+(?:\.\d+)?)\s*%\s+on\s+USAMO", re.I),
    "terminal_bench_2": re.compile(r"(?P<score>\d+(?:\.\d+)?)\s*%\s+on\s+Terminal[- ]Bench\s+2(?:\.0)?", re.I),
    "cybench": re.compile(r"(?P<score>\d+(?:\.\d+)?)\s*%\s+(?:pass@1\s+)?on\s+Cybench", re.I),
}


UNCERTAINTY_TERMS = (
    "reportedly",
    "claimed",
    "claims",
    "limited access",
    "preview",
    "unreleased",
    "requires",
    "uncertainty",
    "not official",
    "independent verification",
)


class MythosReportAnalyzer:
    def analyze(self, sources: list[dict[str, Any]]) -> dict[str, Any]:
        normalized = [self._normalize_source(item) for item in sources]
        source_analyses = [self._analyze_source(item) for item in normalized]
        benchmark_claims = [claim for source in source_analyses for claim in source["benchmark_claims"]]
        lessons = self._distill_lessons(source_analyses, benchmark_claims)
        uncertainty_coverage = sum(1 for item in source_analyses if item["uncertainty"]) / max(len(source_analyses), 1)
        metric_coverage = min(1.0, len({item["axis"] for item in benchmark_claims}) / 3.0)
        source_quality = sum(item["source_quality"] for item in source_analyses) / max(len(source_analyses), 1)
        analysis_depth_score = 100.0 * min(1.0, (len(lessons) / 4.0 + uncertainty_coverage + metric_coverage) / 3.0)
        claim_sharpness_score = 100.0 * min(1.0, (source_quality + uncertainty_coverage + 1.0) / 3.0)
        return {
            "schema_version": "tinymind-mythos-report-analyzer-v1",
            "created_at": datetime.now(timezone.utc).isoformat(),
            "source_count": len(normalized),
            "source_analyses": source_analyses,
            "benchmark_claims": benchmark_claims,
            "distilled_lessons": lessons,
            "scores": {
                "analysis_depth_score": analysis_depth_score,
                "claim_sharpness_score": claim_sharpness_score,
                "source_quality_score": 100.0 * source_quality,
                "uncertainty_coverage_score": 100.0 * uncertainty_coverage,
            },
            "distillation_policy": {
                "main_training_allowed": True,
                "allowed_content": "methodology, uncertainty handling, evaluation design, evidence discipline, defensive reasoning patterns",
                "blocked_content": "raw copied article text, unverified Mythos score claims as labels, exploit instructions, hidden chain-of-thought imitation",
                "strip_raw_claims": True,
            },
            "claim_gate": {
                "usable_as_training_truth": False,
                "usable_as_comparison_boundary": True,
                "can_claim_mythos_scores_as_tinymind_scores": False,
                "can_claim_above_mythos_from_reports_only": False,
                "reason": "Reports can guide eval design and purity policy; they cannot prove TinyMind outperforms Mythos.",
            },
        }

    @staticmethod
    def _normalize_source(source: dict[str, Any]) -> dict[str, str]:
        return {
            "source_id": str(source.get("source_id") or source.get("id") or "unknown"),
            "title": str(source.get("title") or "Untitled Mythos report"),
            "url": str(source.get("url") or ""),
            "published": str(source.get("published") or source.get("date") or ""),
            "text": str(source.get("text") or source.get("content") or ""),
        }

    def _analyze_source(self, source: dict[str, str]) -> dict[str, Any]:
        text = source["text"]
        benchmark_claims = self._extract_metrics(source, text)
        uncertainty = self._uncertainty_sentence(text)
        risk_signals = self._risk_signals(text)
        has_url = source["url"].startswith(("https://", "http://"))
        has_date = bool(source["published"])
        source_quality = (float(has_url) + float(has_date) + min(1.0, len(text) / 240.0)) / 3.0
        return {
            **source,
            "benchmark_claims": benchmark_claims,
            "uncertainty": uncertainty,
            "risk_signals": risk_signals,
            "source_quality": source_quality,
            "analysis_tags": self._tags(text),
        }

    @staticmethod
    def _extract_metrics(source: dict[str, str], text: str) -> list[dict[str, Any]]:
        claims = []
        for axis, pattern in METRIC_PATTERNS.items():
            match = pattern.search(text)
            if match:
                claims.append(
                    {
                        "axis": axis,
                        "reported_score": float(match.group("score")),
                        "source_id": source["source_id"],
                        "source_url": source["url"],
                        "claim_type": "reported_external_model_score",
                        "usable_for_tinymind_score": False,
                    }
                )
        return claims

    @staticmethod
    def _uncertainty_sentence(text: str) -> str:
        lower = text.lower()
        found = [term for term in UNCERTAINTY_TERMS if term in lower]
        if found:
            return "Contains uncertainty markers: " + ", ".join(sorted(set(found)))
        return "No explicit uncertainty marker found; treat as unverified until source context is audited."

    @staticmethod
    def _risk_signals(text: str) -> list[str]:
        lower = text.lower()
        signals = []
        for term, label in [
            ("vulnerability", "security_vulnerability"),
            ("exploit", "exploit_capability"),
            ("offensive", "offensive_security_risk"),
            ("limited access", "limited_access_model"),
            ("unreleased", "unreleased_model"),
        ]:
            if term in lower:
                signals.append(label)
        return sorted(set(signals))

    @staticmethod
    def _tags(text: str) -> list[str]:
        lower = text.lower()
        tags = []
        if "benchmark" in lower or "score" in lower:
            tags.append("benchmark")
        if "vulnerability" in lower or "security" in lower:
            tags.append("security_reasoning")
        if "uncertainty" in lower or "limited access" in lower or "reportedly" in lower:
            tags.append("uncertainty")
        if "evidence" in lower or "verification" in lower:
            tags.append("evidence_policy")
        return tags or ["general_report"]

    @staticmethod
    def _distill_lessons(source_analyses: list[dict[str, Any]], claims: list[dict[str, Any]]) -> list[dict[str, str]]:
        lessons = [
            {
                "lesson_id": "mythos-eval-boundary",
                "kind": "evaluation_policy",
                "principle": "Separate reported frontier-model scores from TinyMind scores until matched raw/external evaluation exists.",
                "training_use": "Teach the model to state evidence boundaries before comparing systems.",
            },
            {
                "lesson_id": "mythos-uncertainty-discipline",
                "kind": "claim_discipline",
                "principle": "Treat preview, limited-access, and reportedly phrased claims as provisional evidence.",
                "training_use": "Improve factual humility and reduce unsupported certainty.",
            },
            {
                "lesson_id": "mythos-security-methodology",
                "kind": "methodology",
                "principle": "Security reasoning claims require target isolation, answer-key removal, repeated runs, and dated source records.",
                "training_use": "Improve defensive analysis workflows without copying exploit instructions.",
            },
        ]
        if claims:
            axes = ", ".join(sorted({claim["axis"] for claim in claims}))
            lessons.append(
                {
                    "lesson_id": "mythos-benchmark-axis-map",
                    "kind": "benchmark_mapping",
                    "principle": f"Map Mythos-reported axes ({axes}) into TinyMind eval slots, but keep the scores as external comparison only.",
                    "training_use": "Guide benchmark planning without contaminating model training labels.",
                }
            )
        if any(item["risk_signals"] for item in source_analyses):
            lessons.append(
                {
                    "lesson_id": "mythos-risk-containment",
                    "kind": "risk_control",
                    "principle": "High-capability security analysis must route through authorization, sandboxing, and evidence-first reporting.",
                    "training_use": "Sharpen tool-grounded safe analysis behavior.",
                }
            )
        return lessons


def _load_sources(path: str | Path | None) -> list[dict[str, Any]]:
    if not path:
        return DEFAULT_MYTHOS_REPORT_SOURCES
    payload = json.loads(Path(path).read_text(encoding="utf-8"))
    if isinstance(payload, list):
        return payload
    return payload.get("sources", DEFAULT_MYTHOS_REPORT_SOURCES)


def build_mythos_report_analysis(out_dir: str | Path, source_path: str | Path | None = None) -> dict[str, Any]:
    sources = _load_sources(source_path)
    report = MythosReportAnalyzer().analyze(sources)
    out = Path(out_dir)
    out.mkdir(parents=True, exist_ok=True)
    json_path = out / "mythos_report_analysis.json"
    md_path = out / "mythos_report_analysis.md"
    lessons_path = out / "mythos_distilled_lessons.jsonl"
    report["json_path"] = str(json_path)
    report["markdown_path"] = str(md_path)
    report["lessons_jsonl"] = str(lessons_path)
    json_path.write_text(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True), encoding="utf-8")
    md_path.write_text(_markdown(report), encoding="utf-8")
    with lessons_path.open("w", encoding="utf-8") as f:
        for lesson in report["distilled_lessons"]:
            f.write(json.dumps(lesson, ensure_ascii=False, sort_keys=True) + "\n")
    return report


def _markdown(report: dict[str, Any]) -> str:
    lines = [
        "# TinyMind Mythos Report Analysis",
        "",
        f"- Sources: {report['source_count']}",
        f"- Analysis depth score: {report['scores']['analysis_depth_score']:.2f}",
        f"- Claim sharpness score: {report['scores']['claim_sharpness_score']:.2f}",
        f"- Can claim Mythos scores as TinyMind scores: {report['claim_gate']['can_claim_mythos_scores_as_tinymind_scores']}",
        "",
        "## Benchmark Claims",
        "",
        "| Axis | Reported Score | Source | Usable as TinyMind Score |",
        "|---|---:|---|---|",
    ]
    for claim in report["benchmark_claims"]:
        lines.append(
            f"| {claim['axis']} | {claim['reported_score']:.2f} | {claim['source_id']} | {claim['usable_for_tinymind_score']} |"
        )
    lines.extend(["", "## Distilled Lessons", ""])
    for lesson in report["distilled_lessons"]:
        lines.append(f"- {lesson['lesson_id']}: {lesson['principle']}")
    return "\n".join(lines) + "\n"