tinymind-native-8b-remote-handoff/bundle/evaluation/lmmarketcap_compare.py

"""LM MarketCap external compare API evidence runner."""

from __future__ import annotations

from datetime import datetime, timezone
import json
import os
from pathlib import Path
from typing import Any, Iterable
from urllib.parse import urlencode


BASE_URL = "https://api.lmmarketcap.com/v1/compare"


def _parse_models(models: str | Iterable[str]) -> list[str]:
    if isinstance(models, str):
        parts = []
        for chunk in models.split(","):
            item = chunk.strip()
            if item:
                parts.append(item)
        return parts
    return [str(model).strip() for model in models if str(model).strip()]


def _json_safe(value: Any) -> Any:
    if value is None or isinstance(value, (str, int, float, bool)):
        return value
    if isinstance(value, dict):
        return {str(k): _json_safe(v) for k, v in value.items()}
    if isinstance(value, (list, tuple)):
        return [_json_safe(v) for v in value]
    return str(value)


def _request_url(models: list[str], category: str) -> str:
    return f"{BASE_URL}?{urlencode({'models': ','.join(models), 'category': category})}"


def run_lmmarketcap_compare(
    out_dir: str | Path,
    models: str | Iterable[str],
    category: str,
    execute: bool = True,
    timeout: float = 30.0,
) -> dict:
    out = Path(out_dir)
    out.mkdir(parents=True, exist_ok=True)
    model_list = _parse_models(models)
    api_key = os.environ.get("LMMARKETCAP_API_KEY", "")
    url = _request_url(model_list, category)
    api_key_present = bool(api_key)

    status = "skipped_missing_api_key"
    status_code = None
    raw_payload: Any = None
    error = None
    if execute and api_key_present:
        try:
            import httpx

            response = httpx.get(
                BASE_URL,
                params={"models": ",".join(model_list), "category": category},
                headers={"X-API-Key": api_key, "Accept": "application/json"},
                timeout=timeout,
            )
            status_code = response.status_code
            response.raise_for_status()
            raw_payload = response.json()
            status = "ok"
        except Exception as exc:
            status = "error"
            error = f"{type(exc).__name__}: {exc}"

    raw_path = out / "lmmarketcap_compare_raw.json"
    raw_path.write_text(json.dumps(_json_safe(raw_payload), ensure_ascii=False, indent=2, sort_keys=True), encoding="utf-8")

    report = {
        "schema_version": "tinymind-lmmarketcap-compare-v1",
        "created_at": datetime.now(timezone.utc).isoformat(),
        "provider": "lmmarketcap",
        "request": {
            "url": url,
            "models": model_list,
            "category": category,
            "method": "GET",
            "headers_saved": {"Accept": "application/json", "X-API-Key": "redacted"},
        },
        "api_key_present": api_key_present,
        "api_key_saved": False,
        "execute": bool(execute),
        "status": status,
        "status_code": status_code,
        "error": error,
        "raw_path": str(raw_path),
        "raw": _json_safe(raw_payload),
        "external_results": _to_external_results(raw_payload, model_list, category, url),
        "claim_gate": {
            "external_official_claim_allowed": False,
            "rank_claim_allowed": False,
            "reason": "LM MarketCap compare output is external reference evidence only until mapped to TinyMind's own listed model and official rank criteria.",
        },
    }
    json_path = out / "lmmarketcap_compare_report.json"
    md_path = out / "lmmarketcap_compare_report.md"
    report["json_path"] = str(json_path)
    report["markdown_path"] = str(md_path)
    json_path.write_text(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True), encoding="utf-8")
    md_path.write_text(_markdown(report), encoding="utf-8")
    return report


def _to_external_results(raw: Any, models: list[str], category: str, url: str) -> list[dict]:
    if raw is None:
        return []
    return [
        {
            "source": "lmmarketcap",
            "model": ",".join(models),
            "source_url": url,
            "as_of": datetime.now(timezone.utc).date().isoformat(),
            "official": False,
            "category": category,
            "scores": {},
            "raw_attached": True,
        }
    ]


def _markdown(report: dict) -> str:
    return "\n".join(
        [
            "# TinyMind LM MarketCap Compare",
            "",
            f"- Status: {report['status']}",
            f"- Status code: {report['status_code']}",
            f"- Models: {', '.join(report['request']['models'])}",
            f"- Category: {report['request']['category']}",
            f"- API key present: {report['api_key_present']}",
            "- API key saved: false",
            "- Rank claim allowed: false",
            "",
            report["claim_gate"]["reason"],
            "",
        ]
    )