tinymind-native-8b-remote-handoff/bundle/evaluation/gguf_evo_upgrade.py

from __future__ import annotations

from datetime import datetime, timezone
import hashlib
import json
from pathlib import Path
from typing import Any


DEFAULT_SYSTEM = """You are TinyMind GGUF Evo, an evidence-first local model runtime.

Operating law:
- Answer from grounded evidence when available; clearly mark uncertainty when evidence is missing.
- Prefer concise structure for simple tasks and deep step-by-step reasoning for hard tasks.
- Preserve Thai and English nuance; do not translate away technical meaning.
- For code, provide runnable, minimal, audited patches or commands.
- For long context, summarize anchors first, then answer from exact anchors.
- Never claim the GGUF weights were retrained unless a saved training/export manifest proves it.
- Refuse credential leakage, destructive actions, exploit chains, stealth, and malware improvement.

Quality style:
- Be direct, natural, and precise.
- Separate Fact, Inference, and Next Verification when stakes are high.
- If unsure, propose the smallest real measurement that resolves uncertainty.
"""


EVAL_PROMPTS = [
    {
        "id": "thai_technical_explain",
        "prompt": "อธิบาย QLoRA กับ GGUF ต่างกันอย่างไรแบบเข้าใจง่ายแต่ครบถ้วน",
        "checks": ["ภาษาไทย", "QLoRA", "GGUF", "ไม่อ้างว่า train GGUF ตรงๆ"],
    },
    {
        "id": "code_patch_reasoning",
        "prompt": "Given a Python JSONL reader that crashes on one bad line, design a robust fix and test plan.",
        "checks": ["skip invalid line", "strict=False or guarded decode", "test"],
    },
    {
        "id": "grounding_boundary",
        "prompt": "Can you claim this model is world best after one local smoke eval?",
        "checks": ["no", "external eval", "saved evidence"],
    },
    {
        "id": "long_context_anchor",
        "prompt": "In a 10M token archive, how can a small model recall exact details without hallucinating?",
        "checks": ["external ledger", "hash", "retrieval", "regenerated KV"],
    },
]


def _sha256(path: Path, chunk_mb: int = 64) -> str:
    h = hashlib.sha256()
    with path.open("rb") as f:
        while True:
            b = f.read(chunk_mb * 1024 * 1024)
            if not b:
                break
            h.update(b)
    return h.hexdigest()


def build_gguf_evo_upgrade(
    out_dir: str | Path,
    *,
    gguf_path: str | Path,
    base_modelfile: str | Path | None = None,
    adapter_manifest: str | Path | None = None,
    data_manifest: str | Path | None = None,
    training_manifest: str | Path | None = None,
    model_name: str = "tinymind-gguf-evo",
    num_ctx: int = 32768,
    num_predict: int = 8192,
    temperature: float = 0.18,
    top_p: float = 0.82,
    top_k: int = 40,
    repeat_penalty: float = 1.17,
) -> dict[str, Any]:
    gguf = Path(gguf_path)
    if not gguf.exists():
        raise FileNotFoundError(f"GGUF artifact not found: {gguf}")
    out = Path(out_dir)
    out.mkdir(parents=True, exist_ok=True)

    modelfile = out / "Modelfile.evo"
    manifest_path = out / "gguf_evo_upgrade_manifest.json"
    eval_path = out / "gguf_evo_eval_prompts.jsonl"
    create_script = out / "create_ollama_evo.ps1"
    compare_script = out / "compare_rawzero_vs_evo.ps1"
    ggufx_spec_path = out / "tinymind_ggufx_spec.json"
    ggufx_readme_path = out / "README_GGUF_X.md"

    system = DEFAULT_SYSTEM.strip()
    modelfile.write_text(
        "\n".join(
            [
                f"FROM {gguf}",
                "",
                f"PARAMETER temperature {temperature}",
                f"PARAMETER top_p {top_p}",
                f"PARAMETER top_k {top_k}",
                f"PARAMETER num_ctx {num_ctx}",
                f"PARAMETER num_predict {num_predict}",
                f"PARAMETER repeat_penalty {repeat_penalty}",
                "PARAMETER repeat_last_n 4096",
                "PARAMETER mirostat 2",
                "PARAMETER mirostat_tau 4.2",
                "PARAMETER mirostat_eta 0.08",
                "",
                'SYSTEM """',
                system,
                '"""',
                "",
            ]
        ),
        encoding="utf-8",
        newline="\n",
    )
    with eval_path.open("w", encoding="utf-8", newline="\n") as f:
        for row in EVAL_PROMPTS:
            f.write(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n")

    create_script.write_text(
        f"ollama create {model_name} -f \"{modelfile}\"\nollama show {model_name}\n",
        encoding="utf-8",
        newline="\n",
    )
    compare_script.write_text(
        "\n".join(
            [
                "$ErrorActionPreference = 'Stop'",
                f"$prompts = Get-Content -LiteralPath '{eval_path}' | ForEach-Object {{ $_ | ConvertFrom-Json }}",
                "$out = @()",
                "foreach ($p in $prompts) {",
                "  $raw = ollama run tinymind-rawzero-fusion $p.prompt",
                f"  $evo = ollama run {model_name} $p.prompt",
                "  $out += [ordered]@{ id=$p.id; prompt=$p.prompt; rawzero=$raw; evo=$evo; checks=$p.checks }",
                "}",
                f"$out | ConvertTo-Json -Depth 8 | Set-Content -Path '{out / 'rawzero_vs_evo_outputs.json'}' -Encoding UTF8",
            ]
        )
        + "\n",
        encoding="utf-8",
        newline="\n",
    )

    def _read_json(path: str | Path | None) -> dict[str, Any]:
        if not path:
            return {}
        p = Path(path)
        if not p.exists():
            return {"missing_path": str(p)}
        try:
            return json.loads(p.read_text(encoding="utf-8"))
        except json.JSONDecodeError:
            return {"unreadable_json_path": str(p)}

    adapter_evidence = _read_json(adapter_manifest)
    data_evidence = _read_json(data_manifest)
    training_evidence = _read_json(training_manifest)
    ggufx_spec = {
        "schema_version": "tinymind-gguf-x-v1",
        "format_kind": "GGUF-v3-compatible-binary-plus-TinyMind-sidecar",
        "binary_compatibility": {
            "base_container": "GGUF",
            "base_container_version": 3,
            "runtime_target": "Ollama / llama.cpp compatible GGUF loader",
            "sidecar_required_for_tinymind_features": True,
        },
        "tinymind_extensions": {
            "purity_lineage_gate": {
                "enabled": bool(data_manifest),
                "manifest": str(data_manifest) if data_manifest else None,
                "purpose": "tie runtime claims to the exact purity-concentrated dataset used for adapter training",
            },
            "adapter_lineage_gate": {
                "enabled": bool(adapter_manifest or training_manifest),
                "adapter_manifest": str(adapter_manifest) if adapter_manifest else None,
                "training_manifest": str(training_manifest) if training_manifest else None,
                "purpose": "separate trained LoRA evidence from GGUF packaging evidence",
            },
            "evidence_first_decode_law": {
                "enabled": True,
                "temperature": temperature,
                "top_p": top_p,
                "repeat_penalty": repeat_penalty,
                "purpose": "reduce hallucination/repetition without pretending the GGUF weights changed",
            },
            "regen_ledger_ready_metadata": {
                "enabled": True,
                "kv_growth_claim": "bounded only when paired with Evidence Ledger/ReGenesis retrieval runtime",
            },
        },
        "claim_rules": {
            "may_claim_custom_tinymind_format_pack": True,
            "may_claim_binary_gguf_v3_compatibility": True,
            "may_claim_weights_retrained_inside_gguf": False,
            "may_claim_better_than_gguf_v3": False,
            "required_to_unlock_better_than_v3_claim": [
                "real adapter-to-GGUF merge/export log",
                "baseline GGUF v3 eval",
                "GGUF-X eval on same prompts",
                "latency/memory report",
                "hashes for every artifact",
            ],
        },
    }
    ggufx_spec_path.write_text(json.dumps(ggufx_spec, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8")
    ggufx_readme_path.write_text(
        "\n".join(
            [
                "# TinyMind GGUF-X",
                "",
                "GGUF-X is a TinyMind runtime package: a GGUF v3-compatible model plus sidecar evidence, decode policy, and lineage gates.",
                "It does not mutate GGUF tensors directly and does not claim better weights without a real merge/export/eval report.",
                "",
                "## Files",
                "",
                f"- Modelfile: `{modelfile}`",
                f"- Spec: `{ggufx_spec_path}`",
                f"- Eval prompts: `{eval_path}`",
                f"- Create script: `{create_script}`",
                f"- Compare script: `{compare_script}`",
                "",
                "## Claim Boundary",
                "",
                "- Custom TinyMind format pack: allowed",
                "- GGUF v3 runtime compatibility: allowed",
                "- Better-than-source GGUF weights: blocked until same-prompt baseline evidence exists",
                "- World-best/runtime superiority: blocked until external benchmark evidence exists",
                "",
            ]
        ),
        encoding="utf-8",
        newline="\n",
    )

    size_bytes = gguf.stat().st_size
    manifest = {
        "schema_version": "tinymind-gguf-evo-upgrade-v1",
        "created_at": datetime.now(timezone.utc).isoformat(),
        "model_name": model_name,
        "source_gguf": str(gguf),
        "source_gguf_size_bytes": size_bytes,
        "source_gguf_size_gb": size_bytes / (1024**3),
        "source_gguf_sha256": _sha256(gguf),
        "base_modelfile": str(base_modelfile) if base_modelfile else None,
        "evo_modelfile": str(modelfile),
        "ggufx_spec": str(ggufx_spec_path),
        "ggufx_readme": str(ggufx_readme_path),
        "eval_prompts": str(eval_path),
        "create_script": str(create_script),
        "compare_script": str(compare_script),
        "lineage": {
            "adapter_manifest": str(adapter_manifest) if adapter_manifest else None,
            "data_manifest": str(data_manifest) if data_manifest else None,
            "training_manifest": str(training_manifest) if training_manifest else None,
            "adapter_eval_loss": adapter_evidence.get("eval_loss"),
            "adapter_perplexity": adapter_evidence.get("perplexity"),
            "training_eval_loss": training_evidence.get("eval_loss"),
            "training_perplexity": training_evidence.get("perplexity"),
            "purity_density": (data_evidence.get("metrics") or {}).get("avg_purity_density_score"),
            "dominant_domain_share": (data_evidence.get("metrics") or {}).get("dominant_domain_share"),
        },
        "ggufx_format": ggufx_spec,
        "runtime_upgrade": {
            "context_window_requested": num_ctx,
            "decode_profile": {
                "temperature": temperature,
                "top_p": top_p,
                "top_k": top_k,
                "repeat_penalty": repeat_penalty,
                "mirostat": 2,
            },
            "quality_controls": [
                "evidence-first system law",
                "Thai-English technical preservation",
                "long-context anchor discipline",
                "claim-boundary enforcement",
                "lower-temperature repetition-resistant decoding",
            ],
        },
        "promotion_gate": {
            "rawzero_baseline_required": True,
            "evo_eval_required": True,
            "must_beat_baseline_on_prompt_suite": True,
            "adapter_training_evidence_present": bool(adapter_evidence or training_evidence),
            "weight_training_performed": bool(training_evidence),
            "gguf_binary_tensor_merge_performed": False,
            "custom_ggufx_sidecar_created": True,
            "can_claim_weights_better_than_source": False,
            "can_claim_runtime_quality_upgrade": True,
            "can_claim_better_than_v3": False,
            "reason": "This pack upgrades GGUF runtime behavior and evaluation path. Weight-level improvement requires real conversion/export from trained adapters plus benchmark evidence.",
        },
    }
    manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True), encoding="utf-8")
    manifest["manifest_path"] = str(manifest_path)
    return manifest