app.py

"""Public Hugging Face demo for scaffold-harness.

The demo intentionally runs only the built-in deterministic smoke comparison.
It does not load a model, call a network service, or execute user-provided code.
"""

from __future__ import annotations

import json
import tempfile
from pathlib import Path

import gradio as gr
import spaces

from scaffold_harness.cli import smoke


@spaces.GPU
def run_smoke_demo(language: str):
    """Run the three-case offline demonstration and expose its signed reports."""
    output_dir = Path(tempfile.mkdtemp(prefix="scaffold-harness-space-"))
    language_code = "fr" if language == "Français" else "en"
    smoke(output_dir, language_code)

    report_path = output_dir / "report.json"
    html_path = output_dir / "report.html"
    report = json.loads(report_path.read_text(encoding="utf-8"))
    variant = report["variants"][0]
    deviation = variant["deviation_vs_reference"]

    if language_code == "fr":
        summary = f"""## Résultat : `{variant['outcome'].upper()}`

- Exactitude de référence : **{report['baseline']['correct']}/{report['case_count']}**
- Exactitude de la couche : **{variant['correct']}/{report['case_count']}**
- Réponses modifiées : **{deviation['changed']}**
- Améliorées : **{deviation['improved']}**
- Détruites : **{deviation['destroyed']}**
- Valeur *p* exacte de McNemar : **{variant['mcnemar_p']:.4f}**

Une seule réponse a été dégradée. Sur trois cas, la preuve est
insuffisante pour conclure statistiquement : le harnais retourne honnêtement
`INCONCLUSIVE`.
"""
    else:
        summary = f"""## Result: `{variant['outcome'].upper()}`

- Reference accuracy: **{report['baseline']['correct']}/{report['case_count']}**
- Layer accuracy: **{variant['correct']}/{report['case_count']}**
- Answers changed: **{deviation['changed']}**
- Improved: **{deviation['improved']}**
- Destroyed: **{deviation['destroyed']}**
- Exact McNemar *p*: **{variant['mcnemar_p']:.4f}**

One answer was degraded. With only three cases, the evidence is insufficient
for a statistical conclusion, so the harness honestly returns
`INCONCLUSIVE`.
"""

    return summary, report, [str(html_path), str(report_path)]


with gr.Blocks(title="scaffold-harness") as demo:
    gr.Markdown(
        """
# scaffold-harness

**Measure whether the layer built on top of an LLM helps or hurts.**

This safe public demonstration compares a perfect deterministic reference with
a layer that rounds one rational answer incorrectly. It runs three built-in
questions, entirely offline, and produces the same signed JSON and standalone
HTML reports as the command-line tool.

No model is loaded. No API is called. No user code is executed.
"""
    )
    language = gr.Radio(
        choices=["English", "Français"], value="English", label="Report language"
    )
    run_button = gr.Button("Run the paired smoke comparison", variant="primary")
    summary_output = gr.Markdown()
    with gr.Accordion("Signed JSON report", open=False):
        json_output = gr.JSON()
    files_output = gr.File(label="Download the standalone reports", file_count="multiple")
    run_button.click(
        fn=run_smoke_demo,
        inputs=language,
        outputs=[summary_output, json_output, files_output],
    )
    gr.Markdown(
        """
---

[Source and documentation](https://github.com/sxc3030-eng/scaffold-harness) ·
[MAT Nexus benchmark dashboard](https://huggingface.co/spaces/genia-dev/MAT-Nexus-Benchmark)

Early public release. Never run an untrusted configuration that uses the
Python adapter; such a configuration names code to import and execute.
"""
    )


if __name__ == "__main__":
    demo.launch()