Spaces:
Sleeping
Sleeping
File size: 3,663 Bytes
ce19bf0 e4f13c4 ce19bf0 e4f13c4 ce19bf0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 | """Public Hugging Face demo for scaffold-harness.
The demo intentionally runs only the built-in deterministic smoke comparison.
It does not load a model, call a network service, or execute user-provided code.
"""
from __future__ import annotations
import json
import tempfile
from pathlib import Path
import gradio as gr
import spaces
from scaffold_harness.cli import smoke
@spaces.GPU
def run_smoke_demo(language: str):
"""Run the three-case offline demonstration and expose its signed reports."""
output_dir = Path(tempfile.mkdtemp(prefix="scaffold-harness-space-"))
language_code = "fr" if language == "Français" else "en"
smoke(output_dir, language_code)
report_path = output_dir / "report.json"
html_path = output_dir / "report.html"
report = json.loads(report_path.read_text(encoding="utf-8"))
variant = report["variants"][0]
deviation = variant["deviation_vs_reference"]
if language_code == "fr":
summary = f"""## Résultat : `{variant['outcome'].upper()}`
- Exactitude de référence : **{report['baseline']['correct']}/{report['case_count']}**
- Exactitude de la couche : **{variant['correct']}/{report['case_count']}**
- Réponses modifiées : **{deviation['changed']}**
- Améliorées : **{deviation['improved']}**
- Détruites : **{deviation['destroyed']}**
- Valeur *p* exacte de McNemar : **{variant['mcnemar_p']:.4f}**
Une seule réponse a été dégradée. Sur trois cas, la preuve est
insuffisante pour conclure statistiquement : le harnais retourne honnêtement
`INCONCLUSIVE`.
"""
else:
summary = f"""## Result: `{variant['outcome'].upper()}`
- Reference accuracy: **{report['baseline']['correct']}/{report['case_count']}**
- Layer accuracy: **{variant['correct']}/{report['case_count']}**
- Answers changed: **{deviation['changed']}**
- Improved: **{deviation['improved']}**
- Destroyed: **{deviation['destroyed']}**
- Exact McNemar *p*: **{variant['mcnemar_p']:.4f}**
One answer was degraded. With only three cases, the evidence is insufficient
for a statistical conclusion, so the harness honestly returns
`INCONCLUSIVE`.
"""
return summary, report, [str(html_path), str(report_path)]
with gr.Blocks(title="scaffold-harness") as demo:
gr.Markdown(
"""
# scaffold-harness
**Measure whether the layer built on top of an LLM helps or hurts.**
This safe public demonstration compares a perfect deterministic reference with
a layer that rounds one rational answer incorrectly. It runs three built-in
questions, entirely offline, and produces the same signed JSON and standalone
HTML reports as the command-line tool.
No model is loaded. No API is called. No user code is executed.
"""
)
language = gr.Radio(
choices=["English", "Français"], value="English", label="Report language"
)
run_button = gr.Button("Run the paired smoke comparison", variant="primary")
summary_output = gr.Markdown()
with gr.Accordion("Signed JSON report", open=False):
json_output = gr.JSON()
files_output = gr.File(label="Download the standalone reports", file_count="multiple")
run_button.click(
fn=run_smoke_demo,
inputs=language,
outputs=[summary_output, json_output, files_output],
)
gr.Markdown(
"""
---
[Source and documentation](https://github.com/sxc3030-eng/scaffold-harness) ·
[MAT Nexus benchmark dashboard](https://huggingface.co/spaces/genia-dev/MAT-Nexus-Benchmark)
Early public release. Never run an untrusted configuration that uses the
Python adapter; such a configuration names code to import and execute.
"""
)
if __name__ == "__main__":
demo.launch()
|