app.py

import os
import subprocess
import json
import gradio as gr

def run_benchmark(model_id, tasks):
    # Construct lm-eval CLI command
    cmd = [
        "lm_eval",
        "--model", "hf",
        "--model_args", f"pretrained={model_id},dtype=float16,trust_remote_code=True",
        "--tasks", tasks,
        "--output_path", "./results.json",
        "--batch_size", "2",
    ]
    
    # Execute benchmark on Hugging Face cloud servers
    result = subprocess.run(cmd, capture_output=True, text=True)
    
    # Read generated results JSON
    try:
        with open("./results.json", "r") as f:
            data = json.load(f)
        return json.dumps(data.get("results", {}), indent=2)
    except Exception as e:
        return f"Execution Log:\n{result.stdout}\n\nErrors:\n{result.stderr}"

# Create Web Interface inside Hugging Face Space
demo = gr.Interface(
    fn=run_benchmark,
    inputs=[
        gr.Textbox(value="Amit0392/AMIT-1.0", label="Model ID"),
        gr.Textbox(value="gsm8k,arc_challenge", label="Benchmark Tasks (comma separated)")
    ],
    outputs=gr.Code(label="Benchmark Results JSON", language="json"),
    title="🏆 A.M.I.T. 1.0 Cloud Benchmark Evaluator",
    description="Runs EleutherAI lm-evaluation-harness directly on Hugging Face Cloud Hardware."
)

demo.launch()