File size: 4,269 Bytes
1c88b90 cca9858 1c88b90 cca9858 1c88b90 cca9858 1c88b90 cca9858 1c88b90 cca9858 1c88b90 cca9858 1c88b90 cca9858 1c88b90 cca9858 1c88b90 cca9858 1c88b90 cca9858 1c88b90 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 | # app.py
import gradio as gr
import pandas as pd
from huggingface_hub import InferenceClient
# Initialize the free Hugging Face Inference Client
# When deployed on HF Spaces, this automatically leverages the environment to run key-free!
client = InferenceClient()
def run_evaluation(csv_file, eval_type):
if csv_file is None:
return "Please upload a CSV file first.", None
# 1. Load the dataset (Expects 'prompt' and 'response' columns)
df = pd.read_csv(csv_file.name)
if not all(col in df.columns for col in ['prompt', 'response']):
return "Error: CSV must contain at least 'prompt' and 'response' columns.", None
results = []
# 2. Run the evaluations
for idx, row in df.iterrows():
prompt = row['prompt']
response = row['response']
score = 0
reasoning = ""
if eval_type == "Basic: Length & Overlap":
# Pure local python check: check word count and keyword overlap
word_count = len(response.split())
overlap_words = set(prompt.lower().split()) & set(response.lower().split())
score = round(min(len(overlap_words) / max(len(prompt.split()), 1) * 5, 5), 1)
reasoning = f"Length: {word_count} words. Prompt keyword overlap: {len(overlap_words)} words."
elif eval_type == "LLM-as-a-Judge: Correctness (Free Llama-3)":
# Use a free open-source model as our evaluator
judge_prompt = f"""<|im_start|>system
You are a precise evaluation assistant. Score the response based on the prompt.
Output your final answer exactly in this format:
Score: [Your Score from 1 to 5]
Reasoning: [One brief sentence explaining the score]
<|im_end|>
<|im_start|>user
Prompt: {prompt}
Response: {response}
<|im_end|>
<|im_start|>assistant
"""
try:
# We use Qwen-2.5-72B-Instruct or Llama-3-8B-Instruct (both are highly capable and free)
completion = client.text_generation(
prompt=judge_prompt,
model="Qwen/Qwen2.5-72B-Instruct",
max_new_tokens=150,
temperature=0.1
)
# Parse the judge's response
for line in completion.split('\n'):
if "Score:" in line:
# Extract number
score_part = line.split("Score:")[-1].strip()
# Clean up any trailing text
score = float(''.join(c for c in score_part if c.isdigit() or c == '.'))
elif "Reasoning:" in line:
reasoning = line.split("Reasoning:")[-1].strip()
except Exception as e:
score, reasoning = 0, f"HF API Error: {str(e)}"
results.append({"Score": score, "Reasoning": reasoning})
# 3. Compile and present results
results_df = pd.concat([df, pd.DataFrame(results)], axis=1)
avg_score = results_df["Score"].mean()
summary_text = f"### Evaluation Complete! \n**Average Score:** {avg_score:.2f} / 5.0"
return summary_text, results_df
# --- Gradio UI Layout ---
with gr.Blocks(title="Free LLM Eval MVP") as demo:
gr.Markdown("# 🧪 Keyless LLM Eval Engine")
gr.Markdown("Upload a CSV dataset with your model's outputs and evaluate them using free open-source models.")
with gr.Row():
with gr.Column(scale=1):
file_input = gr.File(label="Upload Evaluation CSV (must have 'prompt' and 'response')", file_types=[".csv"])
eval_selector = gr.Radio(
choices=["Basic: Length & Overlap", "LLM-as-a-Judge: Correctness (Free Llama-3)"],
value="Basic: Length & Overlap",
label="Select Evaluator"
)
submit_btn = gr.Button("Run Evals", variant="primary")
with gr.Column(scale=2):
output_summary = gr.Markdown()
output_table = gr.DataFrame(label="Evaluation Results")
submit_btn.click(
fn=run_evaluation,
inputs=[file_input, eval_selector],
outputs=[output_summary, output_table]
)
if __name__ == "__main__":
demo.launch() |