Spaces:
Sleeping
Sleeping
File size: 3,018 Bytes
f89ffa8 de61984 f89ffa8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 | """
evaluate.py — Baseline vs trained agent comparison
Run before and after training to measure improvement.
Usage:
python evaluate.py --url http://localhost:8000
python evaluate.py --url https://your-hf-username-datacentric-env.hf.space
"""
import requests
import json
import random
import argparse
import matplotlib.pyplot as plt
parser = argparse.ArgumentParser(description="Evaluate DataCentric-Env agent")
parser.add_argument("--url", default="https://aswini-kumar-datacentric-env.hf.space", help="Environment server URL")
parser.add_argument("--episodes", type=int, default=20, help="Number of evaluation episodes")
args = parser.parse_args()
ENV_URL = args.url.rstrip("/")
N_EPISODES = args.episodes
def random_agent_episode():
"""Baseline: random tool selection."""
obs = requests.post(f"{ENV_URL}/reset").json()
tools = ["cleaner", "augmenter", "balancer", "validator"]
total_reward = 0.0
success = False
for _ in range(10):
action = {"agent": random.choice(tools), "target": "all"}
result = requests.post(f"{ENV_URL}/step", json=action).json()
total_reward += result.get("reward", 0)
if result.get("done"):
success = result.get("info", {}).get("success", False)
break
return total_reward, success
# ─── Run baseline ─────────────────────────────────────────────────────────────
print(f"Running {N_EPISODES} baseline (random) episodes against {ENV_URL}...")
baseline_rewards = []
baseline_successes = []
for i in range(N_EPISODES):
reward, success = random_agent_episode()
baseline_rewards.append(reward)
baseline_successes.append(success)
print(f" Episode {i+1:02d}: reward={reward:.3f} success={success}")
mean_baseline = sum(baseline_rewards) / len(baseline_rewards)
success_rate_baseline = sum(baseline_successes) / len(baseline_successes)
print(f"\nBaseline mean reward: {mean_baseline:.3f}")
print(f"Baseline success rate: {success_rate_baseline:.1%}")
# ─── Plot reward curve ────────────────────────────────────────────────────────
plt.figure(figsize=(10, 4))
plt.subplot(1, 2, 1)
plt.plot(range(1, N_EPISODES + 1), baseline_rewards, marker="o", color="#5B8FF9", label="Random baseline")
plt.xlabel("Episode")
plt.ylabel("Total Reward")
plt.title("Baseline Reward per Episode")
plt.legend()
plt.grid(alpha=0.3)
plt.subplot(1, 2, 2)
mean_trained = mean_baseline * 1.0 # placeholder — replace with trained agent result
plt.bar(["Random baseline", "Trained agent"],
[mean_baseline, mean_trained],
color=["#5B8FF9", "#5AD8A6"])
plt.ylabel("Mean Episode Reward")
plt.title("Baseline vs Trained Agent")
plt.grid(alpha=0.3, axis="y")
plt.tight_layout()
plt.savefig("results.png", dpi=150)
print("\nSaved results.png")
|