yxc20098 commited on
Commit
2771ddd
ยท
1 Parent(s): 824262a

Rename Evaluate tab to Try, stream LLM agent gameplay

Browse files

- Replace Evaluate tab with Try tab for watching an LLM play Red Alert
- Consume SSE stream from OpenRA-RL /try-agent endpoint
- Display live game log: LLM reasoning, tool calls, game state updates
- Show final scorecard with military stats
- No API key needed โ€” uses server-provided model

Files changed (1) hide show
  1. app.py +128 -106
app.py CHANGED
@@ -10,7 +10,6 @@ Deploy on HuggingFace Spaces:
10
  Push app.py, requirements.txt, data/, and README.md to your HF Space.
11
  """
12
 
13
- import asyncio
14
  import csv
15
  import json
16
  import os
@@ -18,9 +17,10 @@ from datetime import datetime, timezone
18
  from pathlib import Path
19
 
20
  import gradio as gr
 
21
  import pandas as pd
22
 
23
- from evaluate_runner import DEFAULT_SERVER, run_evaluation, wake_hf_space
24
 
25
  # โ”€โ”€ Data Loading โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
26
 
@@ -171,88 +171,121 @@ def save_submission(results: dict) -> None:
171
  writer.writerow(results)
172
 
173
 
174
- # โ”€โ”€ Evaluation Handler โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
175
 
176
 
177
- def run_eval_sync(agent_name: str, opponent: str, num_games: int):
178
- """Generator that runs evaluation and yields progress updates."""
179
- if not agent_name or not agent_name.strip():
180
- yield "Error: Please enter an agent name.", None, ""
181
- return
182
-
183
- agent_name = agent_name.strip()
184
- num_games = int(num_games)
185
-
186
  log_lines = []
187
 
188
  def log(msg: str):
189
  log_lines.append(msg)
190
  return "\n".join(log_lines)
191
 
192
- # Wake server
193
- yield log(f"Connecting to {DEFAULT_SERVER}..."), None, ""
194
  status = wake_hf_space(DEFAULT_SERVER)
195
- yield log(status), None, ""
196
-
197
- # Track per-game progress
198
- game_log = []
199
-
200
- def on_game_done(game_num, total, metrics):
201
- result = metrics["result"] or "timeout"
202
- kd = metrics["kd_ratio"]
203
- game_log.append({
204
- "Game": game_num,
205
- "Result": result,
206
- "K/D": round(kd, 1),
207
- "Ticks": metrics["ticks"],
208
- })
209
-
210
- yield log(f"Running {num_games} game(s) vs {opponent} AI..."), None, ""
211
 
212
  try:
213
- results = asyncio.run(
214
- run_evaluation(
215
- agent_name=agent_name,
216
- opponent=opponent,
217
- num_games=num_games,
218
- server_url=DEFAULT_SERVER,
219
- on_game_done=on_game_done,
220
- )
221
- )
222
- except Exception as e:
223
- yield log(f"Error: {e}"), None, ""
224
- return
225
-
226
- # Save results
227
- save_submission(results)
228
-
229
- # Format output
230
- for g in game_log:
231
- log(f" Game {g['Game']}: {g['Result']} (K/D: {g['K/D']}, ticks: {g['Ticks']})")
232
-
233
- log(f"\nEvaluation complete!")
234
-
235
- summary = (
236
- f"### Results: {agent_name}\n\n"
237
- f"| Metric | Value |\n|--------|-------|\n"
238
- f"| **Score** | **{results['score']}** |\n"
239
- f"| Win Rate | {results['win_rate']}% |\n"
240
- f"| K/D Ratio | {results['kd_ratio']} |\n"
241
- f"| Avg Economy | {results['avg_economy']} |\n"
242
- f"| Games | {results['games']} vs {results['opponent']} |\n"
243
- )
244
-
245
- results_df = pd.DataFrame([{
246
- "Agent": results["agent_name"],
247
- "Type": results["agent_type"],
248
- "Opponent": results["opponent"],
249
- "Games": results["games"],
250
- "Win Rate (%)": results["win_rate"],
251
- "Score": results["score"],
252
- "K/D Ratio": results["kd_ratio"],
253
- }])
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
254
 
255
- yield "\n".join(log_lines), results_df, summary
 
 
 
256
 
257
 
258
  # โ”€โ”€ UI โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
@@ -300,10 +333,10 @@ The benchmark score combines three components:
300
  SUBMIT_MD = """
301
  ## How to Submit Results
302
 
303
- ### Option A: In-Browser (no setup needed)
304
 
305
- Use the **Evaluate** tab to run a scripted agent directly from your browser.
306
- Results are saved to the leaderboard automatically.
307
 
308
  ### Option B: CLI with HuggingFace-hosted server (no Docker needed)
309
 
@@ -439,50 +472,39 @@ def build_app() -> gr.Blocks:
439
  outputs=leaderboard,
440
  )
441
 
442
- # โ”€โ”€ Evaluate Tab โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
443
- with gr.Tab("Evaluate"):
444
  gr.Markdown(
445
- "## Run Evaluation\n\n"
446
- "Run a scripted agent against the HuggingFace-hosted "
447
- "OpenRA-RL environment. No Docker or local setup needed."
 
448
  )
449
  with gr.Row():
450
- eval_name = gr.Textbox(
451
- label="Agent Name",
452
- placeholder="e.g. MyBot-v1",
453
- scale=2,
454
- )
455
- eval_opponent = gr.Dropdown(
456
  choices=["Easy", "Normal", "Hard"],
457
  value="Normal",
458
- label="Opponent",
459
  scale=1,
460
  )
461
- eval_games = gr.Slider(
462
- minimum=1,
463
- maximum=20,
464
- value=3,
465
- step=1,
466
- label="Games",
467
  scale=1,
468
  )
469
- eval_btn = gr.Button("Run Evaluation", variant="primary")
470
 
471
- eval_log = gr.Textbox(
472
- label="Progress",
473
- lines=10,
474
- interactive=False,
475
- )
476
- eval_results = gr.Dataframe(
477
- label="Game Results",
478
  interactive=False,
 
479
  )
480
- eval_summary = gr.Markdown()
481
 
482
- eval_btn.click(
483
- fn=run_eval_sync,
484
- inputs=[eval_name, eval_opponent, eval_games],
485
- outputs=[eval_log, eval_results, eval_summary],
486
  )
487
 
488
  # โ”€โ”€ About Tab โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
 
10
  Push app.py, requirements.txt, data/, and README.md to your HF Space.
11
  """
12
 
 
13
  import csv
14
  import json
15
  import os
 
17
  from pathlib import Path
18
 
19
  import gradio as gr
20
+ import httpx
21
  import pandas as pd
22
 
23
+ from evaluate_runner import DEFAULT_SERVER, wake_hf_space
24
 
25
  # โ”€โ”€ Data Loading โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
26
 
 
171
  writer.writerow(results)
172
 
173
 
174
+ # โ”€โ”€ Try Agent Handler โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
175
 
176
 
177
+ def run_try_agent(opponent: str):
178
+ """Generator that streams LLM agent gameplay from the OpenRA-RL server."""
 
 
 
 
 
 
 
179
  log_lines = []
180
 
181
  def log(msg: str):
182
  log_lines.append(msg)
183
  return "\n".join(log_lines)
184
 
185
+ # Wake server first
186
+ yield log(f"Connecting to {DEFAULT_SERVER}..."), ""
187
  status = wake_hf_space(DEFAULT_SERVER)
188
+ yield log(status), ""
189
+ yield log(f"Starting game โ€” LLM agent vs {opponent} AI..."), ""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
190
 
191
  try:
192
+ with httpx.stream(
193
+ "GET",
194
+ f"{DEFAULT_SERVER}/try-agent",
195
+ params={"opponent": opponent},
196
+ timeout=httpx.Timeout(connect=30.0, read=360.0, write=30.0, pool=30.0),
197
+ ) as resp:
198
+ if resp.status_code == 409:
199
+ yield log("A game is already in progress. Please try again later."), ""
200
+ return
201
+ if resp.status_code != 200:
202
+ yield log(f"Error: Server returned {resp.status_code}"), ""
203
+ return
204
+
205
+ final_data = None
206
+ event_type = ""
207
+
208
+ for line in resp.iter_lines():
209
+ if not line.strip():
210
+ continue
211
+
212
+ # Parse SSE: event line sets type, data line has payload
213
+ if line.startswith("event: "):
214
+ event_type = line[7:].strip()
215
+ continue
216
+ if not line.startswith("data: "):
217
+ continue
218
+
219
+ try:
220
+ data = json.loads(line[6:])
221
+ except json.JSONDecodeError:
222
+ continue
223
+
224
+ etype = event_type or data.get("type", "")
225
+
226
+ if etype == "status":
227
+ yield log(data["message"]), ""
228
+
229
+ elif etype == "turn":
230
+ yield log(
231
+ f"[Turn {data['turn']}] "
232
+ f"API calls: {data['api_calls']} | "
233
+ f"Elapsed: {data['elapsed']}s"
234
+ ), ""
235
+
236
+ elif etype == "llm":
237
+ content = data.get("content", "")
238
+ if content:
239
+ # Truncate long LLM reasoning for display
240
+ display = content[:300] + "..." if len(content) > 300 else content
241
+ yield log(f" AI: {display}"), ""
242
+
243
+ elif etype == "tool_call":
244
+ yield log(f" >> {data['name']}({data.get('args', '')})"), ""
245
+
246
+ elif etype == "game_state":
247
+ yield log(
248
+ f" State: tick={data.get('tick', '?')} "
249
+ f"units={data.get('units', '?')} "
250
+ f"buildings={data.get('buildings', '?')} "
251
+ f"cash=${data.get('cash', '?')}"
252
+ ), ""
253
+
254
+ elif etype == "done":
255
+ result = data.get("result", "?").upper()
256
+ yield log(f"\nGAME OVER: {result} (tick {data.get('tick', '?')})"), ""
257
+
258
+ elif etype == "final":
259
+ final_data = data
260
+
261
+ elif etype == "error":
262
+ yield log(f"Error: {data.get('message', 'Unknown error')}"), ""
263
+
264
+ # Show final scorecard
265
+ if final_data:
266
+ result = final_data.get("result", "ongoing").upper()
267
+ summary = (
268
+ f"### Game Result: {result}\n\n"
269
+ f"| Metric | Value |\n|--------|-------|\n"
270
+ f"| Result | **{result}** |\n"
271
+ f"| Ticks | {final_data.get('tick', '?')} |\n"
272
+ f"| LLM Turns | {final_data.get('turns', '?')} |\n"
273
+ f"| Tool Calls | {final_data.get('tool_calls', '?')} |\n"
274
+ f"| Duration | {final_data.get('elapsed', '?')}s |\n"
275
+ f"| Units Killed | {final_data.get('units_killed', 0)} |\n"
276
+ f"| Units Lost | {final_data.get('units_lost', 0)} |\n"
277
+ f"| Kill Value | ${final_data.get('kills_cost', 0)} |\n"
278
+ f"| Death Value | ${final_data.get('deaths_cost', 0)} |\n"
279
+ f"| Cash | ${final_data.get('cash', 0)} |\n"
280
+ )
281
+ yield "\n".join(log_lines), summary
282
+ else:
283
+ yield "\n".join(log_lines), ""
284
 
285
+ except httpx.ReadTimeout:
286
+ yield log("Connection timed out. The game may still be running on the server."), ""
287
+ except Exception as e:
288
+ yield log(f"Error: {e}"), ""
289
 
290
 
291
  # โ”€โ”€ UI โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
 
333
  SUBMIT_MD = """
334
  ## How to Submit Results
335
 
336
+ ### Option A: Watch AI Play (no setup needed)
337
 
338
+ Use the **Try** tab to watch a pre-configured LLM agent play Red Alert
339
+ directly in your browser. No API keys or setup required.
340
 
341
  ### Option B: CLI with HuggingFace-hosted server (no Docker needed)
342
 
 
472
  outputs=leaderboard,
473
  )
474
 
475
+ # โ”€โ”€ Try Tab โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
476
+ with gr.Tab("Try"):
477
  gr.Markdown(
478
+ "## Watch AI Play Red Alert\n\n"
479
+ "Watch a pre-configured LLM agent play a game of Red Alert "
480
+ "against the built-in AI. No setup needed โ€” just pick a "
481
+ "difficulty and click play."
482
  )
483
  with gr.Row():
484
+ try_opponent = gr.Dropdown(
 
 
 
 
 
485
  choices=["Easy", "Normal", "Hard"],
486
  value="Normal",
487
+ label="Opponent Difficulty",
488
  scale=1,
489
  )
490
+ try_btn = gr.Button(
491
+ "Watch AI Play",
492
+ variant="primary",
 
 
 
493
  scale=1,
494
  )
 
495
 
496
+ try_log = gr.Textbox(
497
+ label="Live Game Log",
498
+ lines=18,
 
 
 
 
499
  interactive=False,
500
+ show_copy_button=True,
501
  )
502
+ try_summary = gr.Markdown()
503
 
504
+ try_btn.click(
505
+ fn=run_try_agent,
506
+ inputs=[try_opponent],
507
+ outputs=[try_log, try_summary],
508
  )
509
 
510
  # โ”€โ”€ About Tab โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€