Spaces:
Sleeping
Sleeping
Fix: start_training yield count (9), plt.close memory leak
Browse files
app.py
CHANGED
|
@@ -15,11 +15,13 @@ GRPO_DIR = '/app/runs/grpo'
|
|
| 15 |
|
| 16 |
# ββ Training process ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 17 |
|
|
|
|
|
|
|
| 18 |
def start_training(model_choice, epochs, num_gen, hf_token):
|
| 19 |
global _proc
|
| 20 |
with _lock:
|
| 21 |
if _proc and _proc.poll() is None:
|
| 22 |
-
yield gr.update(
|
| 23 |
return
|
| 24 |
|
| 25 |
env = os.environ.copy()
|
|
@@ -45,8 +47,7 @@ def start_training(model_choice, epochs, num_gen, hf_token):
|
|
| 45 |
output = ''
|
| 46 |
for line in _proc.stdout:
|
| 47 |
output += line
|
| 48 |
-
|
| 49 |
-
yield gr.update(value=output), gr.update(), gr.update(), gr.update()
|
| 50 |
|
| 51 |
_proc.wait()
|
| 52 |
output += f"\n{'='*60}\nDone (exit {_proc.returncode})"
|
|
@@ -445,16 +446,11 @@ def plot_ep_lengths(m):
|
|
| 445 |
return fig
|
| 446 |
|
| 447 |
def build_plots():
|
| 448 |
-
|
| 449 |
-
|
| 450 |
-
m = {}
|
| 451 |
return (plot_reward_curve(m), plot_loss_curve(m), plot_format_rate(m),
|
| 452 |
-
|
| 453 |
-
|
| 454 |
-
|
| 455 |
-
|
| 456 |
-
def refresh_plots():
|
| 457 |
-
return build_plots()
|
| 458 |
|
| 459 |
|
| 460 |
# ββ UI βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
@@ -560,7 +556,7 @@ with gr.Blocks(title='AP Commander Training', theme=gr.themes.Base()) as demo:
|
|
| 560 |
start_btn.click(
|
| 561 |
fn=start_training,
|
| 562 |
inputs=[model_dd, epochs_sl, numgen_sl, hf_token],
|
| 563 |
-
outputs=[logs, plot_reward, plot_loss, plot_fmt, plot_diff, plot_eplen, plot_pie, plot_tasks],
|
| 564 |
)
|
| 565 |
|
| 566 |
eval_btn.click(
|
|
|
|
| 15 |
|
| 16 |
# ββ Training process ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 17 |
|
| 18 |
+
_NO_PLOT = [gr.update()] * 8 # 8 plot slots (reward,loss,fmt,diff,eplen,pie,tasks,stats)
|
| 19 |
+
|
| 20 |
def start_training(model_choice, epochs, num_gen, hf_token):
|
| 21 |
global _proc
|
| 22 |
with _lock:
|
| 23 |
if _proc and _proc.poll() is None:
|
| 24 |
+
yield gr.update(value="Already running."), *_NO_PLOT
|
| 25 |
return
|
| 26 |
|
| 27 |
env = os.environ.copy()
|
|
|
|
| 47 |
output = ''
|
| 48 |
for line in _proc.stdout:
|
| 49 |
output += line
|
| 50 |
+
yield gr.update(value=output), *_NO_PLOT
|
|
|
|
| 51 |
|
| 52 |
_proc.wait()
|
| 53 |
output += f"\n{'='*60}\nDone (exit {_proc.returncode})"
|
|
|
|
| 446 |
return fig
|
| 447 |
|
| 448 |
def build_plots():
|
| 449 |
+
plt.close('all') # close all open figures before creating new ones
|
| 450 |
+
m = _load_metrics() or {}
|
|
|
|
| 451 |
return (plot_reward_curve(m), plot_loss_curve(m), plot_format_rate(m),
|
| 452 |
+
plot_diff_curves(m), plot_ep_lengths(m),
|
| 453 |
+
plot_decision_dist(m), plot_task_scores(m), plot_stats_panel(m))
|
|
|
|
|
|
|
|
|
|
|
|
|
| 454 |
|
| 455 |
|
| 456 |
# ββ UI βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
|
|
| 556 |
start_btn.click(
|
| 557 |
fn=start_training,
|
| 558 |
inputs=[model_dd, epochs_sl, numgen_sl, hf_token],
|
| 559 |
+
outputs=[logs, plot_reward, plot_loss, plot_fmt, plot_diff, plot_eplen, plot_pie, plot_tasks, plot_stats],
|
| 560 |
)
|
| 561 |
|
| 562 |
eval_btn.click(
|