File size: 2,741 Bytes
bdce880 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 | """Live training-loss plot. Reads each run's progress.jsonl and redraws a PNG
every ~20 s (atomic write, so VSCode's image preview never catches a half-file).
Open the PNG in VSCode and it auto-refreshes.
python ezflow_v3/gnn/plot_loss_live.py # loop, updates every 20 s
python ezflow_v3/gnn/plot_loss_live.py --once # single draw (test)
"""
import json, os, sys, time
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
RUNS = r"C:\dev\EZFlow\ezflow_v3\gnn\_runs"
OUT = r"C:\dev\ezflow_eval\live_loss.png"
# name, run-tag, color, linewidth, alpha (the live v3 run drawn boldest)
SERIES = [("v3 GeoReNet s0 (live)", "rans_v5_s0", "#d62728", 2.6, 1.0)]
LIVE_TAG = "rans_v5_s0"
def load(tag):
p = os.path.join(RUNS, tag, "progress.jsonl")
eps, ls = [], []
if not os.path.exists(p):
return eps, ls
with open(p) as f:
for line in f:
line = line.strip()
if not line:
continue
try:
r = json.loads(line)
eps.append(r["epoch"]); ls.append(r["loss"])
except Exception:
continue # tolerate a partially-written last line
return eps, ls
def draw():
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(13, 5))
cur = None
for name, tag, color, lw, alpha in SERIES:
eps, ls = load(tag)
if not eps:
continue
ax1.plot(eps, ls, color=color, lw=lw, alpha=alpha, label=name)
ax2.plot(eps, ls, color=color, lw=lw, alpha=alpha)
if tag == LIVE_TAG:
cur = (eps[-1], ls[-1])
for ax in (ax1, ax2):
ax.scatter([eps[-1]], [ls[-1]], color=color, zorder=5, s=40)
ax1.set_yscale("log")
ax1.set_title("Train loss — full history (log scale)")
ax2.set_title("Train loss — zoom (linear, < 0.6)")
ax2.set_ylim(0, 0.6)
for ax in (ax1, ax2):
ax.set_xlabel("epoch"); ax.set_ylabel("weighted MSE (train)")
ax.grid(alpha=0.3)
ax1.legend(loc="upper right")
ts = time.strftime("%Y-%m-%d %H:%M:%S")
title = f"EZFlow training loss | updated {ts}"
if cur:
title += f" | {LIVE_TAG} ep{cur[0]} loss={cur[1]:.4f}"
fig.suptitle(title, fontsize=12)
fig.tight_layout(rect=[0, 0, 1, 0.96])
tmp = OUT + ".tmp.png"
fig.savefig(tmp, dpi=110)
plt.close(fig)
os.replace(tmp, OUT) # atomic
if __name__ == "__main__":
once = "--once" in sys.argv
while True:
try:
draw()
print(f"drew {OUT} @ {time.strftime('%H:%M:%S')}", flush=True)
except Exception as e:
print("draw error:", e, flush=True)
if once:
break
time.sleep(20)
|