Upload train_monitor.py with huggingface_hub

train_monitor.py

@@ -0,0 +1,129 @@
+#!/usr/bin/env python3
+"""FSI_FELON Training Monitor — watches over the forge while you sleep."""
+import os, time, json, sys, subprocess
+from datetime import datetime
+
+LOG = "/tmp/opencode/snca/train_live.log"
+SCRIPT = "/tmp/opencode/snca/train_live.py"
+SCREEN = "deep_train"
+CHECK_INTERVAL = 600  # 10 min (~100 steps)
+REPORT = "/tmp/fsi_felon/training_report.json"
+ALERT_FILE = "/tmp/fsi_felon/training_alert"
+
+def tail_lines(path, n=5):
+    try:
+        with open(path) as f:
+            lines = f.readlines()
+        return lines[-n:]
+    except:
+        return []
+
+def check_training():
+    now = datetime.now().strftime("%H:%M:%S")
+    lines = tail_lines(LOG, 3)
+    if not lines:
+        return {"status": "NO_LOG", "time": now, "msg": "No log file found"}
+
+    last = lines[-1].strip()
+    # Parse: [03:14:59] step=1,457 loss=7.3952 lr=3.00e-04
+    try:
+        parts = last.split()
+        time_str = parts[0].strip("[]")
+        step = int(parts[1].split("=")[1].replace(",", ""))
+        loss = float(parts[2].split("=")[1])
+        lr = float(parts[3].split("=")[1])
+    except:
+        return {"status": "PARSE_ERROR", "time": now, "msg": f"Could not parse: {last}"}
+
+    report = {
+        "status": "OK",
+        "time": now,
+        "step": step,
+        "loss": loss,
+        "lr": lr,
+        "msg": f"step={step:,} loss={loss:.4f} lr={lr:.2e}"
+    }
+
+    # Check for anomalies
+    if loss > 15.0:
+        report["alert"] = f"Loss spike: {loss:.4f}"
+        report["status"] = "ALERT"
+        with open(ALERT_FILE, "w") as f:
+            f.write(json.dumps(report))
+
+    # Check if training stopped (log not updating)
+    if os.path.exists(REPORT):
+        try:
+            with open(REPORT) as f:
+                prev = json.load(f)
+            if prev.get("step") == step and prev.get("status") != "RESTARTED":
+                report["status"] = "STALLED"
+                report["msg"] += " — LOG NOT UPDATING"
+        except:
+            pass
+
+    with open(REPORT, "w") as f:
+        json.dump(report, f, indent=2)
+    return report
+
+def restart_training():
+    print(f"[{datetime.now().strftime('%H:%M:%S')}] Training stalled or crashed — restarting...")
+    # Kill existing screen
+    subprocess.run(["screen", "-S", SCREEN, "-X", "quit"], capture_output=True)
+    time.sleep(2)
+    # Start fresh
+    cmd = f"cd /tmp/opencode/snca && screen -dmS {SCREEN} python3 train_live.py"
+    subprocess.run(cmd, shell=True)
+    time.sleep(5)
+    report = check_training()
+    report["status"] = "RESTARTED"
+    with open(REPORT, "w") as f:
+        json.dump(report, f, indent=2)
+    print(f"  Restarted at step {report.get('step', '?')}")
+    return report
+
+def monitor():
+    print("=" * 55)
+    print("  FSI_FELON TRAINING MONITOR")
+    print("  Checking every 10 minutes (~100 steps)")
+    print(f"  Started: {datetime.now().strftime('%H:%M:%S')}")
+    print("=" * 55)
+    print()
+
+    # Clear any previous alert
+    if os.path.exists(ALERT_FILE):
+        os.remove(ALERT_FILE)
+
+    cycle = 0
+    while True:
+        cycle += 1
+        report = check_training()
+        ts = report["time"]
+        status = report["status"]
+
+        if status == "OK":
+            print(f"[{ts}] ✓ step={report['step']:,} loss={report['loss']:.4f} lr={report['lr']:.2e}")
+        elif status == "STALLED":
+            print(f"[{ts}] ⚠ STALLED — restarting")
+            report = restart_training()
+        elif status == "ALERT":
+            print(f"[{ts}] 🔴 ALERT: {report.get('alert')}")
+        elif status == "NO_LOG":
+            print(f"[{ts}] 🔴 No log — restarting")
+            report = restart_training()
+        else:
+            print(f"[{ts}] ? {status}: {report.get('msg')}")
+
+        # Check screen is alive
+        r = subprocess.run(["screen", "-ls"], capture_output=True, text=True)
+        if SCREEN not in r.stdout:
+            print(f"[{datetime.now().strftime('%H:%M:%S')}] 🔴 Screen dead — restarting")
+            restart_training()
+
+        time.sleep(CHECK_INTERVAL)
+
+if __name__ == "__main__":
+    try:
+        monitor()
+    except KeyboardInterrupt:
+        print("\nMonitor stopped.")