#!/usr/bin/env python3
"""Simple status server for the overnight ablation run.
Serves monitor dashboards for whichever training is currently active.
"""
import http.server
import json
import os
import time
VAP_DIR = "/home/deployer/laion/Voice-Acting-Pipeline"
ABLATIONS = [
("A_nat_quality", "finetune_output/ablation_nat_quality", 8768),
("B_nat_centroid", "finetune_output/ablation_nat_centroid", 8769),
("C_nat_quality_speaker", "finetune_output/ablation_nat_quality_speaker", 8770),
("D_nat_speaker", "finetune_output/ablation_nat_speaker", 8771),
]
class StatusHandler(http.server.BaseHTTPRequestHandler):
def do_GET(self):
if self.path == "/" or self.path == "/status":
self.send_response(200)
self.send_header("Content-Type", "text/html")
self.end_headers()
html = """
Ablation Study Status
Ablation Study — Overnight Runner
Auto-refreshes every 30s. """ + time.strftime("%Y-%m-%d %H:%M:%S UTC") + """
"""
for name, output_dir, port in ABLATIONS:
status_path = os.path.join(VAP_DIR, output_dir, "status.json")
metrics_path = os.path.join(VAP_DIR, output_dir, "metrics.jsonl")
if os.path.exists(status_path):
try:
with open(status_path) as f:
st = json.load(f)
step = st.get("step", 0)
total = st.get("total_steps", 234)
loss = st.get("flow_loss", st.get("loss", "?"))
best = st.get("best_loss", "?")
elapsed = st.get("elapsed_sec", 0)
eta = st.get("eta_sec", 0)
pct = int(100 * step / total) if total else 0
if step >= total:
cls = "done"
status_text = f"COMPLETED — {step}/{total} steps in {elapsed/60:.0f}min"
else:
age = time.time() - os.path.getmtime(status_path)
if age < 120:
cls = "active"
status_text = f"RUNNING — {step}/{total} ({pct}%) — ETA {eta/60:.0f}min"
else:
cls = "done"
status_text = f"COMPLETED — {step}/{total} steps"
metrics_text = f"flow_loss={loss} best_loss={best} elapsed={elapsed/60:.0f}min"
# Get last naturalness from metrics
if os.path.exists(metrics_path):
try:
with open(metrics_path) as mf:
lines = mf.readlines()
if lines:
last = json.loads(lines[-1])
nat = last.get("naturalness_reward")
if nat is not None:
metrics_text += f" nat={nat:.4f}"
except Exception:
pass
html += f"""
{name}
{status_text}
{metrics_text}
"""
except Exception as e:
html += f'{name}
Error reading status: {e}
{name}
PENDING — not started yet
Evaluation
'
try:
with open(eval_log) as f:
lines = f.readlines()
last_lines = lines[-5:] if lines else ["No output yet"]
html += '
' + "
".join(l.strip() for l in last_lines) + '
'
except Exception:
html += '
Running...
'
html += '
Overall Log (last 10 lines)
'
try:
with open("/tmp/ablation_overnight.log") as f:
lines = f.readlines()
for line in lines[-10:]:
html += line.strip() + "
"
except Exception:
html += "Log not available yet"
html += '