Harden live terminal logging and production runtime reliability (heartbeat + download retries).

README.md

@@ -43,6 +43,7 @@ If no token is available, public dataset training still works and push is automa
 - `Enforce Quality Gate`: enables/disables promotion gate checks.
 - `Gate Min pass@1`, `Gate Min pass@k`, `Gate Min Rows`: runtime gate thresholds.
 - `Live Tactical Telemetry`: real-time stage progression, runtime posture, loss sparkline, and gate/push state.
+- `Runtime Terminal Log (Live)`: line-by-line subprocess stream with heartbeat events during silent phases.
 - `Preflight Only (No Training)`: validates pipeline with `--dry-run`.
 - `Force Dataset Redownload`: bypasses cached parquet files.
 - `Abort Active Run`: cancels active subprocess tree.

app.py

@@ -15,6 +15,7 @@ import signal
 import subprocess
 import sys
 import threading
+import time
 from pathlib import Path
 from typing import Any, Dict, Generator, List, Optional, Tuple
 
@@ -721,15 +722,30 @@ def download_dataset(
             out_files[split] = str(out_path)
             continue
 
-        cached_path = hf_hub_download(
-            repo_id=dataset_repo_id,
-            repo_type="dataset",
-            filename=f"{split}.parquet",
-            token=token,
-            force_download=force_redownload,
-        )
-        shutil.copy2(cached_path, out_path)
-        append_log(log_lines, f"Downloaded {split}.parquet to {out_path}")
+        download_attempts = 3
+        for attempt in range(1, download_attempts + 1):
+            try:
+                cached_path = hf_hub_download(
+                    repo_id=dataset_repo_id,
+                    repo_type="dataset",
+                    filename=f"{split}.parquet",
+                    token=token,
+                    force_download=force_redownload,
+                )
+                shutil.copy2(cached_path, out_path)
+                append_log(log_lines, f"Downloaded {split}.parquet to {out_path}")
+                break
+            except Exception as exc:
+                if attempt >= download_attempts:
+                    raise RuntimeError(
+                        f"Failed downloading {split}.parquet after {download_attempts} attempts: {exc}"
+                    ) from exc
+                wait_s = 2 ** attempt
+                append_log(
+                    log_lines,
+                    f"Download retry {attempt}/{download_attempts - 1} for {split}.parquet after error: {exc}",
+                )
+                time.sleep(wait_s)
         out_files[split] = str(out_path)
     return out_files["train"], out_files["validation"], out_files["test"]
 
@@ -799,6 +815,8 @@ def stream_subprocess(
     set_active_process(proc)
     cancelled = False
     ret = 1
+    last_heartbeat = time.monotonic()
+    heartbeat_interval_s = 20.0
     assert proc.stdout is not None
     try:
         while True:
@@ -819,7 +837,14 @@ def stream_subprocess(
                     line = line.rstrip()
                     if line:
                         append_log(log_lines, line)
+                        last_heartbeat = time.monotonic()
                         yield "\n".join(log_lines), f"{status_prefix}: running"
+            elif proc.poll() is None:
+                now = time.monotonic()
+                if now - last_heartbeat >= heartbeat_interval_s:
+                    append_log(log_lines, f"{status_prefix} heartbeat: process alive, waiting for next log chunk.")
+                    last_heartbeat = now
+                    yield "\n".join(log_lines), f"{status_prefix}: running"
 
             if proc.poll() is not None:
                 break
@@ -1330,7 +1355,7 @@ with gr.Blocks(title="Math Conjecture Trainer Space") as demo:
 
     ops_visual = gr.HTML(value=render_ops_visual({}, "Idle", ""))
     status = gr.Textbox(label="Run Status", value="Idle", interactive=False)
-    logs = make_copyable_textbox(label="Telemetry Log", lines=24, max_lines=30, interactive=False)
+    logs = make_copyable_textbox(label="Runtime Terminal Log (Live)", lines=24, max_lines=30, interactive=False)
     run_summary = make_copyable_textbox(
         label="Mission Summary (JSON)",
         lines=16,