scripts/check_aide_status.py

"""Print training status as KEY=VALUE lines. Runs on the GCP VM."""
import json
import glob
import os
import re
import subprocess

# Process detection - check for both AIDE and direct training
aide_procs = int(os.popen('ps aux | grep -E "run_aide" | grep -v grep | wc -l').read().strip())
train_procs = int(os.popen('ps aux | grep -E "train_100ep|train_optimal|train_fulldata" | grep -v grep | wc -l').read().strip())
print(f"AIDE_RUNNING={aide_procs + train_procs}")

# Determine mode: direct training or AIDE
mode = "unknown"
log_file = None

# Priority 1: Full-data training (train_full.log)
if os.path.exists(os.path.expanduser("~/train_full.log")) and os.path.getsize(os.path.expanduser("~/train_full.log")) > 0:
    log_file = os.path.expanduser("~/train_full.log")
    mode = "direct"
# Priority 2: Direct training (train_100ep.log)
elif os.path.exists(os.path.expanduser("~/train_100ep.log")) and os.path.getsize(os.path.expanduser("~/train_100ep.log")) > 0:
    log_file = os.path.expanduser("~/train_100ep.log")
    mode = "direct"
# Priority 2: Optimal training
elif os.path.exists(os.path.expanduser("~/optimal_train.log")) and os.path.getsize(os.path.expanduser("~/optimal_train.log")) > 0:
    log_file = os.path.expanduser("~/optimal_train.log")
    mode = "direct"
elif os.path.exists(os.path.expanduser("~/optimal_train2.log")) and os.path.getsize(os.path.expanduser("~/optimal_train2.log")) > 0:
    log_file = os.path.expanduser("~/optimal_train2.log")
    mode = "direct"
# Priority 3: AIDE
elif os.path.exists(os.path.expanduser("~/aide-run.log")) and os.path.getsize(os.path.expanduser("~/aide-run.log")) > 0:
    log_file = os.path.expanduser("~/aide-run.log")
    mode = "aide"

print(f"MODE={mode}")

if log_file:
    content = open(log_file).read()
    lines = content.strip().split("\n")
    print(f"LOG_LINES={len(lines)}")
    print(f"LOG_TAIL={lines[-1].strip()[:80] if lines else 'empty'}")

    if mode == "direct":
        # Parse epoch progress from ultralytics output: "31/100  8.1G ..."
        # Match lines like "     31/100      8.1G" - epoch/total at start of training line
        epoch_pattern = re.findall(r'^\s*(\d+)/(\d+)\s+[\d.]+G', content, re.MULTILINE)

        if epoch_pattern:
            cur, total = int(epoch_pattern[-1][0]), int(epoch_pattern[-1][1])
            print(f"EPOCH={cur}/{total}")
            print(f"STEP={cur}")
            print(f"BEST_SCORE={cur/total:.4f}")  # Progress as proxy
            print(f"BUGGY=0")

            # Check if training completed
            if "Done!" in content or cur >= total:
                print(f"TRAINING_DONE=1")
            else:
                print(f"TRAINING_DONE=0")
        else:
            print(f"EPOCH=0/100")
            print(f"STEP=0")
            print(f"BEST_SCORE=0.0000")
            print(f"BUGGY=0")
            print(f"TRAINING_DONE=0")

        # Show last few log lines
        for line in lines[-3:]:
            print(f"LOG_LINE={line.strip()[:90]}")

        # Find trained model weights
        for wdir in glob.glob(os.path.expanduser("~/aide/train_*/run/weights/best.pt")):
            print(f"TRAINED_MODEL={wdir}")
        for wdir in glob.glob(os.path.expanduser("~/aide/working/*/weights/best.pt")):
            print(f"TRAINED_MODEL={wdir}")

    elif mode == "aide":
        # AIDE mode - check journal
        files = glob.glob(os.path.expanduser("~/aide/logs/*/journal.json"))
        if files:
            j = json.load(open(files[0]))
            nodes = j.get("nodes", [])
            step = len(nodes)
            best = 0
            buggy = sum(1 for n in nodes if n.get("is_buggy", False))
            for n in nodes:
                m = n.get("metric", {})
                v = m.get("value") if isinstance(m, dict) else m
                if v is not None and isinstance(v, (int, float)) and v > best:
                    best = v
            print(f"STEP={step}")
            print(f"BEST_SCORE={best:.4f}")
            print(f"BUGGY={buggy}")
        else:
            print("STEP=0")
            print("BEST_SCORE=0.0000")
            print("BUGGY=0")
else:
    print("LOG_LINES=0")
    print("LOG_TAIL=empty")
    print("STEP=0")
    print("BEST_SCORE=0.0000")
    print("BUGGY=0")

# GPU
try:
    gpu_out = subprocess.check_output(
        ["nvidia-smi", "--query-gpu=utilization.gpu,memory.used,memory.total",
         "--format=csv,noheader,nounits"], text=True
    ).strip().split(", ")
    print(f"GPU_UTIL={gpu_out[0]}")
    print(f"GPU_MEM_USED={gpu_out[1]}")
    print(f"GPU_MEM_TOTAL={gpu_out[2]}")
except Exception:
    pass