File size: 1,252 Bytes
a10ba7f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 | import subprocess
import sys
import time
import os
# Toàn bộ danh sách participants từ p00 đến p14
ALL_PARTICIPANTS = [f"p{i:02d}" for i in range(15)]
# Danh sách đã hoàn thành (Cập nhật sau khi dừng)
COMPLETED = ["p00", "p01", "p02", "p03", "p11"]
TODO = [p for p in ALL_PARTICIPANTS if p not in COMPLETED]
print(f"{'='*60}")
print(f" RESUMING 8x8 ABLATION BATCH TRAINING ")
print(f" Participants remaining: {TODO} ")
print(f"{'='*60}")
start_total = time.time()
for participant in TODO:
print(f"\n{'#'*40}")
print(f" TRAINING 8x8 FOR: {participant} ")
print(f"{'#'*40}")
cmd = [
sys.executable, "src/train_8x8_ablation.py",
"--participant", participant,
"--epochs", "50",
"--workers", "4"
]
start_fold = time.time()
result = subprocess.run(cmd)
end_fold = time.time()
if result.returncode == 0:
print(f"SUCCESS: {participant} 8x8 training completed in {(end_fold - start_fold)/60:.2f} mins.")
else:
print(f"FAILED: {participant} 8x8 training with exit code {result.returncode}")
end_total = time.time()
print(f"\n{'='*60}")
print(f" 8x8 BATCH COMPLETED in {(end_total - start_total)/3600:.2f} hours ")
print(f"{'='*60}")
|