Prepare job build context

deploy.sh

@@ -85,9 +85,10 @@ JOB_ID=$(echo "${JOB_OUTPUT}" | awk '/Job ID:/ {print $3}')
 if [ -z "${JOB_ID}" ]; then
     echo "Warning: Could not parse HF job id. See /tmp/hf_image_job.log"
 else
-    echo "Following job logs for 1 minute..."
+    echo "Following job logs until completion..."
     if [ -x "${HF_BIN}" ]; then
         HF_BIN_PATH="${HF_BIN}" JOB_ID="${JOB_ID}" "${PY_BIN}" - <<'PY'
+import json
 import os
 import subprocess
 import sys
@@ -96,15 +97,27 @@ import time
 hf = os.environ["HF_BIN_PATH"]
 job_id = os.environ["JOB_ID"]
 
-proc = subprocess.Popen([hf, "jobs", "logs", job_id], stdout=sys.stdout, stderr=sys.stderr)
+log_proc = subprocess.Popen([hf, "jobs", "logs", job_id], stdout=sys.stdout, stderr=sys.stderr)
 try:
-    time.sleep(60)
+    while True:
+        inspect = subprocess.run([hf, "jobs", "inspect", job_id], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True)
+        if inspect.returncode == 0:
+            try:
+                data = json.loads(inspect.stdout)[0]
+                stage = (data.get("status") or {}).get("stage", "UNKNOWN")
+            except Exception:
+                stage = "UNKNOWN"
+        else:
+            stage = "UNKNOWN"
+        if stage in {"SUCCESS","SUCCEEDED","COMPLETED","DONE","FAILED","ERROR","CANCELED","CANCELLED"}:
+            break
+        time.sleep(15)
 finally:
-    proc.terminate()
+    log_proc.terminate()
     try:
-        proc.wait(timeout=5)
+        log_proc.wait(timeout=5)
     except Exception:
-        proc.kill()
+        log_proc.kill()
 PY
         echo ""
         echo "Job status:"
@@ -154,9 +167,10 @@ JOB_ID_L2=$(echo "${JOB_OUTPUT_L2}" | awk '/Job ID:/ {print $3}')
 if [ -z "${JOB_ID_L2}" ]; then
     echo "Warning: Could not parse HF job id for L2. See /tmp/hf_image_job_l2.log"
 else
-    echo "Following L2 job logs for 1 minute..."
+    echo "Following L2 job logs until completion..."
     if [ -x "${HF_BIN}" ]; then
         HF_BIN_PATH="${HF_BIN}" JOB_ID="${JOB_ID_L2}" "${PY_BIN}" - <<'PY'
+import json
 import os
 import subprocess
 import sys
@@ -165,15 +179,27 @@ import time
 hf = os.environ["HF_BIN_PATH"]
 job_id = os.environ["JOB_ID"]
 
-proc = subprocess.Popen([hf, "jobs", "logs", job_id], stdout=sys.stdout, stderr=sys.stderr)
+log_proc = subprocess.Popen([hf, "jobs", "logs", job_id], stdout=sys.stdout, stderr=sys.stderr)
 try:
-    time.sleep(60)
+    while True:
+        inspect = subprocess.run([hf, "jobs", "inspect", job_id], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True)
+        if inspect.returncode == 0:
+            try:
+                data = json.loads(inspect.stdout)[0]
+                stage = (data.get("status") or {}).get("stage", "UNKNOWN")
+            except Exception:
+                stage = "UNKNOWN"
+        else:
+            stage = "UNKNOWN"
+        if stage in {"SUCCESS","SUCCEEDED","COMPLETED","DONE","FAILED","ERROR","CANCELED","CANCELLED"}:
+            break
+        time.sleep(15)
 finally:
-    proc.terminate()
+    log_proc.terminate()
     try:
-        proc.wait(timeout=5)
+        log_proc.wait(timeout=5)
     except Exception:
-        proc.kill()
+        log_proc.kill()
 PY
         echo ""
         echo "L2 job status:"