Upload queue_status.json with huggingface_hub

queue_status.json

@@ -1,10 +1,10 @@
 {
-  "timestamp": "2026-04-26 00:13:14",
+  "timestamp": "2026-04-26 00:15:14",
   "total": 64,
-  "pending": 39,
+  "pending": 35,
   "running": 8,
-  "done": 15,
-  "failed": 2,
+  "done": 16,
+  "failed": 5,
   "stale": 0,
   "retrying": 0,
   "jobs": [
@@ -15,7 +15,7 @@
       "gpu": 0,
       "status": "done",
       "elapsed": 1254,
-      "idle_time": 1668,
+      "idle_time": 1789,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_000_math_qwen2.5-0.5b_math_0pt5pct_gpu0.log"
@@ -27,7 +27,7 @@
       "gpu": 1,
       "status": "done",
       "elapsed": 1401,
-      "idle_time": 1521,
+      "idle_time": 1642,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_001_math_qwen2.5-0.5b_math_0pt5pct_gpu1.log"
@@ -39,7 +39,7 @@
       "gpu": 2,
       "status": "done",
       "elapsed": 1401,
-      "idle_time": 1521,
+      "idle_time": 1642,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_002_math_qwen2.5-0.5b_math_0pt5pct_gpu2.log"
@@ -51,7 +51,7 @@
       "gpu": 3,
       "status": "done",
       "elapsed": 1426,
-      "idle_time": 1496,
+      "idle_time": 1617,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_003_math_qwen2.5-0.5b_math_0pt5pct_gpu3.log"
@@ -63,7 +63,7 @@
       "gpu": 4,
       "status": "done",
       "elapsed": 1366,
-      "idle_time": 1557,
+      "idle_time": 1677,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_004_math_qwen2.5-0.5b_math_0pt5pct_gpu4.log"
@@ -75,7 +75,7 @@
       "gpu": 5,
       "status": "done",
       "elapsed": 1427,
-      "idle_time": 1497,
+      "idle_time": 1617,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_005_math_qwen2.5-0.5b_math_0pt5pct_gpu5.log"
@@ -87,7 +87,7 @@
       "gpu": 6,
       "status": "done",
       "elapsed": 1427,
-      "idle_time": 1496,
+      "idle_time": 1616,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_006_math_qwen2.5-0.5b_math_0pt5pct_gpu6.log"
@@ -99,7 +99,7 @@
       "gpu": 7,
       "status": "done",
       "elapsed": 1442,
-      "idle_time": 1480,
+      "idle_time": 1601,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_007_math_qwen2.5-0.5b_math_0pt5pct_gpu7.log"
@@ -147,7 +147,7 @@
       "gpu": 3,
       "status": "failed",
       "elapsed": 9,
-      "idle_time": 1472,
+      "idle_time": 1592,
       "exit_code": 1,
       "retries": 1,
       "log_file": "/tmp/stride_applications_queue/job_011_math_qwen2.5-0.5b_math_0pt5pct_gpu3.log"
@@ -171,7 +171,7 @@
       "gpu": 1,
       "status": "done",
       "elapsed": 1307,
-      "idle_time": 211,
+      "idle_time": 331,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_013_math_qwen2.5-0.5b_math_0pt5pct_gpu1.log"
@@ -183,7 +183,7 @@
       "gpu": 6,
       "status": "done",
       "elapsed": 1321,
-      "idle_time": 147,
+      "idle_time": 267,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_014_math_qwen2.5-0.5b_math_0pt5pct_gpu6.log"
@@ -195,7 +195,7 @@
       "gpu": 4,
       "status": "done",
       "elapsed": 1415,
-      "idle_time": 137,
+      "idle_time": 258,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_015_math_qwen2.5-0.5b_math_0pt5pct_gpu4.log"
@@ -219,7 +219,7 @@
       "gpu": 0,
       "status": "done",
       "elapsed": 1349,
-      "idle_time": 318,
+      "idle_time": 439,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_017_math_qwen2.5-0.5b_math_0pt5pct_gpu0.log"
@@ -231,7 +231,7 @@
       "gpu": 2,
       "status": "done",
       "elapsed": 1379,
-      "idle_time": 139,
+      "idle_time": 259,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_018_math_qwen2.5-0.5b_math_0pt5pct_gpu2.log"
@@ -243,7 +243,7 @@
       "gpu": 6,
       "status": "failed",
       "elapsed": 11,
-      "idle_time": 1470,
+      "idle_time": 1590,
       "exit_code": 1,
       "retries": 1,
       "log_file": "/tmp/stride_applications_queue/job_019_math_qwen2.5-0.5b_math_0pt5pct_gpu6.log"
@@ -277,10 +277,10 @@
       "name": "math_qwen2.5-0.5b_math_1pct_seed20",
       "cmd": "PYTHONPATH=. /home/ubuntu/venvs/steering/bin/python -m scripts.train_qwen --mode contaminated --rate 0.01 --seed 20 --pr",
       "gpu": 3,
-      "status": "running",
-      "elapsed": 1471,
-      "idle_time": 1,
-      "exit_code": -1,
+      "status": "done",
+      "elapsed": 1472,
+      "idle_time": 121,
+      "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_022_math_qwen2.5-0.5b_math_1pct_se_gpu3.log"
     },
@@ -290,8 +290,8 @@
       "cmd": "PYTHONPATH=. /home/ubuntu/venvs/steering/bin/python -m scripts.train_qwen --mode contaminated --rate 0.01 --seed 21 --pr",
       "gpu": 2,
       "status": "running",
-      "elapsed": 136,
-      "idle_time": 111,
+      "elapsed": 256,
+      "idle_time": 231,
       "exit_code": -1,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_023_math_qwen2.5-0.5b_math_1pct_se_gpu2.log"
@@ -315,7 +315,7 @@
       "gpu": 5,
       "status": "done",
       "elapsed": 1424,
-      "idle_time": 55,
+      "idle_time": 176,
       "exit_code": 0,
       "retries": 1,
       "log_file": "/tmp/stride_applications_queue/job_025_math_qwen2.5-0.5b_math_1pct_se_gpu5.log"
@@ -326,8 +326,8 @@
       "cmd": "PYTHONPATH=. /home/ubuntu/venvs/steering/bin/python -m scripts.train_qwen --mode contaminated --rate 0.01 --seed 24 --pr",
       "gpu": 1,
       "status": "running",
-      "elapsed": 206,
-      "idle_time": 165,
+      "elapsed": 326,
+      "idle_time": 286,
       "exit_code": -1,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_026_math_qwen2.5-0.5b_math_1pct_se_gpu1.log"
@@ -339,7 +339,7 @@
       "gpu": 7,
       "status": "done",
       "elapsed": 1456,
-      "idle_time": 22,
+      "idle_time": 143,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_027_math_qwen2.5-0.5b_math_1pct_se_gpu7.log"
@@ -386,8 +386,8 @@
       "cmd": "PYTHONPATH=. /home/ubuntu/venvs/steering/bin/python -m scripts.train_qwen --mode contaminated --rate 0.01 --seed 29 --pr",
       "gpu": 5,
       "status": "running",
-      "elapsed": 51,
-      "idle_time": 23,
+      "elapsed": 171,
+      "idle_time": 143,
       "exit_code": -1,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_031_math_qwen2.5-0.5b_math_1pct_se_gpu5.log"
@@ -409,10 +409,10 @@
       "name": "math_qwen2.5-0.5b_math_1pct_seed31",
       "cmd": "PYTHONPATH=. /home/ubuntu/venvs/steering/bin/python -m scripts.train_qwen --mode contaminated --rate 0.01 --seed 31 --pr",
       "gpu": 7,
-      "status": "running",
-      "elapsed": 5,
-      "idle_time": 0,
-      "exit_code": -1,
+      "status": "failed",
+      "elapsed": 10,
+      "idle_time": 116,
+      "exit_code": 1,
       "retries": 1,
       "log_file": "/tmp/stride_applications_queue/job_033_math_qwen2.5-0.5b_math_1pct_se_gpu7.log"
     },
@@ -422,8 +422,8 @@
       "cmd": "PYTHONPATH=. /home/ubuntu/venvs/steering/bin/python -m scripts.train_qwen --mode contaminated --rate 0.01 --seed 32 --pr",
       "gpu": 6,
       "status": "running",
-      "elapsed": 141,
-      "idle_time": 119,
+      "elapsed": 261,
+      "idle_time": 239,
       "exit_code": -1,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_034_math_qwen2.5-0.5b_math_1pct_se_gpu6.log"
@@ -470,8 +470,8 @@
       "cmd": "PYTHONPATH=. /home/ubuntu/venvs/steering/bin/python -m scripts.train_qwen --mode contaminated --rate 0.01 --seed 36 --pr",
       "gpu": 0,
       "status": "running",
-      "elapsed": 316,
-      "idle_time": 293,
+      "elapsed": 436,
+      "idle_time": 414,
       "exit_code": -1,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_038_math_qwen2.5-0.5b_math_1pct_se_gpu0.log"
@@ -492,25 +492,25 @@
       "job_id": 40,
       "name": "math_qwen2.5-0.5b_math_1pct_seed38",
       "cmd": "PYTHONPATH=. /home/ubuntu/venvs/steering/bin/python -m scripts.train_qwen --mode contaminated --rate 0.01 --seed 38 --pr",
-      "gpu": -1,
-      "status": "pending",
-      "elapsed": 0,
-      "idle_time": 0,
+      "gpu": 7,
+      "status": "running",
+      "elapsed": 86,
+      "idle_time": 59,
       "exit_code": -1,
       "retries": 0,
-      "log_file": ""
+      "log_file": "/tmp/stride_applications_queue/job_040_math_qwen2.5-0.5b_math_1pct_se_gpu7.log"
     },
     {
       "job_id": 41,
       "name": "math_qwen2.5-0.5b_math_1pct_seed39",
       "cmd": "PYTHONPATH=. /home/ubuntu/venvs/steering/bin/python -m scripts.train_qwen --mode contaminated --rate 0.01 --seed 39 --pr",
-      "gpu": -1,
-      "status": "pending",
-      "elapsed": 0,
-      "idle_time": 0,
-      "exit_code": -1,
-      "retries": 0,
-      "log_file": ""
+      "gpu": 7,
+      "status": "failed",
+      "elapsed": 11,
+      "idle_time": 90,
+      "exit_code": 1,
+      "retries": 1,
+      "log_file": "/tmp/stride_applications_queue/job_041_math_qwen2.5-0.5b_math_1pct_se_gpu7.log"
     },
     {
       "job_id": 42,
@@ -530,8 +530,8 @@
       "cmd": "PYTHONPATH=. /home/ubuntu/venvs/steering/bin/python -m scripts.train_qwen --mode contaminated --rate 0.01 --seed 41 --pr",
       "gpu": 4,
       "status": "running",
-      "elapsed": 131,
-      "idle_time": 104,
+      "elapsed": 251,
+      "idle_time": 224,
       "exit_code": -1,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_043_math_qwen2.5-0.5b_math_1pct_se_gpu4.log"
@@ -552,13 +552,13 @@
       "job_id": 45,
       "name": "math_qwen2.5-0.5b_math_1pt5pct_seed21",
       "cmd": "PYTHONPATH=. /home/ubuntu/venvs/steering/bin/python -m scripts.train_qwen --mode contaminated --rate 0.015 --seed 21 --p",
-      "gpu": -1,
-      "status": "pending",
-      "elapsed": 0,
-      "idle_time": 0,
-      "exit_code": -1,
-      "retries": 0,
-      "log_file": ""
+      "gpu": 3,
+      "status": "failed",
+      "elapsed": 10,
+      "idle_time": 97,
+      "exit_code": 1,
+      "retries": 1,
+      "log_file": "/tmp/stride_applications_queue/job_045_math_qwen2.5-0.5b_math_1pt5pct_gpu3.log"
     },
     {
       "job_id": 46,
@@ -576,13 +576,13 @@
       "job_id": 47,
       "name": "math_qwen2.5-0.5b_math_1pt5pct_seed23",
       "cmd": "PYTHONPATH=. /home/ubuntu/venvs/steering/bin/python -m scripts.train_qwen --mode contaminated --rate 0.015 --seed 23 --p",
-      "gpu": -1,
-      "status": "pending",
-      "elapsed": 0,
-      "idle_time": 0,
+      "gpu": 3,
+      "status": "running",
+      "elapsed": 91,
+      "idle_time": 59,
       "exit_code": -1,
       "retries": 0,
-      "log_file": ""
+      "log_file": "/tmp/stride_applications_queue/job_047_math_qwen2.5-0.5b_math_1pt5pct_gpu3.log"
     },
     {
       "job_id": 48,