Upload queue_status.json with huggingface_hub

queue_status.json

@@ -1,9 +1,9 @@
 {
-  "timestamp": "2026-04-26 00:59:24",
+  "timestamp": "2026-04-26 01:01:25",
   "total": 64,
-  "pending": 25,
+  "pending": 24,
   "running": 8,
-  "done": 26,
+  "done": 27,
   "failed": 5,
   "stale": 0,
   "retrying": 0,
@@ -15,7 +15,7 @@
       "gpu": 0,
       "status": "done",
       "elapsed": 1254,
-      "idle_time": 4439,
+      "idle_time": 4559,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_000_math_qwen2.5-0.5b_math_0pt5pct_gpu0.log"
@@ -27,7 +27,7 @@
       "gpu": 1,
       "status": "done",
       "elapsed": 1401,
-      "idle_time": 4292,
+      "idle_time": 4412,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_001_math_qwen2.5-0.5b_math_0pt5pct_gpu1.log"
@@ -39,7 +39,7 @@
       "gpu": 2,
       "status": "done",
       "elapsed": 1401,
-      "idle_time": 4292,
+      "idle_time": 4412,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_002_math_qwen2.5-0.5b_math_0pt5pct_gpu2.log"
@@ -51,7 +51,7 @@
       "gpu": 3,
       "status": "done",
       "elapsed": 1426,
-      "idle_time": 4267,
+      "idle_time": 4387,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_003_math_qwen2.5-0.5b_math_0pt5pct_gpu3.log"
@@ -63,7 +63,7 @@
       "gpu": 4,
       "status": "done",
       "elapsed": 1366,
-      "idle_time": 4327,
+      "idle_time": 4447,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_004_math_qwen2.5-0.5b_math_0pt5pct_gpu4.log"
@@ -75,7 +75,7 @@
       "gpu": 5,
       "status": "done",
       "elapsed": 1427,
-      "idle_time": 4267,
+      "idle_time": 4387,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_005_math_qwen2.5-0.5b_math_0pt5pct_gpu5.log"
@@ -87,7 +87,7 @@
       "gpu": 6,
       "status": "done",
       "elapsed": 1427,
-      "idle_time": 4266,
+      "idle_time": 4387,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_006_math_qwen2.5-0.5b_math_0pt5pct_gpu6.log"
@@ -99,7 +99,7 @@
       "gpu": 7,
       "status": "done",
       "elapsed": 1442,
-      "idle_time": 4251,
+      "idle_time": 4371,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_007_math_qwen2.5-0.5b_math_0pt5pct_gpu7.log"
@@ -147,7 +147,7 @@
       "gpu": 3,
       "status": "failed",
       "elapsed": 9,
-      "idle_time": 4242,
+      "idle_time": 4363,
       "exit_code": 1,
       "retries": 1,
       "log_file": "/tmp/stride_applications_queue/job_011_math_qwen2.5-0.5b_math_0pt5pct_gpu3.log"
@@ -171,7 +171,7 @@
       "gpu": 1,
       "status": "done",
       "elapsed": 1307,
-      "idle_time": 2981,
+      "idle_time": 3101,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_013_math_qwen2.5-0.5b_math_0pt5pct_gpu1.log"
@@ -183,7 +183,7 @@
       "gpu": 6,
       "status": "done",
       "elapsed": 1321,
-      "idle_time": 2917,
+      "idle_time": 3038,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_014_math_qwen2.5-0.5b_math_0pt5pct_gpu6.log"
@@ -195,7 +195,7 @@
       "gpu": 4,
       "status": "done",
       "elapsed": 1415,
-      "idle_time": 2908,
+      "idle_time": 3028,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_015_math_qwen2.5-0.5b_math_0pt5pct_gpu4.log"
@@ -219,7 +219,7 @@
       "gpu": 0,
       "status": "done",
       "elapsed": 1349,
-      "idle_time": 3089,
+      "idle_time": 3209,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_017_math_qwen2.5-0.5b_math_0pt5pct_gpu0.log"
@@ -231,7 +231,7 @@
       "gpu": 2,
       "status": "done",
       "elapsed": 1379,
-      "idle_time": 2909,
+      "idle_time": 3030,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_018_math_qwen2.5-0.5b_math_0pt5pct_gpu2.log"
@@ -243,7 +243,7 @@
       "gpu": 6,
       "status": "failed",
       "elapsed": 11,
-      "idle_time": 4240,
+      "idle_time": 4361,
       "exit_code": 1,
       "retries": 1,
       "log_file": "/tmp/stride_applications_queue/job_019_math_qwen2.5-0.5b_math_0pt5pct_gpu6.log"
@@ -279,7 +279,7 @@
       "gpu": 3,
       "status": "done",
       "elapsed": 1472,
-      "idle_time": 2771,
+      "idle_time": 2891,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_022_math_qwen2.5-0.5b_math_1pct_se_gpu3.log"
@@ -291,7 +291,7 @@
       "gpu": 2,
       "status": "done",
       "elapsed": 1480,
-      "idle_time": 1427,
+      "idle_time": 1548,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_023_math_qwen2.5-0.5b_math_1pct_se_gpu2.log"
@@ -315,7 +315,7 @@
       "gpu": 5,
       "status": "done",
       "elapsed": 1424,
-      "idle_time": 2826,
+      "idle_time": 2946,
       "exit_code": 0,
       "retries": 1,
       "log_file": "/tmp/stride_applications_queue/job_025_math_qwen2.5-0.5b_math_1pct_se_gpu5.log"
@@ -327,7 +327,7 @@
       "gpu": 1,
       "status": "done",
       "elapsed": 1348,
-      "idle_time": 1630,
+      "idle_time": 1751,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_026_math_qwen2.5-0.5b_math_1pct_se_gpu1.log"
@@ -339,7 +339,7 @@
       "gpu": 7,
       "status": "done",
       "elapsed": 1456,
-      "idle_time": 2793,
+      "idle_time": 2913,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_027_math_qwen2.5-0.5b_math_1pct_se_gpu7.log"
@@ -349,10 +349,10 @@
       "name": "math_qwen2.5-0.5b_math_1pct_seed26",
       "cmd": "PYTHONPATH=. /home/ubuntu/venvs/steering/bin/python -m scripts.train_qwen --mode contaminated --rate 0.01 --seed 26 --pr",
       "gpu": 2,
-      "status": "running",
-      "elapsed": 1421,
-      "idle_time": 1379,
-      "exit_code": -1,
+      "status": "done",
+      "elapsed": 1498,
+      "idle_time": 45,
+      "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_028_math_qwen2.5-0.5b_math_1pct_se_gpu2.log"
     },
@@ -387,7 +387,7 @@
       "gpu": 5,
       "status": "done",
       "elapsed": 1343,
-      "idle_time": 1480,
+      "idle_time": 1600,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_031_math_qwen2.5-0.5b_math_1pct_se_gpu5.log"
@@ -411,7 +411,7 @@
       "gpu": 7,
       "status": "failed",
       "elapsed": 10,
-      "idle_time": 2766,
+      "idle_time": 2887,
       "exit_code": 1,
       "retries": 1,
       "log_file": "/tmp/stride_applications_queue/job_033_math_qwen2.5-0.5b_math_1pct_se_gpu7.log"
@@ -423,7 +423,7 @@
       "gpu": 6,
       "status": "done",
       "elapsed": 1561,
-      "idle_time": 1351,
+      "idle_time": 1472,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_034_math_qwen2.5-0.5b_math_1pct_se_gpu6.log"
@@ -434,8 +434,8 @@
       "cmd": "PYTHONPATH=. /home/ubuntu/venvs/steering/bin/python -m scripts.train_qwen --mode contaminated --rate 0.01 --seed 33 --pr",
       "gpu": 4,
       "status": "running",
-      "elapsed": 1311,
-      "idle_time": 1262,
+      "elapsed": 1432,
+      "idle_time": 1382,
       "exit_code": -1,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_035_math_qwen2.5-0.5b_math_1pct_se_gpu4.log"
@@ -471,7 +471,7 @@
       "gpu": 0,
       "status": "done",
       "elapsed": 1349,
-      "idle_time": 1738,
+      "idle_time": 1859,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_038_math_qwen2.5-0.5b_math_1pct_se_gpu0.log"
@@ -495,7 +495,7 @@
       "gpu": 7,
       "status": "done",
       "elapsed": 1497,
-      "idle_time": 1241,
+      "idle_time": 1361,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_040_math_qwen2.5-0.5b_math_1pct_se_gpu7.log"
@@ -507,7 +507,7 @@
       "gpu": 7,
       "status": "failed",
       "elapsed": 11,
-      "idle_time": 2740,
+      "idle_time": 2861,
       "exit_code": 1,
       "retries": 1,
       "log_file": "/tmp/stride_applications_queue/job_041_math_qwen2.5-0.5b_math_1pct_se_gpu7.log"
@@ -531,7 +531,7 @@
       "gpu": 4,
       "status": "done",
       "elapsed": 1586,
-      "idle_time": 1317,
+      "idle_time": 1437,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_043_math_qwen2.5-0.5b_math_1pct_se_gpu4.log"
@@ -542,8 +542,8 @@
       "cmd": "PYTHONPATH=. /home/ubuntu/venvs/steering/bin/python -m scripts.train_qwen --mode contaminated --rate 0.015 --seed 20 --p",
       "gpu": 6,
       "status": "running",
-      "elapsed": 1346,
-      "idle_time": 1314,
+      "elapsed": 1467,
+      "idle_time": 27,
       "exit_code": -1,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_044_math_qwen2.5-0.5b_math_1pt5pct_gpu6.log"
@@ -555,7 +555,7 @@
       "gpu": 3,
       "status": "failed",
       "elapsed": 10,
-      "idle_time": 2747,
+      "idle_time": 2867,
       "exit_code": 1,
       "retries": 1,
       "log_file": "/tmp/stride_applications_queue/job_045_math_qwen2.5-0.5b_math_1pt5pct_gpu3.log"
@@ -566,8 +566,8 @@
       "cmd": "PYTHONPATH=. /home/ubuntu/venvs/steering/bin/python -m scripts.train_qwen --mode contaminated --rate 0.015 --seed 22 --p",
       "gpu": 7,
       "status": "running",
-      "elapsed": 1236,
-      "idle_time": 1211,
+      "elapsed": 1357,
+      "idle_time": 1331,
       "exit_code": -1,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_046_math_qwen2.5-0.5b_math_1pt5pct_gpu7.log"
@@ -579,7 +579,7 @@
       "gpu": 3,
       "status": "done",
       "elapsed": 1628,
-      "idle_time": 1115,
+      "idle_time": 1236,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_047_math_qwen2.5-0.5b_math_1pt5pct_gpu3.log"
@@ -602,8 +602,8 @@
       "cmd": "PYTHONPATH=. /home/ubuntu/venvs/steering/bin/python -m scripts.train_qwen --mode contaminated --rate 0.015 --seed 25 --p",
       "gpu": 3,
       "status": "running",
-      "elapsed": 1111,
-      "idle_time": 1087,
+      "elapsed": 1232,
+      "idle_time": 1208,
       "exit_code": -1,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_049_math_qwen2.5-0.5b_math_1pt5pct_gpu3.log"
@@ -614,8 +614,8 @@
       "cmd": "PYTHONPATH=. /home/ubuntu/venvs/steering/bin/python -m scripts.train_qwen --mode contaminated --rate 0.015 --seed 26 --p",
       "gpu": 0,
       "status": "running",
-      "elapsed": 101,
-      "idle_time": 71,
+      "elapsed": 221,
+      "idle_time": 191,
       "exit_code": -1,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_050_math_qwen2.5-0.5b_math_1pt5pct_gpu0.log"
@@ -648,13 +648,13 @@
       "job_id": 53,
       "name": "math_qwen2.5-0.5b_math_1pt5pct_seed29",
       "cmd": "PYTHONPATH=. /home/ubuntu/venvs/steering/bin/python -m scripts.train_qwen --mode contaminated --rate 0.015 --seed 29 --p",
-      "gpu": -1,
-      "status": "pending",
-      "elapsed": 0,
-      "idle_time": 0,
+      "gpu": 2,
+      "status": "running",
+      "elapsed": 41,
+      "idle_time": 17,
       "exit_code": -1,
       "retries": 0,
-      "log_file": ""
+      "log_file": "/tmp/stride_applications_queue/job_053_math_qwen2.5-0.5b_math_1pt5pct_gpu2.log"
     },
     {
       "job_id": 54,
@@ -698,8 +698,8 @@
       "cmd": "PYTHONPATH=. /home/ubuntu/venvs/steering/bin/python -m scripts.train_qwen --mode contaminated --rate 0.015 --seed 33 --p",
       "gpu": 1,
       "status": "running",
-      "elapsed": 21,
-      "idle_time": 21,
+      "elapsed": 141,
+      "idle_time": 114,
       "exit_code": -1,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_057_math_qwen2.5-0.5b_math_1pt5pct_gpu1.log"
@@ -735,7 +735,7 @@
       "gpu": 1,
       "status": "done",
       "elapsed": 1602,
-      "idle_time": 26,
+      "idle_time": 146,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_060_math_qwen2.5-0.5b_math_1pt5pct_gpu1.log"
@@ -758,8 +758,8 @@
       "cmd": "PYTHONPATH=. /home/ubuntu/venvs/steering/bin/python -m scripts.train_qwen --mode contaminated --rate 0.015 --seed 38 --p",
       "gpu": 5,
       "status": "running",
-      "elapsed": 1476,
-      "idle_time": 1435,
+      "elapsed": 1597,
+      "idle_time": 0,
       "exit_code": -1,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_062_math_qwen2.5-0.5b_math_1pt5pct_gpu5.log"
@@ -771,7 +771,7 @@
       "gpu": 0,
       "status": "done",
       "elapsed": 1631,
-      "idle_time": 107,
+      "idle_time": 227,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/stride_applications_queue/job_063_math_qwen2.5-0.5b_math_1pt5pct_gpu0.log"