queue status update

queue_status.json

@@ -1,10 +1,10 @@
 {
-  "timestamp": "2026-04-12 04:21:59",
+  "timestamp": "2026-04-12 04:23:59",
   "total": 98,
-  "pending": 84,
+  "pending": 82,
   "running": 6,
-  "done": 8,
-  "failed": 0,
+  "done": 9,
+  "failed": 1,
   "stale": 0,
   "retrying": 0,
   "jobs": [
@@ -15,7 +15,7 @@
       "gpu": 0,
       "status": "done",
       "elapsed": 1786,
-      "idle_time": 6857,
+      "idle_time": 6977,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/gpu_queue/job_000_add_sub_baseline_10K_gpu0.log"
@@ -27,7 +27,7 @@
       "gpu": 1,
       "status": "done",
       "elapsed": 2101,
-      "idle_time": 6542,
+      "idle_time": 6662,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/gpu_queue/job_001_add_sub_baseline_25K_gpu1.log"
@@ -39,7 +39,7 @@
       "gpu": 2,
       "status": "done",
       "elapsed": 4753,
-      "idle_time": 3890,
+      "idle_time": 4010,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/gpu_queue/job_002_as_sorl_abs10_K1_25K_gpu2.log"
@@ -51,7 +51,7 @@
       "gpu": 0,
       "status": "done",
       "elapsed": 2366,
-      "idle_time": 6276,
+      "idle_time": 6396,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/gpu_queue/job_003_add_sub_baseline_50K_gpu0.log"
@@ -63,7 +63,7 @@
       "gpu": 1,
       "status": "done",
       "elapsed": 6727,
-      "idle_time": 1916,
+      "idle_time": 2036,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/gpu_queue/job_004_as_sorl_abs10_K1_50K_gpu1.log"
@@ -75,7 +75,7 @@
       "gpu": 2,
       "status": "done",
       "elapsed": 3112,
-      "idle_time": 5530,
+      "idle_time": 5650,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/gpu_queue/job_005_add_sub_baseline_100K_gpu2.log"
@@ -86,8 +86,8 @@
       "cmd": "python -m arithmetic.train --mode sorl --ops add_sub --dataset_size 100000 --abs_vocab 10 --K 1 --nu",
       "gpu": 0,
       "status": "running",
-      "elapsed": 6850,
-      "idle_time": 16,
+      "elapsed": 6970,
+      "idle_time": 9,
       "exit_code": -1,
       "retries": 0,
       "log_file": "/tmp/gpu_queue/job_006_as_sorl_abs10_K1_100K_gpu0.log"
@@ -459,7 +459,7 @@
       "gpu": 0,
       "status": "done",
       "elapsed": 4510,
-      "idle_time": 1762,
+      "idle_time": 1882,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/gpu_queue/job_037_as_sorl_abs10_K4_25K_gpu0.log"
@@ -530,8 +530,8 @@
       "cmd": "python -m arithmetic.train --mode sorl --ops add_sub --dataset_size 50000 --abs_vocab 50 --K 4 --num",
       "gpu": 0,
       "status": "running",
-      "elapsed": 1755,
-      "idle_time": 15,
+      "elapsed": 1875,
+      "idle_time": 12,
       "exit_code": -1,
       "retries": 0,
       "log_file": "/tmp/gpu_queue/job_043_as_sorl_abs50_K4_50K_gpu0.log"
@@ -648,13 +648,13 @@
       "job_id": 53,
       "name": "as_sorl_abs10_K4_zipf10.0_500K",
       "cmd": "python -m arithmetic.train --mode sorl --ops add_sub --dataset_size 500000 --abs_vocab 10 --K 4 --al",
-      "gpu": -1,
-      "status": "pending",
-      "elapsed": 0,
-      "idle_time": 0,
-      "exit_code": -1,
-      "retries": 0,
-      "log_file": ""
+      "gpu": 1,
+      "status": "failed",
+      "elapsed": 5,
+      "idle_time": 46,
+      "exit_code": -9,
+      "retries": 1,
+      "log_file": "/tmp/gpu_queue/job_053_as_sorl_abs10_K4_zipf10.0_500K_gpu1.log"
     },
     {
       "job_id": 54,
@@ -842,8 +842,8 @@
       "cmd": "python -m arithmetic.train --mode sorl --ops add_sub --dataset_size 500000 --abs_vocab 10 --K 1 --nu",
       "gpu": 2,
       "status": "running",
-      "elapsed": 1085,
-      "idle_time": 15,
+      "elapsed": 1205,
+      "idle_time": 11,
       "exit_code": -1,
       "retries": 0,
       "log_file": "/tmp/gpu_queue/job_069_as_sorl_abs10_K1_500K_1L3H510d_gpu2.log"
@@ -903,7 +903,7 @@
       "gpu": 2,
       "status": "done",
       "elapsed": 2797,
-      "idle_time": 1090,
+      "idle_time": 1210,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/gpu_queue/job_074_as_baseline_100K_1L2H256d_gpu2.log"
@@ -950,8 +950,8 @@
       "cmd": "python -m arithmetic.train --mode baseline --ops add_sub --dataset_size 500000 --num_epochs 20 --n_l",
       "gpu": 2,
       "status": "running",
-      "elapsed": 5525,
-      "idle_time": 1,
+      "elapsed": 5645,
+      "idle_time": 13,
       "exit_code": -1,
       "retries": 0,
       "log_file": "/tmp/gpu_queue/job_078_as_baseline_500K_1L2H256d_gpu2.log"
@@ -973,10 +973,10 @@
       "name": "as_baseline_25K_2L1H128d",
       "cmd": "python -m arithmetic.train --mode baseline --ops add_sub --dataset_size 25000 --num_epochs 20 --n_la",
       "gpu": 1,
-      "status": "running",
-      "elapsed": 1910,
-      "idle_time": 25,
-      "exit_code": -1,
+      "status": "done",
+      "elapsed": 1971,
+      "idle_time": 61,
+      "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/gpu_queue/job_080_as_baseline_25K_2L1H128d_gpu1.log"
     },
@@ -1056,13 +1056,13 @@
       "job_id": 87,
       "name": "as_sorl_abs10_K1_250K_2L1H128d",
       "cmd": "python -m arithmetic.train --mode sorl --ops add_sub --dataset_size 250000 --abs_vocab 10 --K 1 --nu",
-      "gpu": -1,
-      "status": "pending",
-      "elapsed": 0,
-      "idle_time": 0,
+      "gpu": 1,
+      "status": "running",
+      "elapsed": 45,
+      "idle_time": 13,
       "exit_code": -1,
       "retries": 0,
-      "log_file": ""
+      "log_file": "/tmp/gpu_queue/job_087_as_sorl_abs10_K1_250K_2L1H128d_gpu1.log"
     },
     {
       "job_id": 88,
@@ -1082,8 +1082,8 @@
       "cmd": "python -m arithmetic.train --mode sorl --ops add_sub --dataset_size 500000 --abs_vocab 10 --K 1 --nu",
       "gpu": 1,
       "status": "running",
-      "elapsed": 6535,
-      "idle_time": 11,
+      "elapsed": 6655,
+      "idle_time": 18,
       "exit_code": -1,
       "retries": 0,
       "log_file": "/tmp/gpu_queue/job_089_as_sorl_abs10_K1_500K_2L1H128d_gpu1.log"