queue status update

queue_status.json

@@ -1,10 +1,10 @@
 {
-  "timestamp": "2026-04-12 07:49:00",
-  "total": 98,
-  "pending": 75,
+  "timestamp": "2026-04-12 07:51:00",
+  "total": 120,
+  "pending": 96,
   "running": 6,
   "done": 15,
-  "failed": 2,
+  "failed": 3,
   "stale": 0,
   "retrying": 0,
   "jobs": [
@@ -15,7 +15,7 @@
       "gpu": 0,
       "status": "done",
       "elapsed": 1786,
-      "idle_time": 19277,
+      "idle_time": 19397,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/gpu_queue/job_000_add_sub_baseline_10K_gpu0.log"
@@ -27,7 +27,7 @@
       "gpu": 1,
       "status": "done",
       "elapsed": 2101,
-      "idle_time": 18962,
+      "idle_time": 19082,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/gpu_queue/job_001_add_sub_baseline_25K_gpu1.log"
@@ -39,7 +39,7 @@
       "gpu": 2,
       "status": "done",
       "elapsed": 4753,
-      "idle_time": 16311,
+      "idle_time": 16431,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/gpu_queue/job_002_as_sorl_abs10_K1_25K_gpu2.log"
@@ -51,7 +51,7 @@
       "gpu": 0,
       "status": "done",
       "elapsed": 2366,
-      "idle_time": 18697,
+      "idle_time": 18817,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/gpu_queue/job_003_add_sub_baseline_50K_gpu0.log"
@@ -63,7 +63,7 @@
       "gpu": 1,
       "status": "done",
       "elapsed": 6727,
-      "idle_time": 14336,
+      "idle_time": 14456,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/gpu_queue/job_004_as_sorl_abs10_K1_50K_gpu1.log"
@@ -75,7 +75,7 @@
       "gpu": 2,
       "status": "done",
       "elapsed": 3112,
-      "idle_time": 17951,
+      "idle_time": 18071,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/gpu_queue/job_005_add_sub_baseline_100K_gpu2.log"
@@ -87,7 +87,7 @@
       "gpu": 0,
       "status": "done",
       "elapsed": 8856,
-      "idle_time": 10417,
+      "idle_time": 10537,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/gpu_queue/job_006_as_sorl_abs10_K1_100K_gpu0.log"
@@ -169,11 +169,11 @@
       "name": "as_sorl_abs10_K4_500K",
       "cmd": "python -m arithmetic.train --mode sorl --ops add_sub --dataset_size 500000 --abs_vocab 10 --K 4 --nu",
       "gpu": 0,
-      "status": "running",
-      "elapsed": 5046,
-      "idle_time": 16,
-      "exit_code": -1,
-      "retries": 0,
+      "status": "failed",
+      "elapsed": 5,
+      "idle_time": 44,
+      "exit_code": -9,
+      "retries": 1,
       "log_file": "/tmp/gpu_queue/job_013_as_sorl_abs10_K4_500K_gpu0.log"
     },
     {
@@ -459,7 +459,7 @@
       "gpu": 0,
       "status": "done",
       "elapsed": 4510,
-      "idle_time": 14182,
+      "idle_time": 14302,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/gpu_queue/job_037_as_sorl_abs10_K4_25K_gpu0.log"
@@ -531,7 +531,7 @@
       "gpu": 0,
       "status": "done",
       "elapsed": 5510,
-      "idle_time": 8668,
+      "idle_time": 8788,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/gpu_queue/job_043_as_sorl_abs50_K4_50K_gpu0.log"
@@ -651,7 +651,7 @@
       "gpu": 1,
       "status": "failed",
       "elapsed": 5,
-      "idle_time": 12347,
+      "idle_time": 12467,
       "exit_code": -9,
       "retries": 1,
       "log_file": "/tmp/gpu_queue/job_053_as_sorl_abs10_K4_zipf10.0_500K_gpu1.log"
@@ -675,7 +675,7 @@
       "gpu": 0,
       "status": "failed",
       "elapsed": 5,
-      "idle_time": 5047,
+      "idle_time": 5167,
       "exit_code": -9,
       "retries": 1,
       "log_file": "/tmp/gpu_queue/job_055_as_sorl_abs100_K1_zipf5.0_500K_gpu0.log"
@@ -771,7 +771,7 @@
       "gpu": 0,
       "status": "done",
       "elapsed": 5352,
-      "idle_time": 5061,
+      "idle_time": 5181,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/gpu_queue/job_063_as_sorl_abs10_K1_50K_1L3H510d_gpu0.log"
@@ -794,8 +794,8 @@
       "cmd": "python -m arithmetic.train --mode sorl --ops add_sub --dataset_size 100000 --abs_vocab 10 --K 1 --nu",
       "gpu": 0,
       "status": "running",
-      "elapsed": 351,
-      "idle_time": 2,
+      "elapsed": 471,
+      "idle_time": 0,
       "exit_code": -1,
       "retries": 0,
       "log_file": "/tmp/gpu_queue/job_065_as_sorl_abs10_K1_100K_1L3H510d_gpu0.log"
@@ -842,8 +842,8 @@
       "cmd": "python -m arithmetic.train --mode sorl --ops add_sub --dataset_size 500000 --abs_vocab 10 --K 1 --nu",
       "gpu": 2,
       "status": "running",
-      "elapsed": 13506,
-      "idle_time": 24,
+      "elapsed": 13626,
+      "idle_time": 17,
       "exit_code": -1,
       "retries": 0,
       "log_file": "/tmp/gpu_queue/job_069_as_sorl_abs10_K1_500K_1L3H510d_gpu2.log"
@@ -852,13 +852,13 @@
       "job_id": 70,
       "name": "as_baseline_25K_1L2H256d",
       "cmd": "python -m arithmetic.train --mode baseline --ops add_sub --dataset_size 25000 --num_epochs 20 --n_la",
-      "gpu": -1,
-      "status": "pending",
-      "elapsed": 0,
-      "idle_time": 0,
+      "gpu": 0,
+      "status": "running",
+      "elapsed": 41,
+      "idle_time": 17,
       "exit_code": -1,
       "retries": 0,
-      "log_file": ""
+      "log_file": "/tmp/gpu_queue/job_070_as_baseline_25K_1L2H256d_gpu0.log"
     },
     {
       "job_id": 71,
@@ -903,7 +903,7 @@
       "gpu": 2,
       "status": "done",
       "elapsed": 2797,
-      "idle_time": 13511,
+      "idle_time": 13631,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/gpu_queue/job_074_as_baseline_100K_1L2H256d_gpu2.log"
@@ -927,7 +927,7 @@
       "gpu": 0,
       "status": "done",
       "elapsed": 4832,
-      "idle_time": 356,
+      "idle_time": 476,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/gpu_queue/job_076_as_baseline_250K_1L2H256d_gpu0.log"
@@ -951,7 +951,7 @@
       "gpu": 2,
       "status": "done",
       "elapsed": 7064,
-      "idle_time": 10884,
+      "idle_time": 11004,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/gpu_queue/job_078_as_baseline_500K_1L2H256d_gpu2.log"
@@ -975,7 +975,7 @@
       "gpu": 1,
       "status": "done",
       "elapsed": 1971,
-      "idle_time": 12362,
+      "idle_time": 12482,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/gpu_queue/job_080_as_baseline_25K_2L1H128d_gpu1.log"
@@ -1058,8 +1058,8 @@
       "cmd": "python -m arithmetic.train --mode sorl --ops add_sub --dataset_size 250000 --abs_vocab 10 --K 1 --nu",
       "gpu": 1,
       "status": "running",
-      "elapsed": 12346,
-      "idle_time": 24,
+      "elapsed": 12466,
+      "idle_time": 15,
       "exit_code": -1,
       "retries": 0,
       "log_file": "/tmp/gpu_queue/job_087_as_sorl_abs10_K1_250K_2L1H128d_gpu1.log"
@@ -1082,8 +1082,8 @@
       "cmd": "python -m arithmetic.train --mode sorl --ops add_sub --dataset_size 500000 --abs_vocab 10 --K 1 --nu",
       "gpu": 1,
       "status": "running",
-      "elapsed": 18956,
-      "idle_time": 15,
+      "elapsed": 19076,
+      "idle_time": 5,
       "exit_code": -1,
       "retries": 0,
       "log_file": "/tmp/gpu_queue/job_089_as_sorl_abs10_K1_500K_2L1H128d_gpu1.log"
@@ -1119,7 +1119,7 @@
       "gpu": 0,
       "status": "done",
       "elapsed": 3477,
-      "idle_time": 5191,
+      "idle_time": 5311,
       "exit_code": 0,
       "retries": 0,
       "log_file": "/tmp/gpu_queue/job_092_as_sorl_abs30_K4_10K_gpu0.log"
@@ -1154,8 +1154,8 @@
       "cmd": "python -m arithmetic.train --mode sorl --ops add_sub --dataset_size 250000 --abs_vocab 10 --K 4 --nu",
       "gpu": 2,
       "status": "running",
-      "elapsed": 10881,
-      "idle_time": 4,
+      "elapsed": 11001,
+      "idle_time": 2,
       "exit_code": -1,
       "retries": 0,
       "log_file": "/tmp/gpu_queue/job_095_as_sorl_abs10_K4_250K_gpu2.log"
@@ -1183,6 +1183,270 @@
       "exit_code": -1,
       "retries": 0,
       "log_file": ""
+    },
+    {
+      "job_id": 98,
+      "name": "as_sorl_abs2_K1_100K",
+      "cmd": "python -m arithmetic.train --mode sorl --ops add_sub --dataset_size 100000 --abs_vocab 2 --K 1 --num",
+      "gpu": -1,
+      "status": "pending",
+      "elapsed": 0,
+      "idle_time": 0,
+      "exit_code": -1,
+      "retries": 0,
+      "log_file": ""
+    },
+    {
+      "job_id": 99,
+      "name": "as_sorl_abs2_K4_100K",
+      "cmd": "python -m arithmetic.train --mode sorl --ops add_sub --dataset_size 100000 --abs_vocab 2 --K 4 --num",
+      "gpu": -1,
+      "status": "pending",
+      "elapsed": 0,
+      "idle_time": 0,
+      "exit_code": -1,
+      "retries": 0,
+      "log_file": ""
+    },
+    {
+      "job_id": 100,
+      "name": "as_sorl_abs16_K1_100K",
+      "cmd": "python -m arithmetic.train --mode sorl --ops add_sub --dataset_size 100000 --abs_vocab 16 --K 1 --nu",
+      "gpu": -1,
+      "status": "pending",
+      "elapsed": 0,
+      "idle_time": 0,
+      "exit_code": -1,
+      "retries": 0,
+      "log_file": ""
+    },
+    {
+      "job_id": 101,
+      "name": "as_sorl_abs16_K4_100K",
+      "cmd": "python -m arithmetic.train --mode sorl --ops add_sub --dataset_size 100000 --abs_vocab 16 --K 4 --nu",
+      "gpu": -1,
+      "status": "pending",
+      "elapsed": 0,
+      "idle_time": 0,
+      "exit_code": -1,
+      "retries": 0,
+      "log_file": ""
+    },
+    {
+      "job_id": 102,
+      "name": "as_sorl_abs20_K1_100K",
+      "cmd": "python -m arithmetic.train --mode sorl --ops add_sub --dataset_size 100000 --abs_vocab 20 --K 1 --nu",
+      "gpu": -1,
+      "status": "pending",
+      "elapsed": 0,
+      "idle_time": 0,
+      "exit_code": -1,
+      "retries": 0,
+      "log_file": ""
+    },
+    {
+      "job_id": 103,
+      "name": "as_sorl_abs20_K4_100K",
+      "cmd": "python -m arithmetic.train --mode sorl --ops add_sub --dataset_size 100000 --abs_vocab 20 --K 4 --nu",
+      "gpu": -1,
+      "status": "pending",
+      "elapsed": 0,
+      "idle_time": 0,
+      "exit_code": -1,
+      "retries": 0,
+      "log_file": ""
+    },
+    {
+      "job_id": 104,
+      "name": "as_sorl_abs70_K1_100K",
+      "cmd": "python -m arithmetic.train --mode sorl --ops add_sub --dataset_size 100000 --abs_vocab 70 --K 1 --nu",
+      "gpu": -1,
+      "status": "pending",
+      "elapsed": 0,
+      "idle_time": 0,
+      "exit_code": -1,
+      "retries": 0,
+      "log_file": ""
+    },
+    {
+      "job_id": 105,
+      "name": "as_sorl_abs70_K4_100K",
+      "cmd": "python -m arithmetic.train --mode sorl --ops add_sub --dataset_size 100000 --abs_vocab 70 --K 4 --nu",
+      "gpu": -1,
+      "status": "pending",
+      "elapsed": 0,
+      "idle_time": 0,
+      "exit_code": -1,
+      "retries": 0,
+      "log_file": ""
+    },
+    {
+      "job_id": 106,
+      "name": "as_sorl_abs100_K1_100K",
+      "cmd": "python -m arithmetic.train --mode sorl --ops add_sub --dataset_size 100000 --abs_vocab 100 --K 1 --n",
+      "gpu": -1,
+      "status": "pending",
+      "elapsed": 0,
+      "idle_time": 0,
+      "exit_code": -1,
+      "retries": 0,
+      "log_file": ""
+    },
+    {
+      "job_id": 107,
+      "name": "as_sorl_abs100_K4_100K",
+      "cmd": "python -m arithmetic.train --mode sorl --ops add_sub --dataset_size 100000 --abs_vocab 100 --K 4 --n",
+      "gpu": -1,
+      "status": "pending",
+      "elapsed": 0,
+      "idle_time": 0,
+      "exit_code": -1,
+      "retries": 0,
+      "log_file": ""
+    },
+    {
+      "job_id": 108,
+      "name": "as_sorl_abs10_K1_zipf2.0_100K",
+      "cmd": "python -m arithmetic.train --mode sorl --ops add_sub --dataset_size 100000 --abs_vocab 10 --K 1 --al",
+      "gpu": -1,
+      "status": "pending",
+      "elapsed": 0,
+      "idle_time": 0,
+      "exit_code": -1,
+      "retries": 0,
+      "log_file": ""
+    },
+    {
+      "job_id": 109,
+      "name": "as_sorl_abs10_K4_zipf2.0_100K",
+      "cmd": "python -m arithmetic.train --mode sorl --ops add_sub --dataset_size 100000 --abs_vocab 10 --K 4 --al",
+      "gpu": -1,
+      "status": "pending",
+      "elapsed": 0,
+      "idle_time": 0,
+      "exit_code": -1,
+      "retries": 0,
+      "log_file": ""
+    },
+    {
+      "job_id": 110,
+      "name": "as_sorl_abs10_K1_zipf5.0_100K",
+      "cmd": "python -m arithmetic.train --mode sorl --ops add_sub --dataset_size 100000 --abs_vocab 10 --K 1 --al",
+      "gpu": -1,
+      "status": "pending",
+      "elapsed": 0,
+      "idle_time": 0,
+      "exit_code": -1,
+      "retries": 0,
+      "log_file": ""
+    },
+    {
+      "job_id": 111,
+      "name": "as_sorl_abs10_K4_zipf5.0_100K",
+      "cmd": "python -m arithmetic.train --mode sorl --ops add_sub --dataset_size 100000 --abs_vocab 10 --K 4 --al",
+      "gpu": -1,
+      "status": "pending",
+      "elapsed": 0,
+      "idle_time": 0,
+      "exit_code": -1,
+      "retries": 0,
+      "log_file": ""
+    },
+    {
+      "job_id": 112,
+      "name": "as_sorl_abs10_K1_zipf10.0_100K",
+      "cmd": "python -m arithmetic.train --mode sorl --ops add_sub --dataset_size 100000 --abs_vocab 10 --K 1 --al",
+      "gpu": -1,
+      "status": "pending",
+      "elapsed": 0,
+      "idle_time": 0,
+      "exit_code": -1,
+      "retries": 0,
+      "log_file": ""
+    },
+    {
+      "job_id": 113,
+      "name": "as_sorl_abs10_K4_zipf10.0_100K",
+      "cmd": "python -m arithmetic.train --mode sorl --ops add_sub --dataset_size 100000 --abs_vocab 10 --K 4 --al",
+      "gpu": -1,
+      "status": "pending",
+      "elapsed": 0,
+      "idle_time": 0,
+      "exit_code": -1,
+      "retries": 0,
+      "log_file": ""
+    },
+    {
+      "job_id": 114,
+      "name": "as_sorl_abs100_K1_zipf2.0_100K",
+      "cmd": "python -m arithmetic.train --mode sorl --ops add_sub --dataset_size 100000 --abs_vocab 100 --K 1 --a",
+      "gpu": -1,
+      "status": "pending",
+      "elapsed": 0,
+      "idle_time": 0,
+      "exit_code": -1,
+      "retries": 0,
+      "log_file": ""
+    },
+    {
+      "job_id": 115,
+      "name": "as_sorl_abs100_K4_zipf2.0_100K",
+      "cmd": "python -m arithmetic.train --mode sorl --ops add_sub --dataset_size 100000 --abs_vocab 100 --K 4 --a",
+      "gpu": -1,
+      "status": "pending",
+      "elapsed": 0,
+      "idle_time": 0,
+      "exit_code": -1,
+      "retries": 0,
+      "log_file": ""
+    },
+    {
+      "job_id": 116,
+      "name": "as_sorl_abs100_K1_zipf5.0_100K",
+      "cmd": "python -m arithmetic.train --mode sorl --ops add_sub --dataset_size 100000 --abs_vocab 100 --K 1 --a",
+      "gpu": -1,
+      "status": "pending",
+      "elapsed": 0,
+      "idle_time": 0,
+      "exit_code": -1,
+      "retries": 0,
+      "log_file": ""
+    },
+    {
+      "job_id": 117,
+      "name": "as_sorl_abs100_K4_zipf5.0_100K",
+      "cmd": "python -m arithmetic.train --mode sorl --ops add_sub --dataset_size 100000 --abs_vocab 100 --K 4 --a",
+      "gpu": -1,
+      "status": "pending",
+      "elapsed": 0,
+      "idle_time": 0,
+      "exit_code": -1,
+      "retries": 0,
+      "log_file": ""
+    },
+    {
+      "job_id": 118,
+      "name": "as_sorl_abs100_K1_zipf10.0_100K",
+      "cmd": "python -m arithmetic.train --mode sorl --ops add_sub --dataset_size 100000 --abs_vocab 100 --K 1 --a",
+      "gpu": -1,
+      "status": "pending",
+      "elapsed": 0,
+      "idle_time": 0,
+      "exit_code": -1,
+      "retries": 0,
+      "log_file": ""
+    },
+    {
+      "job_id": 119,
+      "name": "as_sorl_abs100_K4_zipf10.0_100K",
+      "cmd": "python -m arithmetic.train --mode sorl --ops add_sub --dataset_size 100000 --abs_vocab 100 --K 4 --a",
+      "gpu": -1,
+      "status": "pending",
+      "elapsed": 0,
+      "idle_time": 0,
+      "exit_code": -1,
+      "retries": 0,
+      "log_file": ""
     }
   ]
 }