eval/mauve_temp094_lambda_grid_8gpu/driver.log

[2026-05-06 03:20:33] starting temp=0.94 lambda grid on GPUs: 0,1,2,3,4,5,6,7
[2026-05-06 03:20:33] out_root=/home/sunhc/diffusion/sad/eval/mauve_temp094_lambda_grid_8gpu
[2026-05-06 03:20:33] num_samples=1024 sample_batch_size=256 mauve_batch_size=16
[2026-05-06 03:20:33] pps_list=1,2,4 lambda_list=0.2,0.4,0.6,0.8
[2026-05-06 03:20:33] models=top1,top2,top3,h2
[2026-05-06 03:20:33] launched top1_pps1_lam0.2_temp0.94 on gpu0 pid=456027
[2026-05-06 03:20:33] launched top2_pps1_lam0.2_temp0.94 on gpu1 pid=456033
[2026-05-06 03:20:33] launched top3_pps1_lam0.2_temp0.94 on gpu2 pid=456039
[2026-05-06 03:20:33] launched h2_pps1_lam0.2_temp0.94 on gpu3 pid=456045
[2026-05-06 03:20:33] launched top1_pps1_lam0.4_temp0.94 on gpu4 pid=456051
[2026-05-06 03:20:33] launched top2_pps1_lam0.4_temp0.94 on gpu5 pid=456057
[2026-05-06 03:20:33] launched top3_pps1_lam0.4_temp0.94 on gpu6 pid=456063
[2026-05-06 03:20:33] launched h2_pps1_lam0.4_temp0.94 on gpu7 pid=456069
[2026-05-06 03:27:18] nonzero exit (-1) for top2_pps1_lam0.2_temp0.94 on gpu1 pid=456033 but output exists
[2026-05-06 03:27:18] launched top1_pps1_lam0.6_temp0.94 on gpu1 pid=468378
[2026-05-06 03:27:23] nonzero exit (-1) for top1_pps1_lam0.2_temp0.94 on gpu0 pid=456027 but output exists
[2026-05-06 03:27:23] launched top2_pps1_lam0.6_temp0.94 on gpu0 pid=468749
[2026-05-06 03:27:38] nonzero exit (-1) for top2_pps1_lam0.4_temp0.94 on gpu5 pid=456057 but output exists
[2026-05-06 03:27:38] launched top3_pps1_lam0.6_temp0.94 on gpu5 pid=469389
[2026-05-06 03:27:48] nonzero exit (-1) for top1_pps1_lam0.4_temp0.94 on gpu4 pid=456051 but output exists
[2026-05-06 03:27:48] launched h2_pps1_lam0.6_temp0.94 on gpu4 pid=469796
[2026-05-06 03:28:08] nonzero exit (-1) for top3_pps1_lam0.2_temp0.94 on gpu2 pid=456039 but output exists
[2026-05-06 03:28:08] launched top1_pps1_lam0.8_temp0.94 on gpu2 pid=470271
[2026-05-06 03:28:18] nonzero exit (-1) for top3_pps1_lam0.4_temp0.94 on gpu6 pid=456063 but output exists
[2026-05-06 03:28:18] launched top2_pps1_lam0.8_temp0.94 on gpu6 pid=470678
[2026-05-06 03:28:23] nonzero exit (-1) for h2_pps1_lam0.2_temp0.94 on gpu3 pid=456045 but output exists
[2026-05-06 03:28:23] launched top3_pps1_lam0.8_temp0.94 on gpu3 pid=470913
[2026-05-06 03:29:33] nonzero exit (-1) for h2_pps1_lam0.4_temp0.94 on gpu7 pid=456069 but output exists
[2026-05-06 03:29:33] launched h2_pps1_lam0.8_temp0.94 on gpu7 pid=472074
[2026-05-06 03:33:08] nonzero exit (-1) for top2_pps1_lam0.6_temp0.94 on gpu0 pid=468749 but output exists
[2026-05-06 03:33:08] launched top1_pps2_lam0.2_temp0.94 on gpu0 pid=478221
[2026-05-06 03:33:43] nonzero exit (-1) for top1_pps1_lam0.6_temp0.94 on gpu1 pid=468378 but output exists
[2026-05-06 03:33:43] launched top2_pps2_lam0.2_temp0.94 on gpu1 pid=479757
[2026-05-06 03:34:13] nonzero exit (-1) for top2_pps1_lam0.8_temp0.94 on gpu6 pid=470678 but output exists
[2026-05-06 03:34:13] launched top3_pps2_lam0.2_temp0.94 on gpu6 pid=480552
[2026-05-06 03:34:23] nonzero exit (-1) for top1_pps1_lam0.8_temp0.94 on gpu2 pid=470271 but output exists
[2026-05-06 03:34:23] launched h2_pps2_lam0.2_temp0.94 on gpu2 pid=480959
[2026-05-06 03:34:48] nonzero exit (-1) for top3_pps1_lam0.6_temp0.94 on gpu5 pid=469389 but output exists
[2026-05-06 03:34:48] launched top1_pps2_lam0.4_temp0.94 on gpu5 pid=482176
[2026-05-06 03:35:48] nonzero exit (-1) for top3_pps1_lam0.8_temp0.94 on gpu3 pid=470913 but output exists
[2026-05-06 03:35:48] launched top2_pps2_lam0.4_temp0.94 on gpu3 pid=484330
[2026-05-06 03:36:48] nonzero exit (-1) for h2_pps1_lam0.6_temp0.94 on gpu4 pid=469796 but output exists
[2026-05-06 03:36:48] launched top3_pps2_lam0.4_temp0.94 on gpu4 pid=487427
[2026-05-06 03:37:38] nonzero exit (-1) for top1_pps2_lam0.2_temp0.94 on gpu0 pid=478221 but output exists
[2026-05-06 03:37:38] launched h2_pps2_lam0.4_temp0.94 on gpu0 pid=489498
[2026-05-06 03:38:13] nonzero exit (-1) for top2_pps2_lam0.2_temp0.94 on gpu1 pid=479757 but output exists
[2026-05-06 03:38:13] launched top1_pps2_lam0.6_temp0.94 on gpu1 pid=490229
[2026-05-06 03:39:04] nonzero exit (-1) for h2_pps1_lam0.8_temp0.94 on gpu7 pid=472074 but output exists
[2026-05-06 03:39:04] launched top2_pps2_lam0.6_temp0.94 on gpu7 pid=491856
[2026-05-06 03:39:14] nonzero exit (-1) for top3_pps2_lam0.2_temp0.94 on gpu6 pid=480552 but output exists
[2026-05-06 03:39:14] launched top3_pps2_lam0.6_temp0.94 on gpu6 pid=492387
[2026-05-06 03:39:34] nonzero exit (-1) for top1_pps2_lam0.4_temp0.94 on gpu5 pid=482176 but output exists
[2026-05-06 03:39:34] launched h2_pps2_lam0.6_temp0.94 on gpu5 pid=492861
[2026-05-06 03:39:44] nonzero exit (-1) for h2_pps2_lam0.2_temp0.94 on gpu2 pid=480959 but output exists
[2026-05-06 03:39:44] launched top1_pps2_lam0.8_temp0.94 on gpu2 pid=493268
[2026-05-06 03:40:19] nonzero exit (-1) for top2_pps2_lam0.4_temp0.94 on gpu3 pid=484330 but output exists
[2026-05-06 03:40:19] launched top2_pps2_lam0.8_temp0.94 on gpu3 pid=494944
[2026-05-06 03:42:09] nonzero exit (-1) for top3_pps2_lam0.4_temp0.94 on gpu4 pid=487427 but output exists
[2026-05-06 03:42:09] launched top3_pps2_lam0.8_temp0.94 on gpu4 pid=499723
[2026-05-06 03:42:54] nonzero exit (-1) for top1_pps2_lam0.6_temp0.94 on gpu1 pid=490229 but output exists
[2026-05-06 03:42:54] launched h2_pps2_lam0.8_temp0.94 on gpu1 pid=501553
[2026-05-06 03:43:39] nonzero exit (-1) for h2_pps2_lam0.4_temp0.94 on gpu0 pid=489498 but output exists
[2026-05-06 03:43:39] launched top1_pps4_lam0.2_temp0.94 on gpu0 pid=502578
[2026-05-06 03:43:44] nonzero exit (-1) for top2_pps2_lam0.6_temp0.94 on gpu7 pid=491856 but output exists
[2026-05-06 03:43:44] launched top2_pps4_lam0.2_temp0.94 on gpu7 pid=502763
[2026-05-06 03:44:49] nonzero exit (-1) for top3_pps2_lam0.6_temp0.94 on gpu6 pid=492387 but output exists
[2026-05-06 03:44:49] launched top3_pps4_lam0.2_temp0.94 on gpu6 pid=505899
[2026-05-06 03:44:59] nonzero exit (-1) for top1_pps2_lam0.8_temp0.94 on gpu2 pid=493268 but output exists
[2026-05-06 03:44:59] launched h2_pps4_lam0.2_temp0.94 on gpu2 pid=506306
[2026-05-06 03:45:14] nonzero exit (-1) for top2_pps2_lam0.8_temp0.94 on gpu3 pid=494944 but output exists
[2026-05-06 03:45:14] launched top1_pps4_lam0.4_temp0.94 on gpu3 pid=506870
[2026-05-06 03:45:54] nonzero exit (-1) for h2_pps2_lam0.6_temp0.94 on gpu5 pid=492861 but output exists
[2026-05-06 03:45:54] launched top2_pps4_lam0.4_temp0.94 on gpu5 pid=508811
[2026-05-06 03:47:59] nonzero exit (-1) for top1_pps4_lam0.2_temp0.94 on gpu0 pid=502578 but output exists
[2026-05-06 03:47:59] launched top3_pps4_lam0.4_temp0.94 on gpu0 pid=513497
[2026-05-06 03:47:59] finished top3_pps2_lam0.8_temp0.94 on gpu4 pid=499723
[2026-05-06 03:47:59] launched h2_pps4_lam0.4_temp0.94 on gpu4 pid=513510
[2026-05-06 03:48:04] nonzero exit (-1) for top2_pps4_lam0.2_temp0.94 on gpu7 pid=502763 but output exists
[2026-05-06 03:48:04] launched top1_pps4_lam0.6_temp0.94 on gpu7 pid=513856
[2026-05-06 03:49:19] nonzero exit (-1) for top3_pps4_lam0.2_temp0.94 on gpu6 pid=505899 but output exists
[2026-05-06 03:49:19] launched top2_pps4_lam0.6_temp0.94 on gpu6 pid=516794
[2026-05-06 03:49:59] nonzero exit (-1) for h2_pps2_lam0.8_temp0.94 on gpu1 pid=501553 but output exists
[2026-05-06 03:49:59] launched top3_pps4_lam0.6_temp0.94 on gpu1 pid=518105
[2026-05-06 03:49:59] finished top1_pps4_lam0.4_temp0.94 on gpu3 pid=506870
[2026-05-06 03:49:59] launched h2_pps4_lam0.6_temp0.94 on gpu3 pid=518114
[2026-05-06 03:50:14] nonzero exit (-1) for h2_pps4_lam0.2_temp0.94 on gpu2 pid=506306 but output exists
[2026-05-06 03:50:14] launched top1_pps4_lam0.8_temp0.94 on gpu2 pid=518885
[2026-05-06 03:50:29] nonzero exit (-1) for top2_pps4_lam0.4_temp0.94 on gpu5 pid=508811 but output exists
[2026-05-06 03:50:29] launched top2_pps4_lam0.8_temp0.94 on gpu5 pid=520033
[2026-05-06 03:52:44] nonzero exit (-1) for top1_pps4_lam0.6_temp0.94 on gpu7 pid=513856 but output exists
[2026-05-06 03:52:44] launched top3_pps4_lam0.8_temp0.94 on gpu7 pid=524768
[2026-05-06 03:52:54] nonzero exit (-1) for top3_pps4_lam0.4_temp0.94 on gpu0 pid=513497 but output exists
[2026-05-06 03:52:54] launched h2_pps4_lam0.8_temp0.94 on gpu0 pid=525436
[2026-05-06 03:57:53] finished h2_pps4_lam0.8_temp0.94 on gpu0 pid=525436
[2026-05-06 03:57:53] finished top3_pps4_lam0.6_temp0.94 on gpu1 pid=518105
[2026-05-06 03:57:53] finished top1_pps4_lam0.8_temp0.94 on gpu2 pid=518885
[2026-05-06 03:57:53] finished h2_pps4_lam0.6_temp0.94 on gpu3 pid=518114
[2026-05-06 03:57:53] finished h2_pps4_lam0.4_temp0.94 on gpu4 pid=513510
[2026-05-06 03:57:53] finished top2_pps4_lam0.8_temp0.94 on gpu5 pid=520033
[2026-05-06 03:57:53] finished top2_pps4_lam0.6_temp0.94 on gpu6 pid=516794
[2026-05-06 03:57:53] finished top3_pps4_lam0.8_temp0.94 on gpu7 pid=524768
[2026-05-06 03:57:53] all temp=0.94 lambda grid jobs completed
[2026-05-06 04:34:23] starting temp=0.94 lambda grid on GPUs: 0,1
[2026-05-06 04:34:23] out_root=/home/sunhc/diffusion/sad/eval/mauve_temp094_lambda_grid_8gpu
[2026-05-06 04:34:23] num_samples=1024 sample_batch_size=256 mauve_batch_size=16
[2026-05-06 04:34:23] pps_list=1 lambda_list=0.9,1.0
[2026-05-06 04:34:23] models=top1
[2026-05-06 04:34:23] launched top1_pps1_lam0.9_temp0.94 on gpu0 pid=554536
[2026-05-06 04:34:23] launched top1_pps1_lam1.0_temp0.94 on gpu1 pid=554542
[2026-05-06 04:40:48] finished top1_pps1_lam0.9_temp0.94 on gpu0 pid=554536
[2026-05-06 04:41:03] finished top1_pps1_lam1.0_temp0.94 on gpu1 pid=554542
[2026-05-06 04:41:03] all temp=0.94 lambda grid jobs completed
[2026-05-06 04:41:03] starting temp=0.94 lambda grid on GPUs: 2
[2026-05-06 04:41:03] out_root=/home/sunhc/diffusion/sad/eval/mauve_temp094_lambda_grid_8gpu
[2026-05-06 04:41:03] num_samples=1024 sample_batch_size=256 mauve_batch_size=16
[2026-05-06 04:41:03] pps_list=1 lambda_list=1.0
[2026-05-06 04:41:03] models=top2
[2026-05-06 04:41:03] launched top2_pps1_lam1.0_temp0.94 on gpu2 pid=561475
[2026-05-06 04:47:00] finished top2_pps1_lam1.0_temp0.94 on gpu2 pid=561475
[2026-05-06 04:47:00] all temp=0.94 lambda grid jobs completed
[2026-05-06 04:55:07] starting temp=0.94 lambda grid on GPUs: 3,4,5,6
[2026-05-06 04:55:07] out_root=/home/sunhc/diffusion/sad/eval/mauve_temp094_lambda_grid_8gpu
[2026-05-06 04:55:07] num_samples=1024 sample_batch_size=256 mauve_batch_size=16
[2026-05-06 04:55:07] pps_list=2,4 lambda_list=0.9,1.0
[2026-05-06 04:55:07] models=top1
[2026-05-06 04:55:07] launched top1_pps2_lam0.9_temp0.94 on gpu3 pid=589485
[2026-05-06 04:55:07] launched top1_pps2_lam1.0_temp0.94 on gpu4 pid=589491
[2026-05-06 04:55:07] launched top1_pps4_lam0.9_temp0.94 on gpu5 pid=589497
[2026-05-06 04:55:07] launched top1_pps4_lam1.0_temp0.94 on gpu6 pid=589503
[2026-05-06 05:00:03] finished top1_pps2_lam0.9_temp0.94 on gpu3 pid=589485
[2026-05-06 05:00:16] finished top1_pps2_lam1.0_temp0.94 on gpu4 pid=589491
[2026-05-06 05:00:16] finished top1_pps4_lam0.9_temp0.94 on gpu5 pid=589497
[2026-05-06 05:00:16] finished top1_pps4_lam1.0_temp0.94 on gpu6 pid=589503
[2026-05-06 05:00:16] all temp=0.94 lambda grid jobs completed
[2026-05-06 05:00:16] starting temp=0.94 lambda grid on GPUs: 3,4,5
[2026-05-06 05:00:16] out_root=/home/sunhc/diffusion/sad/eval/mauve_temp094_lambda_grid_8gpu
[2026-05-06 05:00:16] num_samples=1024 sample_batch_size=256 mauve_batch_size=16
[2026-05-06 05:00:16] pps_list=1,2,4 lambda_list=0.9
[2026-05-06 05:00:16] models=top2
[2026-05-06 05:00:16] launched top2_pps1_lam0.9_temp0.94 on gpu3 pid=596360
[2026-05-06 05:00:16] launched top2_pps2_lam0.9_temp0.94 on gpu4 pid=596366
[2026-05-06 05:00:16] launched top2_pps4_lam0.9_temp0.94 on gpu5 pid=596372
[2026-05-06 05:06:10] finished top2_pps1_lam0.9_temp0.94 on gpu3 pid=596360
[2026-05-06 05:06:10] finished top2_pps2_lam0.9_temp0.94 on gpu4 pid=596366
[2026-05-06 05:06:10] finished top2_pps4_lam0.9_temp0.94 on gpu5 pid=596372
[2026-05-06 05:06:10] all temp=0.94 lambda grid jobs completed
[2026-05-06 05:06:10] starting temp=0.94 lambda grid on GPUs: 6,7
[2026-05-06 05:06:10] out_root=/home/sunhc/diffusion/sad/eval/mauve_temp094_lambda_grid_8gpu
[2026-05-06 05:06:10] num_samples=1024 sample_batch_size=256 mauve_batch_size=16
[2026-05-06 05:06:10] pps_list=2,4 lambda_list=1.0
[2026-05-06 05:06:10] models=top2
[2026-05-06 05:06:10] launched top2_pps2_lam1.0_temp0.94 on gpu6 pid=602299
[2026-05-06 05:06:10] launched top2_pps4_lam1.0_temp0.94 on gpu7 pid=602305
[2026-05-06 05:10:49] finished top2_pps2_lam1.0_temp0.94 on gpu6 pid=602299
[2026-05-06 05:10:49] finished top2_pps4_lam1.0_temp0.94 on gpu7 pid=602305
[2026-05-06 05:10:49] all temp=0.94 lambda grid jobs completed