decoding results

common.py

@@ -792,7 +792,8 @@ def launch_parallel(jobs, poll_s=180, tail=3):
     assert len(jobs) <= n_gpu, f"{len(jobs)} jobs but only {n_gpu} GPU(s)"
     procs = []
     for gpu, (name, code) in enumerate(jobs):
-        env = dict(os.environ, CUDA_VISIBLE_DEVICES=str(gpu), PYTHONUNBUFFERED="1")
+        env = dict(os.environ, CUDA_VISIBLE_DEVICES=str(gpu), PYTHONUNBUFFERED="1",
+                   PYTORCH_CUDA_ALLOC_CONF="expandable_segments:True")
         logf = open(f"{name}.log", "w")
         procs.append((name, subprocess.Popen([sys.executable, "-u", "-c", code], env=env,
                                              stdout=logf, stderr=subprocess.STDOUT), logf))

decoding_results.json

@@ -27,8 +27,8 @@
         "name": "greedy",
         "strategy": "greedy"
       },
-      "time_s": 26.33022141456604,
-      "ms_per_prompt": 26.33022141456604,
+      "time_s": 27.15269684791565,
+      "ms_per_prompt": 27.15269684791565,
       "gen_tokens_mean": 69.995,
       "judge_ppl_median": 3.7136934995651245,
       "judge_ppl_mean": 3.8431873861551287,
@@ -55,8 +55,8 @@
         "strategy": "beam",
         "width": 1
       },
-      "time_s": 110.271555185318,
-      "ms_per_prompt": 110.271555185318,
+      "time_s": 111.54685831069946,
+      "ms_per_prompt": 111.54685831069946,
       "gen_tokens_mean": 69.995,
       "judge_ppl_median": 3.7136934995651245,
       "judge_ppl_mean": 3.839161422610283,
@@ -83,8 +83,8 @@
         "strategy": "beam",
         "width": 2
       },
-      "time_s": 107.95216870307922,
-      "ms_per_prompt": 107.95216870307922,
+      "time_s": 110.16097116470337,
+      "ms_per_prompt": 110.16097116470337,
       "gen_tokens_mean": 69.827,
       "judge_ppl_median": 4.08447003364563,
       "judge_ppl_mean": 5.812772500276566,
@@ -111,12 +111,12 @@
         "strategy": "beam",
         "width": 4
       },
-      "time_s": 118.95717024803162,
-      "ms_per_prompt": 118.95717024803162,
+      "time_s": 121.36622357368469,
+      "ms_per_prompt": 121.36622357368469,
       "gen_tokens_mean": 69.253,
       "judge_ppl_median": 3.7220958471298218,
       "judge_ppl_mean": 549.211596192956,
-      "bertscore_p": 0.8594744801521301,
+      "bertscore_p": 0.8594745397567749,
       "bertscore_r": 0.8445523977279663,
       "bertscore_f1": 0.851769208908081
     },
@@ -139,8 +139,8 @@
         "strategy": "top_k",
         "k": 5
       },
-      "time_s": 28.06767702102661,
-      "ms_per_prompt": 28.06767702102661,
+      "time_s": 28.873475074768066,
+      "ms_per_prompt": 28.873475074768066,
       "gen_tokens_mean": 69.697,
       "judge_ppl_median": 8.458185195922852,
       "judge_ppl_mean": 9.329122072458267,
@@ -167,8 +167,8 @@
         "strategy": "top_k",
         "k": 20
       },
-      "time_s": 27.953922510147095,
-      "ms_per_prompt": 27.953922510147095,
+      "time_s": 28.941107749938965,
+      "ms_per_prompt": 28.941107749938965,
       "gen_tokens_mean": 69.165,
       "judge_ppl_median": 14.761709213256836,
       "judge_ppl_mean": 50.361770061254504
@@ -192,8 +192,8 @@
         "strategy": "top_k",
         "k": 50
       },
-      "time_s": 27.92880129814148,
-      "ms_per_prompt": 27.92880129814148,
+      "time_s": 29.148484706878662,
+      "ms_per_prompt": 29.148484706878662,
       "gen_tokens_mean": 68.635,
       "judge_ppl_median": 20.48440647125244,
       "judge_ppl_mean": 72.74511349225044
@@ -218,8 +218,8 @@
         "k": 50,
         "temperature": 0.7
       },
-      "time_s": 27.997068405151367,
-      "ms_per_prompt": 27.997068405151367,
+      "time_s": 29.37712073326111,
+      "ms_per_prompt": 29.37712073326111,
       "gen_tokens_mean": 69.563,
       "judge_ppl_median": 10.548301219940186,
       "judge_ppl_mean": 1104.2365723810196
@@ -243,8 +243,8 @@
         "strategy": "top_p",
         "p": 0.5
       },
-      "time_s": 31.63517665863037,
-      "ms_per_prompt": 31.635176658630375,
+      "time_s": 32.68005704879761,
+      "ms_per_prompt": 32.68005704879761,
       "gen_tokens_mean": 69.823,
       "judge_ppl_median": 9.4617280960083,
       "judge_ppl_mean": 1095.229723884821
@@ -268,8 +268,8 @@
         "strategy": "top_p",
         "p": 0.8
       },
-      "time_s": 31.48725962638855,
-      "ms_per_prompt": 31.48725962638855,
+      "time_s": 32.03048324584961,
+      "ms_per_prompt": 32.03048324584961,
       "gen_tokens_mean": 69.009,
       "judge_ppl_median": 23.211604118347168,
       "judge_ppl_mean": 74.37143414640427
@@ -293,8 +293,8 @@
         "strategy": "top_p",
         "p": 0.95
       },
-      "time_s": 31.551427125930786,
-      "ms_per_prompt": 31.551427125930786,
+      "time_s": 32.22635364532471,
+      "ms_per_prompt": 32.22635364532471,
       "gen_tokens_mean": 68.636,
       "judge_ppl_median": 54.274309158325195,
       "judge_ppl_mean": 297.4285540444851
@@ -319,8 +319,8 @@
         "p": 0.9,
         "temperature": 0.7
       },
-      "time_s": 31.56927514076233,
-      "ms_per_prompt": 31.569275140762326,
+      "time_s": 32.13967037200928,
+      "ms_per_prompt": 32.13967037200928,
       "gen_tokens_mean": 69.865,
       "judge_ppl_median": 9.213471412658691,
       "judge_ppl_mean": 1094.454825058937
@@ -330,81 +330,81 @@
     {
       "mode": "natural",
       "width": 1,
-      "mean_s_per_prompt": 0.781981463432312,
-      "std_s": 0.01558768772931337,
-      "mean_ms_per_token": 11.171163763318747,
+      "mean_s_per_prompt": 0.7966235804557801,
+      "std_s": 0.01989696312825657,
+      "mean_ms_per_token": 11.380336863654001,
       "mean_output_tokens": 70.0,
-      "total_s": 78.1981463432312,
+      "total_s": 79.662358045578,
       "n_prompts": 100
     },
     {
       "mode": "natural",
       "width": 2,
-      "mean_s_per_prompt": 0.793917977809906,
-      "std_s": 0.022438927991426688,
-      "mean_ms_per_token": 11.364886324126998,
+      "mean_s_per_prompt": 0.8142524075508117,
+      "std_s": 0.024042614408421762,
+      "mean_ms_per_token": 11.656413194421047,
       "mean_output_tokens": 69.86,
-      "total_s": 79.3917977809906,
+      "total_s": 81.42524075508118,
       "n_prompts": 100
     },
     {
       "mode": "natural",
       "width": 4,
-      "mean_s_per_prompt": 0.8327497005462646,
-      "std_s": 0.033384079158313755,
-      "mean_ms_per_token": 11.970075382877122,
+      "mean_s_per_prompt": 0.8594671440124512,
+      "std_s": 0.04051314764123496,
+      "mean_ms_per_token": 12.351310080320381,
       "mean_output_tokens": 69.59,
-      "total_s": 83.27497005462646,
+      "total_s": 85.94671440124512,
       "n_prompts": 100
     },
     {
       "mode": "natural",
       "width": 8,
-      "mean_s_per_prompt": 0.8716441249847412,
-      "std_s": 0.043158742726592765,
-      "mean_ms_per_token": 12.592297802702353,
+      "mean_s_per_prompt": 0.9159316921234131,
+      "std_s": 0.04308222295630671,
+      "mean_ms_per_token": 13.236482083315467,
       "mean_output_tokens": 69.34,
-      "total_s": 87.16441249847412,
+      "total_s": 91.59316921234131,
       "n_prompts": 100
     },
     {
       "mode": "fixed",
       "width": 1,
-      "mean_s_per_prompt": 0.7738243556022644,
-      "std_s": 0.01421080819544866,
-      "mean_ms_per_token": 11.054633651460922,
+      "mean_s_per_prompt": 0.793733959197998,
+      "std_s": 0.01947922482082487,
+      "mean_ms_per_token": 11.3390565599714,
       "mean_output_tokens": 70.0,
-      "total_s": 77.38243556022644,
+      "total_s": 79.3733959197998,
       "n_prompts": 100
     },
     {
       "mode": "fixed",
       "width": 2,
-      "mean_s_per_prompt": 0.7971447992324829,
-      "std_s": 0.02233234077874458,
-      "mean_ms_per_token": 11.387782846178329,
+      "mean_s_per_prompt": 0.8047142720222473,
+      "std_s": 0.01562980802083294,
+      "mean_ms_per_token": 11.495918171746387,
       "mean_output_tokens": 70.0,
-      "total_s": 79.71447992324829,
+      "total_s": 80.47142720222473,
       "n_prompts": 100
     },
     {
       "mode": "fixed",
       "width": 4,
-      "mean_s_per_prompt": 0.8367985081672669,
-      "std_s": 0.013029678554817858,
-      "mean_ms_per_token": 11.954264402389526,
+      "mean_s_per_prompt": 0.8514944720268249,
+      "std_s": 0.014158784290349734,
+      "mean_ms_per_token": 12.164206743240355,
       "mean_output_tokens": 70.0,
-      "total_s": 83.67985081672668,
+      "total_s": 85.1494472026825,
       "n_prompts": 100
     },
     {
       "mode": "fixed",
       "width": 8,
-      "mean_s_per_prompt": 0.8789880871772766,
-      "std_s": 0.012277801042479799,
-      "mean_ms_per_token": 12.556972673961095,
+      "mean_s_per_prompt": 0.8935490798950195,
+      "std_s": 0.015420650987721442,
+      "mean_ms_per_token": 12.764986855643139,
       "mean_output_tokens": 70.0,
-      "total_s": 87.89880871772766,
+      "total_s": 89.35490798950195,
       "n_prompts": 100
     }
   ],