{ "summary": { "reference_ppl_mean": 27.426984312057495, "reference_ppl_median": 24.0395565032959, "n_samples": 1000, "model": "EleutherAI/pythia-160m", "max_new_tokens": 70, "beam_w1_equals_greedy_frac": 0.997, "reference_judge_ppl_median": 16.601248741149902 }, "results": [ { "precision": 0.156788215552466, "recall": 0.10761634910100995, "f1": 0.11682790615891263, "accuracy": 0.013836985105826744, "bleu": 0.45975591061692944, "self_ppl_mean": 2.129623557711418, "self_ppl_median": 1.9777633712182454, "distinct_1": 0.037811864720014784, "distinct_2": 0.11828823642295144, "rep_4gram": 0.3296551592928797, "avg_words": 27.055, "name": "greedy", "strategy": "greedy", "params": { "name": "greedy", "strategy": "greedy" }, "time_s": 27.15269684791565, "ms_per_prompt": 27.15269684791565, "gen_tokens_mean": 69.995, "judge_ppl_median": 3.7136934995651245, "judge_ppl_mean": 3.8431873861551287, "bertscore_p": 0.8487190008163452, "bertscore_r": 0.8463276028633118, "bertscore_f1": 0.847396731376648 }, { "precision": 0.1568385962211188, "recall": 0.10770206338672424, "f1": 0.11688205526981743, "accuracy": 0.013886833130142854, "bleu": 0.4603869452185167, "self_ppl_mean": 2.1293727944369447, "self_ppl_median": 1.977726615504968, "distinct_1": 0.03780348102435239, "distinct_2": 0.11810751697939449, "rep_4gram": 0.3296463873630552, "avg_words": 27.061, "name": "beam_w1", "strategy": "beam", "params": { "name": "beam_w1", "strategy": "beam", "width": 1 }, "time_s": 111.54685831069946, "ms_per_prompt": 111.54685831069946, "gen_tokens_mean": 69.995, "judge_ppl_median": 3.7136934995651245, "judge_ppl_mean": 3.839161422610283, "bertscore_p": 0.8486897945404053, "bertscore_r": 0.8463218808174133, "bertscore_f1": 0.8473787903785706 }, { "precision": 0.20124718334613445, "recall": 0.08660508798842184, "f1": 0.11012341441029258, "accuracy": 0.010045733414015445, "bleu": 0.29556628142570907, "self_ppl_mean": 1.8974243870210339, "self_ppl_median": 1.8302103567597376, "distinct_1": 0.045782342657342656, "distinct_2": 0.13661581137309292, "rep_4gram": 0.20882754144393487, "avg_words": 18.304, "name": "beam_w2", "strategy": "beam", "params": { "name": "beam_w2", "strategy": "beam", "width": 2 }, "time_s": 110.16097116470337, "ms_per_prompt": 110.16097116470337, "gen_tokens_mean": 69.827, "judge_ppl_median": 4.08447003364563, "judge_ppl_mean": 5.812772500276566, "bertscore_p": 0.8609820604324341, "bertscore_r": 0.8465735912322998, "bertscore_f1": 0.8535703420639038 }, { "precision": 0.1976862812247924, "recall": 0.07404912467040367, "f1": 0.09568651928951545, "accuracy": 0.008065531697039307, "bleu": 0.340259807839598, "self_ppl_mean": 1.7576852720858818, "self_ppl_median": 1.739890666494589, "distinct_1": 0.040265558374264585, "distinct_2": 0.11690534603754207, "rep_4gram": 0.23782374503889867, "avg_words": 18.527, "name": "beam_w4", "strategy": "beam", "params": { "name": "beam_w4", "strategy": "beam", "width": 4 }, "time_s": 121.36622357368469, "ms_per_prompt": 121.36622357368469, "gen_tokens_mean": 69.253, "judge_ppl_median": 3.7220958471298218, "judge_ppl_mean": 549.211596192956, "bertscore_p": 0.8594745397567749, "bertscore_r": 0.8445523977279663, "bertscore_f1": 0.851769208908081 }, { "precision": 0.20828856855773425, "recall": 0.16455016427602215, "f1": 0.16573516044843375, "accuracy": 0.015756478767379786, "bleu": 0.5589188575413241, "self_ppl_mean": 5.8339501863641905, "self_ppl_median": 5.838575948334771, "distinct_1": 0.07212438521339044, "distinct_2": 0.35802064558413893, "rep_4gram": 0.028893427447938293, "avg_words": 31.515, "name": "topk_k5", "strategy": "top_k", "params": { "name": "topk_k5", "strategy": "top_k", "k": 5 }, "time_s": 28.873475074768066, "ms_per_prompt": 28.873475074768066, "gen_tokens_mean": 69.697, "judge_ppl_median": 8.458185195922852, "judge_ppl_mean": 9.329122072458267, "bertscore_p": 0.8579514026641846, "bertscore_r": 0.8530383706092834, "bertscore_f1": 0.8553735613822937 }, { "precision": 0.1919000986912784, "recall": 0.1684283187083047, "f1": 0.1622514196784391, "accuracy": 0.013794441209097598, "bleu": 0.44804585179427503, "self_ppl_mean": 10.859201539564154, "self_ppl_median": 9.966973243847942, "distinct_1": 0.09244218671549662, "distinct_2": 0.4732998630762209, "rep_4gram": 0.004260111395545372, "avg_words": 33.859, "name": "topk_k20", "strategy": "top_k", "params": { "name": "topk_k20", "strategy": "top_k", "k": 20 }, "time_s": 28.941107749938965, "ms_per_prompt": 28.941107749938965, "gen_tokens_mean": 69.165, "judge_ppl_median": 14.761709213256836, "judge_ppl_mean": 50.361770061254504 }, { "precision": 0.17549255996014995, "recall": 0.16199041954465634, "f1": 0.15203213911919755, "accuracy": 0.011857919480724241, "bleu": 0.44096547944277925, "self_ppl_mean": 15.653030294977679, "self_ppl_median": 13.604209219791796, "distinct_1": 0.106938636169668, "distinct_2": 0.5367388758782201, "rep_4gram": 0.002577424448111978, "avg_words": 35.151, "name": "topk_k50", "strategy": "top_k", "params": { "name": "topk_k50", "strategy": "top_k", "k": 50 }, "time_s": 29.148484706878662, "ms_per_prompt": 29.148484706878662, "gen_tokens_mean": 68.635, "judge_ppl_median": 20.48440647125244, "judge_ppl_mean": 72.74511349225044 }, { "precision": 0.20069336987864045, "recall": 0.14639978279849664, "f1": 0.1519355828806502, "accuracy": 0.01273002667646736, "bleu": 0.5404882348932311, "self_ppl_mean": 7.576500820452772, "self_ppl_median": 6.75727633465444, "distinct_1": 0.09244264507422402, "distinct_2": 0.43183742591024554, "rep_4gram": 0.021025283581782708, "avg_words": 28.158, "name": "topk_k50_t0.7", "strategy": "top_k", "params": { "name": "topk_k50_t0.7", "strategy": "top_k", "k": 50, "temperature": 0.7 }, "time_s": 29.37712073326111, "ms_per_prompt": 29.37712073326111, "gen_tokens_mean": 69.563, "judge_ppl_median": 10.548301219940186, "judge_ppl_mean": 1104.2365723810196 }, { "precision": 0.21224675405441254, "recall": 0.14685590502611967, "f1": 0.15687408460192412, "accuracy": 0.013971389777601836, "bleu": 0.5397175501288309, "self_ppl_mean": 6.038022332522205, "self_ppl_median": 5.858416877967823, "distinct_1": 0.1009201217053411, "distinct_2": 0.4334855403348554, "rep_4gram": 0.02484618782785406, "avg_words": 27.279, "name": "topp_p0.5", "strategy": "top_p", "params": { "name": "topp_p0.5", "strategy": "top_p", "p": 0.5 }, "time_s": 32.68005704879761, "ms_per_prompt": 32.68005704879761, "gen_tokens_mean": 69.823, "judge_ppl_median": 9.4617280960083, "judge_ppl_mean": 1095.229723884821 }, { "precision": 0.1754558860783798, "recall": 0.15128935460540213, "f1": 0.14627172467982152, "accuracy": 0.010996779233892341, "bleu": 0.3233499868741911, "self_ppl_mean": 17.14533789994608, "self_ppl_median": 14.304176726370176, "distinct_1": 0.13880592419250545, "distinct_2": 0.5860382392174299, "rep_4gram": 0.003378257483535917, "avg_words": 32.477, "name": "topp_p0.8", "strategy": "top_p", "params": { "name": "topp_p0.8", "strategy": "top_p", "p": 0.8 }, "time_s": 32.03048324584961, "ms_per_prompt": 32.03048324584961, "gen_tokens_mean": 69.009, "judge_ppl_median": 23.211604118347168, "judge_ppl_mean": 74.37143414640427 }, { "precision": 0.15602106748659775, "recall": 0.14042348676499086, "f1": 0.13287360175699992, "accuracy": 0.00936425787467936, "bleu": 0.16536685767069867, "self_ppl_mean": 39.594243798258624, "self_ppl_median": 33.30959383748847, "distinct_1": 0.18891279361056262, "distinct_2": 0.701587687076895, "rep_4gram": 0.0007256760411977545, "avg_words": 33.931, "name": "topp_p0.95", "strategy": "top_p", "params": { "name": "topp_p0.95", "strategy": "top_p", "p": 0.95 }, "time_s": 32.22635364532471, "ms_per_prompt": 32.22635364532471, "gen_tokens_mean": 68.636, "judge_ppl_median": 54.274309158325195, "judge_ppl_mean": 297.4285540444851 }, { "precision": 0.19970724391003492, "recall": 0.14364941747564147, "f1": 0.15108641777377704, "accuracy": 0.01395932104782926, "bleu": 0.5577671411688147, "self_ppl_mean": 5.73871449144444, "self_ppl_median": 5.589571006013263, "distinct_1": 0.09601266733841946, "distinct_2": 0.408413901228116, "rep_4gram": 0.034714608915333796, "avg_words": 27.788, "name": "topp_p0.9_t0.7", "strategy": "top_p", "params": { "name": "topp_p0.9_t0.7", "strategy": "top_p", "p": 0.9, "temperature": 0.7 }, "time_s": 32.13967037200928, "ms_per_prompt": 32.13967037200928, "gen_tokens_mean": 69.865, "judge_ppl_median": 9.213471412658691, "judge_ppl_mean": 1094.454825058937 } ], "beam_timing": [ { "mode": "natural", "width": 1, "mean_s_per_prompt": 0.7966235804557801, "std_s": 0.01989696312825657, "mean_ms_per_token": 11.380336863654001, "mean_output_tokens": 70.0, "total_s": 79.662358045578, "n_prompts": 100 }, { "mode": "natural", "width": 2, "mean_s_per_prompt": 0.8142524075508117, "std_s": 0.024042614408421762, "mean_ms_per_token": 11.656413194421047, "mean_output_tokens": 69.86, "total_s": 81.42524075508118, "n_prompts": 100 }, { "mode": "natural", "width": 4, "mean_s_per_prompt": 0.8594671440124512, "std_s": 0.04051314764123496, "mean_ms_per_token": 12.351310080320381, "mean_output_tokens": 69.59, "total_s": 85.94671440124512, "n_prompts": 100 }, { "mode": "natural", "width": 8, "mean_s_per_prompt": 0.9159316921234131, "std_s": 0.04308222295630671, "mean_ms_per_token": 13.236482083315467, "mean_output_tokens": 69.34, "total_s": 91.59316921234131, "n_prompts": 100 }, { "mode": "fixed", "width": 1, "mean_s_per_prompt": 0.793733959197998, "std_s": 0.01947922482082487, "mean_ms_per_token": 11.3390565599714, "mean_output_tokens": 70.0, "total_s": 79.3733959197998, "n_prompts": 100 }, { "mode": "fixed", "width": 2, "mean_s_per_prompt": 0.8047142720222473, "std_s": 0.01562980802083294, "mean_ms_per_token": 11.495918171746387, "mean_output_tokens": 70.0, "total_s": 80.47142720222473, "n_prompts": 100 }, { "mode": "fixed", "width": 4, "mean_s_per_prompt": 0.8514944720268249, "std_s": 0.014158784290349734, "mean_ms_per_token": 12.164206743240355, "mean_output_tokens": 70.0, "total_s": 85.1494472026825, "n_prompts": 100 }, { "mode": "fixed", "width": 8, "mean_s_per_prompt": 0.8935490798950195, "std_s": 0.015420650987721442, "mean_ms_per_token": 12.764986855643139, "mean_output_tokens": 70.0, "total_s": 89.35490798950195, "n_prompts": 100 } ], "config": { "model_name": "EleutherAI/pythia-160m", "dataset": "hamishivi/ROCStories", "split": "test", "n_samples": 1000, "seed": 0, "max_new_tokens": 70, "batch_size": 32, "beam_batch_size": 8, "dtype": "float32", "length_penalty": 1.0, "configs": [ { "name": "greedy", "strategy": "greedy" }, { "name": "beam_w1", "strategy": "beam", "width": 1 }, { "name": "beam_w2", "strategy": "beam", "width": 2 }, { "name": "beam_w4", "strategy": "beam", "width": 4 }, { "name": "topk_k5", "strategy": "top_k", "k": 5 }, { "name": "topk_k20", "strategy": "top_k", "k": 20 }, { "name": "topk_k50", "strategy": "top_k", "k": 50 }, { "name": "topk_k50_t0.7", "strategy": "top_k", "k": 50, "temperature": 0.7 }, { "name": "topp_p0.5", "strategy": "top_p", "p": 0.5 }, { "name": "topp_p0.8", "strategy": "top_p", "p": 0.8 }, { "name": "topp_p0.95", "strategy": "top_p", "p": 0.95 }, { "name": "topp_p0.9_t0.7", "strategy": "top_p", "p": 0.9, "temperature": 0.7 } ], "timing_widths": [ 1, 2, 4, 8 ], "timing_samples": 100, "judge_model": "EleutherAI/pythia-410m", "use_bertscore": true, "out_dir": "/kaggle/working/outputs/decoding", "hf_user": null, "repo_name": "anlp-a2-decoding", "push_to_hub": true, "private": false } }