| { |
| "summary": { |
| "reference_ppl_mean": 27.426984312057495, |
| "reference_ppl_median": 24.0395565032959, |
| "n_samples": 1000, |
| "model": "EleutherAI/pythia-160m", |
| "max_new_tokens": 70, |
| "beam_w1_equals_greedy_frac": 0.997, |
| "reference_judge_ppl_median": 16.601248741149902 |
| }, |
| "results": [ |
| { |
| "precision": 0.156788215552466, |
| "recall": 0.10761634910100995, |
| "f1": 0.11682790615891263, |
| "accuracy": 0.013836985105826744, |
| "bleu": 0.45975591061692944, |
| "self_ppl_mean": 2.129623557711418, |
| "self_ppl_median": 1.9777633712182454, |
| "distinct_1": 0.037811864720014784, |
| "distinct_2": 0.11828823642295144, |
| "rep_4gram": 0.3296551592928797, |
| "avg_words": 27.055, |
| "name": "greedy", |
| "strategy": "greedy", |
| "params": { |
| "name": "greedy", |
| "strategy": "greedy" |
| }, |
| "time_s": 27.15269684791565, |
| "ms_per_prompt": 27.15269684791565, |
| "gen_tokens_mean": 69.995, |
| "judge_ppl_median": 3.7136934995651245, |
| "judge_ppl_mean": 3.8431873861551287, |
| "bertscore_p": 0.8487190008163452, |
| "bertscore_r": 0.8463276028633118, |
| "bertscore_f1": 0.847396731376648 |
| }, |
| { |
| "precision": 0.1568385962211188, |
| "recall": 0.10770206338672424, |
| "f1": 0.11688205526981743, |
| "accuracy": 0.013886833130142854, |
| "bleu": 0.4603869452185167, |
| "self_ppl_mean": 2.1293727944369447, |
| "self_ppl_median": 1.977726615504968, |
| "distinct_1": 0.03780348102435239, |
| "distinct_2": 0.11810751697939449, |
| "rep_4gram": 0.3296463873630552, |
| "avg_words": 27.061, |
| "name": "beam_w1", |
| "strategy": "beam", |
| "params": { |
| "name": "beam_w1", |
| "strategy": "beam", |
| "width": 1 |
| }, |
| "time_s": 111.54685831069946, |
| "ms_per_prompt": 111.54685831069946, |
| "gen_tokens_mean": 69.995, |
| "judge_ppl_median": 3.7136934995651245, |
| "judge_ppl_mean": 3.839161422610283, |
| "bertscore_p": 0.8486897945404053, |
| "bertscore_r": 0.8463218808174133, |
| "bertscore_f1": 0.8473787903785706 |
| }, |
| { |
| "precision": 0.20124718334613445, |
| "recall": 0.08660508798842184, |
| "f1": 0.11012341441029258, |
| "accuracy": 0.010045733414015445, |
| "bleu": 0.29556628142570907, |
| "self_ppl_mean": 1.8974243870210339, |
| "self_ppl_median": 1.8302103567597376, |
| "distinct_1": 0.045782342657342656, |
| "distinct_2": 0.13661581137309292, |
| "rep_4gram": 0.20882754144393487, |
| "avg_words": 18.304, |
| "name": "beam_w2", |
| "strategy": "beam", |
| "params": { |
| "name": "beam_w2", |
| "strategy": "beam", |
| "width": 2 |
| }, |
| "time_s": 110.16097116470337, |
| "ms_per_prompt": 110.16097116470337, |
| "gen_tokens_mean": 69.827, |
| "judge_ppl_median": 4.08447003364563, |
| "judge_ppl_mean": 5.812772500276566, |
| "bertscore_p": 0.8609820604324341, |
| "bertscore_r": 0.8465735912322998, |
| "bertscore_f1": 0.8535703420639038 |
| }, |
| { |
| "precision": 0.1976862812247924, |
| "recall": 0.07404912467040367, |
| "f1": 0.09568651928951545, |
| "accuracy": 0.008065531697039307, |
| "bleu": 0.340259807839598, |
| "self_ppl_mean": 1.7576852720858818, |
| "self_ppl_median": 1.739890666494589, |
| "distinct_1": 0.040265558374264585, |
| "distinct_2": 0.11690534603754207, |
| "rep_4gram": 0.23782374503889867, |
| "avg_words": 18.527, |
| "name": "beam_w4", |
| "strategy": "beam", |
| "params": { |
| "name": "beam_w4", |
| "strategy": "beam", |
| "width": 4 |
| }, |
| "time_s": 121.36622357368469, |
| "ms_per_prompt": 121.36622357368469, |
| "gen_tokens_mean": 69.253, |
| "judge_ppl_median": 3.7220958471298218, |
| "judge_ppl_mean": 549.211596192956, |
| "bertscore_p": 0.8594745397567749, |
| "bertscore_r": 0.8445523977279663, |
| "bertscore_f1": 0.851769208908081 |
| }, |
| { |
| "precision": 0.20828856855773425, |
| "recall": 0.16455016427602215, |
| "f1": 0.16573516044843375, |
| "accuracy": 0.015756478767379786, |
| "bleu": 0.5589188575413241, |
| "self_ppl_mean": 5.8339501863641905, |
| "self_ppl_median": 5.838575948334771, |
| "distinct_1": 0.07212438521339044, |
| "distinct_2": 0.35802064558413893, |
| "rep_4gram": 0.028893427447938293, |
| "avg_words": 31.515, |
| "name": "topk_k5", |
| "strategy": "top_k", |
| "params": { |
| "name": "topk_k5", |
| "strategy": "top_k", |
| "k": 5 |
| }, |
| "time_s": 28.873475074768066, |
| "ms_per_prompt": 28.873475074768066, |
| "gen_tokens_mean": 69.697, |
| "judge_ppl_median": 8.458185195922852, |
| "judge_ppl_mean": 9.329122072458267, |
| "bertscore_p": 0.8579514026641846, |
| "bertscore_r": 0.8530383706092834, |
| "bertscore_f1": 0.8553735613822937 |
| }, |
| { |
| "precision": 0.1919000986912784, |
| "recall": 0.1684283187083047, |
| "f1": 0.1622514196784391, |
| "accuracy": 0.013794441209097598, |
| "bleu": 0.44804585179427503, |
| "self_ppl_mean": 10.859201539564154, |
| "self_ppl_median": 9.966973243847942, |
| "distinct_1": 0.09244218671549662, |
| "distinct_2": 0.4732998630762209, |
| "rep_4gram": 0.004260111395545372, |
| "avg_words": 33.859, |
| "name": "topk_k20", |
| "strategy": "top_k", |
| "params": { |
| "name": "topk_k20", |
| "strategy": "top_k", |
| "k": 20 |
| }, |
| "time_s": 28.941107749938965, |
| "ms_per_prompt": 28.941107749938965, |
| "gen_tokens_mean": 69.165, |
| "judge_ppl_median": 14.761709213256836, |
| "judge_ppl_mean": 50.361770061254504 |
| }, |
| { |
| "precision": 0.17549255996014995, |
| "recall": 0.16199041954465634, |
| "f1": 0.15203213911919755, |
| "accuracy": 0.011857919480724241, |
| "bleu": 0.44096547944277925, |
| "self_ppl_mean": 15.653030294977679, |
| "self_ppl_median": 13.604209219791796, |
| "distinct_1": 0.106938636169668, |
| "distinct_2": 0.5367388758782201, |
| "rep_4gram": 0.002577424448111978, |
| "avg_words": 35.151, |
| "name": "topk_k50", |
| "strategy": "top_k", |
| "params": { |
| "name": "topk_k50", |
| "strategy": "top_k", |
| "k": 50 |
| }, |
| "time_s": 29.148484706878662, |
| "ms_per_prompt": 29.148484706878662, |
| "gen_tokens_mean": 68.635, |
| "judge_ppl_median": 20.48440647125244, |
| "judge_ppl_mean": 72.74511349225044 |
| }, |
| { |
| "precision": 0.20069336987864045, |
| "recall": 0.14639978279849664, |
| "f1": 0.1519355828806502, |
| "accuracy": 0.01273002667646736, |
| "bleu": 0.5404882348932311, |
| "self_ppl_mean": 7.576500820452772, |
| "self_ppl_median": 6.75727633465444, |
| "distinct_1": 0.09244264507422402, |
| "distinct_2": 0.43183742591024554, |
| "rep_4gram": 0.021025283581782708, |
| "avg_words": 28.158, |
| "name": "topk_k50_t0.7", |
| "strategy": "top_k", |
| "params": { |
| "name": "topk_k50_t0.7", |
| "strategy": "top_k", |
| "k": 50, |
| "temperature": 0.7 |
| }, |
| "time_s": 29.37712073326111, |
| "ms_per_prompt": 29.37712073326111, |
| "gen_tokens_mean": 69.563, |
| "judge_ppl_median": 10.548301219940186, |
| "judge_ppl_mean": 1104.2365723810196 |
| }, |
| { |
| "precision": 0.21224675405441254, |
| "recall": 0.14685590502611967, |
| "f1": 0.15687408460192412, |
| "accuracy": 0.013971389777601836, |
| "bleu": 0.5397175501288309, |
| "self_ppl_mean": 6.038022332522205, |
| "self_ppl_median": 5.858416877967823, |
| "distinct_1": 0.1009201217053411, |
| "distinct_2": 0.4334855403348554, |
| "rep_4gram": 0.02484618782785406, |
| "avg_words": 27.279, |
| "name": "topp_p0.5", |
| "strategy": "top_p", |
| "params": { |
| "name": "topp_p0.5", |
| "strategy": "top_p", |
| "p": 0.5 |
| }, |
| "time_s": 32.68005704879761, |
| "ms_per_prompt": 32.68005704879761, |
| "gen_tokens_mean": 69.823, |
| "judge_ppl_median": 9.4617280960083, |
| "judge_ppl_mean": 1095.229723884821 |
| }, |
| { |
| "precision": 0.1754558860783798, |
| "recall": 0.15128935460540213, |
| "f1": 0.14627172467982152, |
| "accuracy": 0.010996779233892341, |
| "bleu": 0.3233499868741911, |
| "self_ppl_mean": 17.14533789994608, |
| "self_ppl_median": 14.304176726370176, |
| "distinct_1": 0.13880592419250545, |
| "distinct_2": 0.5860382392174299, |
| "rep_4gram": 0.003378257483535917, |
| "avg_words": 32.477, |
| "name": "topp_p0.8", |
| "strategy": "top_p", |
| "params": { |
| "name": "topp_p0.8", |
| "strategy": "top_p", |
| "p": 0.8 |
| }, |
| "time_s": 32.03048324584961, |
| "ms_per_prompt": 32.03048324584961, |
| "gen_tokens_mean": 69.009, |
| "judge_ppl_median": 23.211604118347168, |
| "judge_ppl_mean": 74.37143414640427 |
| }, |
| { |
| "precision": 0.15602106748659775, |
| "recall": 0.14042348676499086, |
| "f1": 0.13287360175699992, |
| "accuracy": 0.00936425787467936, |
| "bleu": 0.16536685767069867, |
| "self_ppl_mean": 39.594243798258624, |
| "self_ppl_median": 33.30959383748847, |
| "distinct_1": 0.18891279361056262, |
| "distinct_2": 0.701587687076895, |
| "rep_4gram": 0.0007256760411977545, |
| "avg_words": 33.931, |
| "name": "topp_p0.95", |
| "strategy": "top_p", |
| "params": { |
| "name": "topp_p0.95", |
| "strategy": "top_p", |
| "p": 0.95 |
| }, |
| "time_s": 32.22635364532471, |
| "ms_per_prompt": 32.22635364532471, |
| "gen_tokens_mean": 68.636, |
| "judge_ppl_median": 54.274309158325195, |
| "judge_ppl_mean": 297.4285540444851 |
| }, |
| { |
| "precision": 0.19970724391003492, |
| "recall": 0.14364941747564147, |
| "f1": 0.15108641777377704, |
| "accuracy": 0.01395932104782926, |
| "bleu": 0.5577671411688147, |
| "self_ppl_mean": 5.73871449144444, |
| "self_ppl_median": 5.589571006013263, |
| "distinct_1": 0.09601266733841946, |
| "distinct_2": 0.408413901228116, |
| "rep_4gram": 0.034714608915333796, |
| "avg_words": 27.788, |
| "name": "topp_p0.9_t0.7", |
| "strategy": "top_p", |
| "params": { |
| "name": "topp_p0.9_t0.7", |
| "strategy": "top_p", |
| "p": 0.9, |
| "temperature": 0.7 |
| }, |
| "time_s": 32.13967037200928, |
| "ms_per_prompt": 32.13967037200928, |
| "gen_tokens_mean": 69.865, |
| "judge_ppl_median": 9.213471412658691, |
| "judge_ppl_mean": 1094.454825058937 |
| } |
| ], |
| "beam_timing": [ |
| { |
| "mode": "natural", |
| "width": 1, |
| "mean_s_per_prompt": 0.7966235804557801, |
| "std_s": 0.01989696312825657, |
| "mean_ms_per_token": 11.380336863654001, |
| "mean_output_tokens": 70.0, |
| "total_s": 79.662358045578, |
| "n_prompts": 100 |
| }, |
| { |
| "mode": "natural", |
| "width": 2, |
| "mean_s_per_prompt": 0.8142524075508117, |
| "std_s": 0.024042614408421762, |
| "mean_ms_per_token": 11.656413194421047, |
| "mean_output_tokens": 69.86, |
| "total_s": 81.42524075508118, |
| "n_prompts": 100 |
| }, |
| { |
| "mode": "natural", |
| "width": 4, |
| "mean_s_per_prompt": 0.8594671440124512, |
| "std_s": 0.04051314764123496, |
| "mean_ms_per_token": 12.351310080320381, |
| "mean_output_tokens": 69.59, |
| "total_s": 85.94671440124512, |
| "n_prompts": 100 |
| }, |
| { |
| "mode": "natural", |
| "width": 8, |
| "mean_s_per_prompt": 0.9159316921234131, |
| "std_s": 0.04308222295630671, |
| "mean_ms_per_token": 13.236482083315467, |
| "mean_output_tokens": 69.34, |
| "total_s": 91.59316921234131, |
| "n_prompts": 100 |
| }, |
| { |
| "mode": "fixed", |
| "width": 1, |
| "mean_s_per_prompt": 0.793733959197998, |
| "std_s": 0.01947922482082487, |
| "mean_ms_per_token": 11.3390565599714, |
| "mean_output_tokens": 70.0, |
| "total_s": 79.3733959197998, |
| "n_prompts": 100 |
| }, |
| { |
| "mode": "fixed", |
| "width": 2, |
| "mean_s_per_prompt": 0.8047142720222473, |
| "std_s": 0.01562980802083294, |
| "mean_ms_per_token": 11.495918171746387, |
| "mean_output_tokens": 70.0, |
| "total_s": 80.47142720222473, |
| "n_prompts": 100 |
| }, |
| { |
| "mode": "fixed", |
| "width": 4, |
| "mean_s_per_prompt": 0.8514944720268249, |
| "std_s": 0.014158784290349734, |
| "mean_ms_per_token": 12.164206743240355, |
| "mean_output_tokens": 70.0, |
| "total_s": 85.1494472026825, |
| "n_prompts": 100 |
| }, |
| { |
| "mode": "fixed", |
| "width": 8, |
| "mean_s_per_prompt": 0.8935490798950195, |
| "std_s": 0.015420650987721442, |
| "mean_ms_per_token": 12.764986855643139, |
| "mean_output_tokens": 70.0, |
| "total_s": 89.35490798950195, |
| "n_prompts": 100 |
| } |
| ], |
| "config": { |
| "model_name": "EleutherAI/pythia-160m", |
| "dataset": "hamishivi/ROCStories", |
| "split": "test", |
| "n_samples": 1000, |
| "seed": 0, |
| "max_new_tokens": 70, |
| "batch_size": 32, |
| "beam_batch_size": 8, |
| "dtype": "float32", |
| "length_penalty": 1.0, |
| "configs": [ |
| { |
| "name": "greedy", |
| "strategy": "greedy" |
| }, |
| { |
| "name": "beam_w1", |
| "strategy": "beam", |
| "width": 1 |
| }, |
| { |
| "name": "beam_w2", |
| "strategy": "beam", |
| "width": 2 |
| }, |
| { |
| "name": "beam_w4", |
| "strategy": "beam", |
| "width": 4 |
| }, |
| { |
| "name": "topk_k5", |
| "strategy": "top_k", |
| "k": 5 |
| }, |
| { |
| "name": "topk_k20", |
| "strategy": "top_k", |
| "k": 20 |
| }, |
| { |
| "name": "topk_k50", |
| "strategy": "top_k", |
| "k": 50 |
| }, |
| { |
| "name": "topk_k50_t0.7", |
| "strategy": "top_k", |
| "k": 50, |
| "temperature": 0.7 |
| }, |
| { |
| "name": "topp_p0.5", |
| "strategy": "top_p", |
| "p": 0.5 |
| }, |
| { |
| "name": "topp_p0.8", |
| "strategy": "top_p", |
| "p": 0.8 |
| }, |
| { |
| "name": "topp_p0.95", |
| "strategy": "top_p", |
| "p": 0.95 |
| }, |
| { |
| "name": "topp_p0.9_t0.7", |
| "strategy": "top_p", |
| "p": 0.9, |
| "temperature": 0.7 |
| } |
| ], |
| "timing_widths": [ |
| 1, |
| 2, |
| 4, |
| 8 |
| ], |
| "timing_samples": 100, |
| "judge_model": "EleutherAI/pythia-410m", |
| "use_bertscore": true, |
| "out_dir": "/kaggle/working/outputs/decoding", |
| "hf_user": null, |
| "repo_name": "anlp-a2-decoding", |
| "push_to_hub": true, |
| "private": false |
| } |
| } |
