Upload folder using huggingface_hub

m14/benchmark_results.json

@@ -0,0 +1,540 @@
+{
+  "model_id": "m14",
+  "checkpoint": "results/m14/final.pt",
+  "trained_step": 15258,
+  "num_fewshot": 0,
+  "limit": null,
+  "results": {
+    "lambada_openai": {
+      "name": "lambada_openai",
+      "alias": "lambada_openai",
+      "sample_len": 5153,
+      "perplexity,none": 404.5905237231641,
+      "perplexity_stderr,none": 21.19147790393626,
+      "acc,none": 0.18105957694546865,
+      "acc_stderr,none": 0.005364743691197477
+    },
+    "hellaswag": {
+      "name": "hellaswag",
+      "alias": "hellaswag",
+      "sample_len": 10042,
+      "acc,none": 0.2801234813782115,
+      "acc_stderr,none": 0.004481416180499928,
+      "acc_norm,none": 0.293168691495718,
+      "acc_norm_stderr,none": 0.0045428479356060215
+    },
+    "piqa": {
+      "name": "piqa",
+      "alias": "piqa",
+      "sample_len": 1838,
+      "acc,none": 0.6093579978237215,
+      "acc_stderr,none": 0.011383377760053567,
+      "acc_norm,none": 0.6066376496191512,
+      "acc_norm_stderr,none": 0.011397418546689315
+    },
+    "sciq": {
+      "name": "sciq",
+      "alias": "sciq",
+      "sample_len": 1000,
+      "acc,none": 0.685,
+      "acc_stderr,none": 0.014696631960792617,
+      "acc_norm,none": 0.594,
+      "acc_norm_stderr,none": 0.01553722643863458
+    },
+    "arc_easy": {
+      "name": "arc_easy",
+      "alias": "arc_easy",
+      "sample_len": 2376,
+      "acc,none": 0.4654882154882155,
+      "acc_stderr,none": 0.010235314238969277,
+      "acc_norm,none": 0.4313973063973064,
+      "acc_norm_stderr,none": 0.010162752847747605
+    },
+    "winogrande": {
+      "name": "winogrande",
+      "alias": "winogrande",
+      "sample_len": 1267,
+      "acc,none": 0.5122336227308603,
+      "acc_stderr,none": 0.014048278820405729
+    },
+    "blimp_adjunct_island": {
+      "name": "blimp_adjunct_island",
+      "alias": "blimp_adjunct_island",
+      "sample_len": 1000,
+      "acc,none": 0.804,
+      "acc_stderr,none": 0.012559527926707347
+    },
+    "blimp_anaphor_gender_agreement": {
+      "name": "blimp_anaphor_gender_agreement",
+      "alias": "blimp_anaphor_gender_agreement",
+      "sample_len": 1000,
+      "acc,none": 0.962,
+      "acc_stderr,none": 0.0060491811505849775
+    },
+    "blimp_anaphor_number_agreement": {
+      "name": "blimp_anaphor_number_agreement",
+      "alias": "blimp_anaphor_number_agreement",
+      "sample_len": 1000,
+      "acc,none": 0.984,
+      "acc_stderr,none": 0.003969856390319398
+    },
+    "blimp_animate_subject_passive": {
+      "name": "blimp_animate_subject_passive",
+      "alias": "blimp_animate_subject_passive",
+      "sample_len": 1000,
+      "acc,none": 0.719,
+      "acc_stderr,none": 0.01422115470843494
+    },
+    "blimp_animate_subject_trans": {
+      "name": "blimp_animate_subject_trans",
+      "alias": "blimp_animate_subject_trans",
+      "sample_len": 1000,
+      "acc,none": 0.907,
+      "acc_stderr,none": 0.009188875634996669
+    },
+    "blimp_causative": {
+      "name": "blimp_causative",
+      "alias": "blimp_causative",
+      "sample_len": 1000,
+      "acc,none": 0.749,
+      "acc_stderr,none": 0.013718133516888775
+    },
+    "blimp_complex_NP_island": {
+      "name": "blimp_complex_NP_island",
+      "alias": "blimp_complex_NP_island",
+      "sample_len": 1000,
+      "acc,none": 0.522,
+      "acc_stderr,none": 0.01580397942816194
+    },
+    "blimp_coordinate_structure_constraint_complex_left_branch": {
+      "name": "blimp_coordinate_structure_constraint_complex_left_branch",
+      "alias": "blimp_coordinate_structure_constraint_complex_left_branch",
+      "sample_len": 1000,
+      "acc,none": 0.511,
+      "acc_stderr,none": 0.01581547119529257
+    },
+    "blimp_coordinate_structure_constraint_object_extraction": {
+      "name": "blimp_coordinate_structure_constraint_object_extraction",
+      "alias": "blimp_coordinate_structure_constraint_object_extraction",
+      "sample_len": 1000,
+      "acc,none": 0.816,
+      "acc_stderr,none": 0.01225945734093864
+    },
+    "blimp_determiner_noun_agreement_1": {
+      "name": "blimp_determiner_noun_agreement_1",
+      "alias": "blimp_determiner_noun_agreement_1",
+      "sample_len": 1000,
+      "acc,none": 0.976,
+      "acc_stderr,none": 0.004842256441727084
+    },
+    "blimp_determiner_noun_agreement_2": {
+      "name": "blimp_determiner_noun_agreement_2",
+      "alias": "blimp_determiner_noun_agreement_2",
+      "sample_len": 1000,
+      "acc,none": 0.954,
+      "acc_stderr,none": 0.006627814717380679
+    },
+    "blimp_determiner_noun_agreement_irregular_1": {
+      "name": "blimp_determiner_noun_agreement_irregular_1",
+      "alias": "blimp_determiner_noun_agreement_irregular_1",
+      "sample_len": 1000,
+      "acc,none": 0.903,
+      "acc_stderr,none": 0.009363689373248062
+    },
+    "blimp_determiner_noun_agreement_irregular_2": {
+      "name": "blimp_determiner_noun_agreement_irregular_2",
+      "alias": "blimp_determiner_noun_agreement_irregular_2",
+      "sample_len": 1000,
+      "acc,none": 0.931,
+      "acc_stderr,none": 0.008018934050315124
+    },
+    "blimp_determiner_noun_agreement_with_adj_2": {
+      "name": "blimp_determiner_noun_agreement_with_adj_2",
+      "alias": "blimp_determiner_noun_agreement_with_adj_2",
+      "sample_len": 1000,
+      "acc,none": 0.927,
+      "acc_stderr,none": 0.008230354715243976
+    },
+    "blimp_determiner_noun_agreement_with_adj_irregular_1": {
+      "name": "blimp_determiner_noun_agreement_with_adj_irregular_1",
+      "alias": "blimp_determiner_noun_agreement_with_adj_irregular_1",
+      "sample_len": 1000,
+      "acc,none": 0.878,
+      "acc_stderr,none": 0.010354864712936776
+    },
+    "blimp_determiner_noun_agreement_with_adj_irregular_2": {
+      "name": "blimp_determiner_noun_agreement_with_adj_irregular_2",
+      "alias": "blimp_determiner_noun_agreement_with_adj_irregular_2",
+      "sample_len": 1000,
+      "acc,none": 0.891,
+      "acc_stderr,none": 0.009859828407037124
+    },
+    "blimp_determiner_noun_agreement_with_adjective_1": {
+      "name": "blimp_determiner_noun_agreement_with_adjective_1",
+      "alias": "blimp_determiner_noun_agreement_with_adjective_1",
+      "sample_len": 1000,
+      "acc,none": 0.934,
+      "acc_stderr,none": 0.007855297938697532
+    },
+    "blimp_distractor_agreement_relational_noun": {
+      "name": "blimp_distractor_agreement_relational_noun",
+      "alias": "blimp_distractor_agreement_relational_noun",
+      "sample_len": 1000,
+      "acc,none": 0.757,
+      "acc_stderr,none": 0.013569640199177543
+    },
+    "blimp_distractor_agreement_relative_clause": {
+      "name": "blimp_distractor_agreement_relative_clause",
+      "alias": "blimp_distractor_agreement_relative_clause",
+      "sample_len": 1000,
+      "acc,none": 0.617,
+      "acc_stderr,none": 0.0153801023256527
+    },
+    "blimp_drop_argument": {
+      "name": "blimp_drop_argument",
+      "alias": "blimp_drop_argument",
+      "sample_len": 1000,
+      "acc,none": 0.771,
+      "acc_stderr,none": 0.013294199326613684
+    },
+    "blimp_ellipsis_n_bar_1": {
+      "name": "blimp_ellipsis_n_bar_1",
+      "alias": "blimp_ellipsis_n_bar_1",
+      "sample_len": 1000,
+      "acc,none": 0.801,
+      "acc_stderr,none": 0.01263164908309911
+    },
+    "blimp_ellipsis_n_bar_2": {
+      "name": "blimp_ellipsis_n_bar_2",
+      "alias": "blimp_ellipsis_n_bar_2",
+      "sample_len": 1000,
+      "acc,none": 0.838,
+      "acc_stderr,none": 0.011657267771304384
+    },
+    "blimp_existential_there_object_raising": {
+      "name": "blimp_existential_there_object_raising",
+      "alias": "blimp_existential_there_object_raising",
+      "sample_len": 1000,
+      "acc,none": 0.774,
+      "acc_stderr,none": 0.013232501619085452
+    },
+    "blimp_existential_there_quantifiers_1": {
+      "name": "blimp_existential_there_quantifiers_1",
+      "alias": "blimp_existential_there_quantifiers_1",
+      "sample_len": 1000,
+      "acc,none": 0.97,
+      "acc_stderr,none": 0.0053971408290991564
+    },
+    "blimp_existential_there_quantifiers_2": {
+      "name": "blimp_existential_there_quantifiers_2",
+      "alias": "blimp_existential_there_quantifiers_2",
+      "sample_len": 1000,
+      "acc,none": 0.326,
+      "acc_stderr,none": 0.014830507204541047
+    },
+    "blimp_existential_there_subject_raising": {
+      "name": "blimp_existential_there_subject_raising",
+      "alias": "blimp_existential_there_subject_raising",
+      "sample_len": 1000,
+      "acc,none": 0.845,
+      "acc_stderr,none": 0.011450157470799522
+    },
+    "blimp_expletive_it_object_raising": {
+      "name": "blimp_expletive_it_object_raising",
+      "alias": "blimp_expletive_it_object_raising",
+      "sample_len": 1000,
+      "acc,none": 0.791,
+      "acc_stderr,none": 0.012864077288499337
+    },
+    "blimp_inchoative": {
+      "name": "blimp_inchoative",
+      "alias": "blimp_inchoative",
+      "sample_len": 1000,
+      "acc,none": 0.691,
+      "acc_stderr,none": 0.014619600977206347
+    },
+    "blimp_intransitive": {
+      "name": "blimp_intransitive",
+      "alias": "blimp_intransitive",
+      "sample_len": 1000,
+      "acc,none": 0.782,
+      "acc_stderr,none": 0.013063179040595235
+    },
+    "blimp_irregular_past_participle_adjectives": {
+      "name": "blimp_irregular_past_participle_adjectives",
+      "alias": "blimp_irregular_past_participle_adjectives",
+      "sample_len": 1000,
+      "acc,none": 0.934,
+      "acc_stderr,none": 0.007855297938697532
+    },
+    "blimp_irregular_past_participle_verbs": {
+      "name": "blimp_irregular_past_participle_verbs",
+      "alias": "blimp_irregular_past_participle_verbs",
+      "sample_len": 1000,
+      "acc,none": 0.9,
+      "acc_stderr,none": 0.00949157995752499
+    },
+    "blimp_irregular_plural_subject_verb_agreement_1": {
+      "name": "blimp_irregular_plural_subject_verb_agreement_1",
+      "alias": "blimp_irregular_plural_subject_verb_agreement_1",
+      "sample_len": 1000,
+      "acc,none": 0.911,
+      "acc_stderr,none": 0.009008893392651516
+    },
+    "blimp_irregular_plural_subject_verb_agreement_2": {
+      "name": "blimp_irregular_plural_subject_verb_agreement_2",
+      "alias": "blimp_irregular_plural_subject_verb_agreement_2",
+      "sample_len": 1000,
+      "acc,none": 0.896,
+      "acc_stderr,none": 0.009658016218524242
+    },
+    "blimp_left_branch_island_echo_question": {
+      "name": "blimp_left_branch_island_echo_question",
+      "alias": "blimp_left_branch_island_echo_question",
+      "sample_len": 1000,
+      "acc,none": 0.576,
+      "acc_stderr,none": 0.015635487471405186
+    },
+    "blimp_left_branch_island_simple_question": {
+      "name": "blimp_left_branch_island_simple_question",
+      "alias": "blimp_left_branch_island_simple_question",
+      "sample_len": 1000,
+      "acc,none": 0.615,
+      "acc_stderr,none": 0.015395194445410945
+    },
+    "blimp_matrix_question_npi_licensor_present": {
+      "name": "blimp_matrix_question_npi_licensor_present",
+      "alias": "blimp_matrix_question_npi_licensor_present",
+      "sample_len": 1000,
+      "acc,none": 0.521,
+      "acc_stderr,none": 0.015805341148131185
+    },
+    "blimp_npi_present_1": {
+      "name": "blimp_npi_present_1",
+      "alias": "blimp_npi_present_1",
+      "sample_len": 1000,
+      "acc,none": 0.614,
+      "acc_stderr,none": 0.015402637476784335
+    },
+    "blimp_npi_present_2": {
+      "name": "blimp_npi_present_2",
+      "alias": "blimp_npi_present_2",
+      "sample_len": 1000,
+      "acc,none": 0.691,
+      "acc_stderr,none": 0.014619600977206347
+    },
+    "blimp_only_npi_licensor_present": {
+      "name": "blimp_only_npi_licensor_present",
+      "alias": "blimp_only_npi_licensor_present",
+      "sample_len": 1000,
+      "acc,none": 0.976,
+      "acc_stderr,none": 0.004842256441727084
+    },
+    "blimp_only_npi_scope": {
+      "name": "blimp_only_npi_scope",
+      "alias": "blimp_only_npi_scope",
+      "sample_len": 1000,
+      "acc,none": 0.582,
+      "acc_stderr,none": 0.015605111967541904
+    },
+    "blimp_passive_1": {
+      "name": "blimp_passive_1",
+      "alias": "blimp_passive_1",
+      "sample_len": 1000,
+      "acc,none": 0.87,
+      "acc_stderr,none": 0.010640169792499236
+    },
+    "blimp_passive_2": {
+      "name": "blimp_passive_2",
+      "alias": "blimp_passive_2",
+      "sample_len": 1000,
+      "acc,none": 0.825,
+      "acc_stderr,none": 0.012021627157731998
+    },
+    "blimp_principle_A_c_command": {
+      "name": "blimp_principle_A_c_command",
+      "alias": "blimp_principle_A_c_command",
+      "sample_len": 1000,
+      "acc,none": 0.58,
+      "acc_stderr,none": 0.015615500115072966
+    },
+    "blimp_principle_A_case_1": {
+      "name": "blimp_principle_A_case_1",
+      "alias": "blimp_principle_A_case_1",
+      "sample_len": 1000,
+      "acc,none": 1.0,
+      "acc_stderr,none": 0.0
+    },
+    "blimp_principle_A_case_2": {
+      "name": "blimp_principle_A_case_2",
+      "alias": "blimp_principle_A_case_2",
+      "sample_len": 1000,
+      "acc,none": 0.915,
+      "acc_stderr,none": 0.008823426366942316
+    },
+    "blimp_principle_A_domain_1": {
+      "name": "blimp_principle_A_domain_1",
+      "alias": "blimp_principle_A_domain_1",
+      "sample_len": 1000,
+      "acc,none": 0.877,
+      "acc_stderr,none": 0.010391293421849803
+    },
+    "blimp_principle_A_domain_2": {
+      "name": "blimp_principle_A_domain_2",
+      "alias": "blimp_principle_A_domain_2",
+      "sample_len": 1000,
+      "acc,none": 0.764,
+      "acc_stderr,none": 0.013434451402438602
+    },
+    "blimp_principle_A_domain_3": {
+      "name": "blimp_principle_A_domain_3",
+      "alias": "blimp_principle_A_domain_3",
+      "sample_len": 1000,
+      "acc,none": 0.637,
+      "acc_stderr,none": 0.01521389044467136
+    },
+    "blimp_principle_A_reconstruction": {
+      "name": "blimp_principle_A_reconstruction",
+      "alias": "blimp_principle_A_reconstruction",
+      "sample_len": 1000,
+      "acc,none": 0.51,
+      "acc_stderr,none": 0.015816135752773183
+    },
+    "blimp_regular_plural_subject_verb_agreement_1": {
+      "name": "blimp_regular_plural_subject_verb_agreement_1",
+      "alias": "blimp_regular_plural_subject_verb_agreement_1",
+      "sample_len": 1000,
+      "acc,none": 0.92,
+      "acc_stderr,none": 0.00858333697775368
+    },
+    "blimp_regular_plural_subject_verb_agreement_2": {
+      "name": "blimp_regular_plural_subject_verb_agreement_2",
+      "alias": "blimp_regular_plural_subject_verb_agreement_2",
+      "sample_len": 1000,
+      "acc,none": 0.878,
+      "acc_stderr,none": 0.010354864712936776
+    },
+    "blimp_sentential_negation_npi_licensor_present": {
+      "name": "blimp_sentential_negation_npi_licensor_present",
+      "alias": "blimp_sentential_negation_npi_licensor_present",
+      "sample_len": 1000,
+      "acc,none": 0.987,
+      "acc_stderr,none": 0.003583830889403662
+    },
+    "blimp_sentential_negation_npi_scope": {
+      "name": "blimp_sentential_negation_npi_scope",
+      "alias": "blimp_sentential_negation_npi_scope",
+      "sample_len": 1000,
+      "acc,none": 0.652,
+      "acc_stderr,none": 0.015070604603768328
+    },
+    "blimp_sentential_subject_island": {
+      "name": "blimp_sentential_subject_island",
+      "alias": "blimp_sentential_subject_island",
+      "sample_len": 1000,
+      "acc,none": 0.506,
+      "acc_stderr,none": 0.015818160898606836
+    },
+    "blimp_superlative_quantifiers_1": {
+      "name": "blimp_superlative_quantifiers_1",
+      "alias": "blimp_superlative_quantifiers_1",
+      "sample_len": 1000,
+      "acc,none": 0.934,
+      "acc_stderr,none": 0.007855297938697532
+    },
+    "blimp_superlative_quantifiers_2": {
+      "name": "blimp_superlative_quantifiers_2",
+      "alias": "blimp_superlative_quantifiers_2",
+      "sample_len": 1000,
+      "acc,none": 0.683,
+      "acc_stderr,none": 0.014721675438880174
+    },
+    "blimp_tough_vs_raising_1": {
+      "name": "blimp_tough_vs_raising_1",
+      "alias": "blimp_tough_vs_raising_1",
+      "sample_len": 1000,
+      "acc,none": 0.673,
+      "acc_stderr,none": 0.014842213153411162
+    },
+    "blimp_tough_vs_raising_2": {
+      "name": "blimp_tough_vs_raising_2",
+      "alias": "blimp_tough_vs_raising_2",
+      "sample_len": 1000,
+      "acc,none": 0.876,
+      "acc_stderr,none": 0.010427498872343873
+    },
+    "blimp_transitive": {
+      "name": "blimp_transitive",
+      "alias": "blimp_transitive",
+      "sample_len": 1000,
+      "acc,none": 0.809,
+      "acc_stderr,none": 0.012436787112179482
+    },
+    "blimp_wh_island": {
+      "name": "blimp_wh_island",
+      "alias": "blimp_wh_island",
+      "sample_len": 1000,
+      "acc,none": 0.855,
+      "acc_stderr,none": 0.0111399775178902
+    },
+    "blimp_wh_questions_object_gap": {
+      "name": "blimp_wh_questions_object_gap",
+      "alias": "blimp_wh_questions_object_gap",
+      "sample_len": 1000,
+      "acc,none": 0.767,
+      "acc_stderr,none": 0.01337497251921999
+    },
+    "blimp_wh_questions_subject_gap": {
+      "name": "blimp_wh_questions_subject_gap",
+      "alias": "blimp_wh_questions_subject_gap",
+      "sample_len": 1000,
+      "acc,none": 0.917,
+      "acc_stderr,none": 0.008728527206074756
+    },
+    "blimp_wh_questions_subject_gap_long_distance": {
+      "name": "blimp_wh_questions_subject_gap_long_distance",
+      "alias": "blimp_wh_questions_subject_gap_long_distance",
+      "sample_len": 1000,
+      "acc,none": 0.847,
+      "acc_stderr,none": 0.011389500459665504
+    },
+    "blimp_wh_vs_that_no_gap": {
+      "name": "blimp_wh_vs_that_no_gap",
+      "alias": "blimp_wh_vs_that_no_gap",
+      "sample_len": 1000,
+      "acc,none": 0.969,
+      "acc_stderr,none": 0.005483527064679181
+    },
+    "blimp_wh_vs_that_no_gap_long_distance": {
+      "name": "blimp_wh_vs_that_no_gap_long_distance",
+      "alias": "blimp_wh_vs_that_no_gap_long_distance",
+      "sample_len": 1000,
+      "acc,none": 0.95,
+      "acc_stderr,none": 0.006895472974897918
+    },
+    "blimp_wh_vs_that_with_gap": {
+      "name": "blimp_wh_vs_that_with_gap",
+      "alias": "blimp_wh_vs_that_with_gap",
+      "sample_len": 1000,
+      "acc,none": 0.528,
+      "acc_stderr,none": 0.015794475789511517
+    },
+    "blimp_wh_vs_that_with_gap_long_distance": {
+      "name": "blimp_wh_vs_that_with_gap_long_distance",
+      "alias": "blimp_wh_vs_that_with_gap_long_distance",
+      "sample_len": 1000,
+      "acc,none": 0.284,
+      "acc_stderr,none": 0.014267009061031241
+    },
+    "blimp": {
+      "alias": "blimp",
+      "name": "blimp",
+      "sample_len": 67000,
+      "acc,none": 0.7844776119402985,
+      "acc_stderr,none": 0.0014535278370615976,
+      "sample_count": {
+        "acc,none": 67000
+      }
+    }
+  }
+}

m14/config.json

@@ -0,0 +1,50 @@
+{
+  "model_id": "m14",
+  "run_name": "m14-mha-rope-moe-muon",
+  "data_dir": "data/fineweb-edu",
+  "vocab_size": 50257,
+  "block_size": 1024,
+  "d_model": 512,
+  "num_layers": 12,
+  "num_heads": 8,
+  "ffn_mult": 4,
+  "dropout": 0.0,
+  "bias": false,
+  "norm_type": "rmsnorm",
+  "tie_embeddings": true,
+  "attention_type": "mha",
+  "gqa_num_kv_heads": 2,
+  "mla_latent_dim": 64,
+  "pos_type": "rope",
+  "rope_base": 10000,
+  "ffn_type": "moe",
+  "moe_num_experts": 8,
+  "moe_top_k": 2,
+  "optimizer": "muon",
+  "lr": 0.0006,
+  "min_lr": 6e-05,
+  "weight_decay": 0.1,
+  "beta1": 0.9,
+  "beta2": 0.95,
+  "grad_clip": 1.0,
+  "muon_lr": 0.02,
+  "muon_momentum": 0.95,
+  "training_tokens": 1000000000,
+  "warmup_tokens": 6000000,
+  "lr_schedule": "cosine",
+  "batch_size": 2,
+  "grad_accum_steps": 32,
+  "device": "cuda",
+  "dtype": "bfloat16",
+  "compile": true,
+  "seed": 1337,
+  "eval_interval": 250,
+  "eval_iters": 100,
+  "log_interval": 10,
+  "checkpoint_interval": 1000,
+  "results_dir": "results",
+  "mlflow_experiment": "slm-ablation",
+  "mlflow_tracking_uri": "sqlite:///mlflow.db",
+  "mlflow_artifact_location": "./mlartifacts",
+  "moe_aux_loss_coef": 0.01
+}

m14/eval_results.json

@@ -0,0 +1,223 @@
+{
+  "model_id": "m14",
+  "checkpoint": "results/m14/final.pt",
+  "trained_step": 15258,
+  "total_params": 239703040,
+  "active_params": 88708096,
+  "flops_per_forward": 177416192.0,
+  "val_loss": 3.4363590747118,
+  "val_perplexity": 31.07361524541882,
+  "bits_per_token": 4.9576181958005,
+  "ppl_tokens_evaluated": 655360,
+  "prefill_tok_per_sec": 80126.71676878858,
+  "inference_tok_per_sec": 46.77562740153019,
+  "inference_tok_per_sec_kvcache": 79.65645893983364,
+  "kvcache_speedup": 1.7029479531305616,
+  "gen_tokens": 128,
+  "peak_infer_vram_gb": 1.452,
+  "peak_train_vram_gb": 3.608,
+  "closeness_answer_ppl": 13.041,
+  "closeness_hit_rate": 0.55,
+  "closeness_token_f1": 0.068,
+  "closeness_n": 20,
+  "closeness_items": [
+    {
+      "prompt": "The capital of France is",
+      "answer": " Paris",
+      "generated": " the city of Paris. The",
+      "answer_nll": 3.202,
+      "answer_ppl": 24.582,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The capital of Japan is",
+      "answer": " Tokyo",
+      "generated": " the city of Tokyo. The",
+      "answer_nll": 2.5489,
+      "answer_ppl": 12.793,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The largest planet in the solar system is",
+      "answer": " Jupiter",
+      "generated": " the Sun, which is the",
+      "answer_nll": 2.4559,
+      "answer_ppl": 11.657,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "Water is made of hydrogen and",
+      "answer": " oxygen",
+      "generated": " oxygen.\nThe hydrogen is",
+      "answer_nll": 0.2959,
+      "answer_ppl": 1.344,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The opposite of hot is",
+      "answer": " cold",
+      "generated": " hot. Hot is the heat",
+      "answer_nll": 4.2344,
+      "answer_ppl": 69.019,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The opposite of up is",
+      "answer": " down",
+      "generated": " the difference between the two.",
+      "answer_nll": 4.0606,
+      "answer_ppl": 58.008,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "Two plus two equals",
+      "answer": " four",
+      "generated": " two.\nThe first is",
+      "answer_nll": 2.1366,
+      "answer_ppl": 8.471,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The sun rises in the",
+      "answer": " east",
+      "generated": " morning and sets in the evening",
+      "answer_nll": 2.7328,
+      "answer_ppl": 15.375,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The first president of the United States was George",
+      "answer": " Washington",
+      "generated": " Washington.\nThe first president",
+      "answer_nll": 0.2571,
+      "answer_ppl": 1.293,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The speed of light is approximately 300,000 kilometers per",
+      "answer": " second",
+      "generated": " second.\nThe speed of",
+      "answer_nll": 0.0656,
+      "answer_ppl": 1.068,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "A triangle has three",
+      "answer": " sides",
+      "generated": " sides, and the third side",
+      "answer_nll": 1.3205,
+      "answer_ppl": 3.745,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The chemical symbol for gold is",
+      "answer": " Au",
+      "generated": " gold. Gold is the most",
+      "answer_nll": 6.5487,
+      "answer_ppl": 698.318,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The freezing point of water in Celsius is",
+      "answer": " zero",
+      "generated": " about 1.5 \u00b0C",
+      "answer_nll": 5.9355,
+      "answer_ppl": 378.246,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The author of Romeo and Juliet is William",
+      "answer": " Shakespeare",
+      "generated": " Shakespeare.\nThe play is",
+      "answer_nll": 0.1597,
+      "answer_ppl": 1.173,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The Earth revolves around the",
+      "answer": " Sun",
+      "generated": " Sun, and the Sun is",
+      "answer_nll": 0.6747,
+      "answer_ppl": 1.963,
+      "hit": true,
+      "token_f1": 0.333
+    },
+    {
+      "prompt": "The plural of mouse is",
+      "answer": " mice",
+      "generated": " the mouse.\nThe word",
+      "answer_nll": 5.1476,
+      "answer_ppl": 172.017,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "Photosynthesis occurs in the",
+      "answer": " leaves",
+      "generated": " leaves of the plant.\n",
+      "answer_nll": 3.5603,
+      "answer_ppl": 35.174,
+      "hit": true,
+      "token_f1": 0.4
+    },
+    {
+      "prompt": "The human heart pumps",
+      "answer": " blood",
+      "generated": " blood to the rest of the",
+      "answer_nll": 1.1786,
+      "answer_ppl": 3.25,
+      "hit": true,
+      "token_f1": 0.286
+    },
+    {
+      "prompt": "The currency used in the United States is the",
+      "answer": " dollar",
+      "generated": " dollar. The dollar is the",
+      "answer_nll": 2.5244,
+      "answer_ppl": 12.483,
+      "hit": true,
+      "token_f1": 0.333
+    },
+    {
+      "prompt": "The tallest animal in the world is the",
+      "answer": " giraffe",
+      "generated": " lion.\nThe lion is",
+      "answer_nll": 2.3226,
+      "answer_ppl": 10.202,
+      "hit": false,
+      "token_f1": 0.0
+    }
+  ],
+  "moe/aux_loss": 0.12053199759684503,
+  "moe/routing_entropy": 0.9948502505819002,
+  "moe/max_load_frac": 0.1416552225748698,
+  "moe/expert_0_prob": 0.12510733157396317,
+  "moe/expert_1_prob": 0.1254948280751705,
+  "moe/expert_2_prob": 0.130660605058074,
+  "moe/expert_3_prob": 0.12439362742006779,
+  "moe/expert_4_prob": 0.12149989195168018,
+  "moe/expert_5_prob": 0.13262441903352737,
+  "moe/expert_6_prob": 0.12078449092805385,
+  "moe/expert_7_prob": 0.11943480558693409,
+  "generation_samples": {
+    "The most important thing about sleep is": "The most important thing about sleep is that it\u2019s a good idea to prepare yourself and your family for the day ahead. If you find that you get into bed, you might be craving for the night. You might be feeling a bit hungry later, and also feeling the energy back to sleep.\nWhat happens if you can",
+    "Scientists have discovered that": "Scientists have discovered that, in many ways, the human body is a complex machine with four chambers, eight organs, five nerve paths, one central nerve, two peripheral nerve pathways, three peripheral nerve pathways, and a small number of other systems.\nThe human body is a complex machine with four distinct layers. The outer",
+    "In the history of mathematics,": "In the history of mathematics, the term \"curriculum\" is often used to describe any subject that is relevant to the science or mathematics curriculum. The term \"curriculum\" is often used in relation to the broader subject of mathematics curriculum.\nThe term \"curriculum\" is used more widely in the United States",
+    "Python code for prime number": "Python code for prime number of elements for free. However, the program is not capable to compute integers and can only compute integers, which is a requirement for the first and last elements of the Java package.<|endoftext|>The name Gogol is derived from the Greek word for \"horn\",\nwhere the word \"horn\" means",
+    "The best way to learn a new language is": "The best way to learn a new language is to learn about its culture, culture, and language. Learning a new language is something that can be enjoyed by everyone. If you have ever wondered how the language works, you know that it\u2019s used in different cultures. Learning the language is a beautiful language because it allows you to express yourself"
+  }
+}

m14/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd1ca02cd62bf3170452324a914b4d51064a30bf6569a6676d6bcabbdc88dcda
+size 958841736