Upload folder using huggingface_hub

m15/benchmark_results.json

@@ -0,0 +1,540 @@
+{
+  "model_id": "m15",
+  "checkpoint": "results/m15/final.pt",
+  "trained_step": 15258,
+  "num_fewshot": 0,
+  "limit": null,
+  "results": {
+    "lambada_openai": {
+      "name": "lambada_openai",
+      "alias": "lambada_openai",
+      "sample_len": 5153,
+      "perplexity,none": 413.4500055271363,
+      "perplexity_stderr,none": 23.383529548584082,
+      "acc,none": 0.19483795847079372,
+      "acc_stderr,none": 0.005518111913121999
+    },
+    "hellaswag": {
+      "name": "hellaswag",
+      "alias": "hellaswag",
+      "sample_len": 10042,
+      "acc,none": 0.2808205536745668,
+      "acc_stderr,none": 0.004484815647064684,
+      "acc_norm,none": 0.2944632543318064,
+      "acc_norm_stderr,none": 0.004548695749620993
+    },
+    "piqa": {
+      "name": "piqa",
+      "alias": "piqa",
+      "sample_len": 1838,
+      "acc,none": 0.6088139281828074,
+      "acc_stderr,none": 0.011386215606728629,
+      "acc_norm,none": 0.6077257889009793,
+      "acc_norm_stderr,none": 0.011391846744072072
+    },
+    "sciq": {
+      "name": "sciq",
+      "alias": "sciq",
+      "sample_len": 1000,
+      "acc,none": 0.679,
+      "acc_stderr,none": 0.01477082181793475,
+      "acc_norm,none": 0.583,
+      "acc_norm_stderr,none": 0.015599819048769583
+    },
+    "arc_easy": {
+      "name": "arc_easy",
+      "alias": "arc_easy",
+      "sample_len": 2376,
+      "acc,none": 0.4591750841750842,
+      "acc_stderr,none": 0.010225526906982679,
+      "acc_norm,none": 0.4078282828282828,
+      "acc_norm_stderr,none": 0.01008395024004126
+    },
+    "winogrande": {
+      "name": "winogrande",
+      "alias": "winogrande",
+      "sample_len": 1267,
+      "acc,none": 0.4988161010260458,
+      "acc_stderr,none": 0.014052446290528871
+    },
+    "blimp_adjunct_island": {
+      "name": "blimp_adjunct_island",
+      "alias": "blimp_adjunct_island",
+      "sample_len": 1000,
+      "acc,none": 0.836,
+      "acc_stderr,none": 0.011715000693181425
+    },
+    "blimp_anaphor_gender_agreement": {
+      "name": "blimp_anaphor_gender_agreement",
+      "alias": "blimp_anaphor_gender_agreement",
+      "sample_len": 1000,
+      "acc,none": 0.862,
+      "acc_stderr,none": 0.010912152632504508
+    },
+    "blimp_anaphor_number_agreement": {
+      "name": "blimp_anaphor_number_agreement",
+      "alias": "blimp_anaphor_number_agreement",
+      "sample_len": 1000,
+      "acc,none": 0.966,
+      "acc_stderr,none": 0.0057338361396953854
+    },
+    "blimp_animate_subject_passive": {
+      "name": "blimp_animate_subject_passive",
+      "alias": "blimp_animate_subject_passive",
+      "sample_len": 1000,
+      "acc,none": 0.706,
+      "acc_stderr,none": 0.014414290540008265
+    },
+    "blimp_animate_subject_trans": {
+      "name": "blimp_animate_subject_trans",
+      "alias": "blimp_animate_subject_trans",
+      "sample_len": 1000,
+      "acc,none": 0.898,
+      "acc_stderr,none": 0.009575368801653944
+    },
+    "blimp_causative": {
+      "name": "blimp_causative",
+      "alias": "blimp_causative",
+      "sample_len": 1000,
+      "acc,none": 0.702,
+      "acc_stderr,none": 0.014470846741134585
+    },
+    "blimp_complex_NP_island": {
+      "name": "blimp_complex_NP_island",
+      "alias": "blimp_complex_NP_island",
+      "sample_len": 1000,
+      "acc,none": 0.528,
+      "acc_stderr,none": 0.015794475789511517
+    },
+    "blimp_coordinate_structure_constraint_complex_left_branch": {
+      "name": "blimp_coordinate_structure_constraint_complex_left_branch",
+      "alias": "blimp_coordinate_structure_constraint_complex_left_branch",
+      "sample_len": 1000,
+      "acc,none": 0.43,
+      "acc_stderr,none": 0.01566350361015521
+    },
+    "blimp_coordinate_structure_constraint_object_extraction": {
+      "name": "blimp_coordinate_structure_constraint_object_extraction",
+      "alias": "blimp_coordinate_structure_constraint_object_extraction",
+      "sample_len": 1000,
+      "acc,none": 0.783,
+      "acc_stderr,none": 0.013041513757270706
+    },
+    "blimp_determiner_noun_agreement_1": {
+      "name": "blimp_determiner_noun_agreement_1",
+      "alias": "blimp_determiner_noun_agreement_1",
+      "sample_len": 1000,
+      "acc,none": 0.973,
+      "acc_stderr,none": 0.005128089049275305
+    },
+    "blimp_determiner_noun_agreement_2": {
+      "name": "blimp_determiner_noun_agreement_2",
+      "alias": "blimp_determiner_noun_agreement_2",
+      "sample_len": 1000,
+      "acc,none": 0.942,
+      "acc_stderr,none": 0.0073953154557929906
+    },
+    "blimp_determiner_noun_agreement_irregular_1": {
+      "name": "blimp_determiner_noun_agreement_irregular_1",
+      "alias": "blimp_determiner_noun_agreement_irregular_1",
+      "sample_len": 1000,
+      "acc,none": 0.878,
+      "acc_stderr,none": 0.010354864712936776
+    },
+    "blimp_determiner_noun_agreement_irregular_2": {
+      "name": "blimp_determiner_noun_agreement_irregular_2",
+      "alias": "blimp_determiner_noun_agreement_irregular_2",
+      "sample_len": 1000,
+      "acc,none": 0.923,
+      "acc_stderr,none": 0.00843458014024066
+    },
+    "blimp_determiner_noun_agreement_with_adj_2": {
+      "name": "blimp_determiner_noun_agreement_with_adj_2",
+      "alias": "blimp_determiner_noun_agreement_with_adj_2",
+      "sample_len": 1000,
+      "acc,none": 0.921,
+      "acc_stderr,none": 0.00853415677333337
+    },
+    "blimp_determiner_noun_agreement_with_adj_irregular_1": {
+      "name": "blimp_determiner_noun_agreement_with_adj_irregular_1",
+      "alias": "blimp_determiner_noun_agreement_with_adj_irregular_1",
+      "sample_len": 1000,
+      "acc,none": 0.862,
+      "acc_stderr,none": 0.010912152632504508
+    },
+    "blimp_determiner_noun_agreement_with_adj_irregular_2": {
+      "name": "blimp_determiner_noun_agreement_with_adj_irregular_2",
+      "alias": "blimp_determiner_noun_agreement_with_adj_irregular_2",
+      "sample_len": 1000,
+      "acc,none": 0.892,
+      "acc_stderr,none": 0.009820001651345708
+    },
+    "blimp_determiner_noun_agreement_with_adjective_1": {
+      "name": "blimp_determiner_noun_agreement_with_adjective_1",
+      "alias": "blimp_determiner_noun_agreement_with_adjective_1",
+      "sample_len": 1000,
+      "acc,none": 0.93,
+      "acc_stderr,none": 0.008072494358323525
+    },
+    "blimp_distractor_agreement_relational_noun": {
+      "name": "blimp_distractor_agreement_relational_noun",
+      "alias": "blimp_distractor_agreement_relational_noun",
+      "sample_len": 1000,
+      "acc,none": 0.829,
+      "acc_stderr,none": 0.011912216456264588
+    },
+    "blimp_distractor_agreement_relative_clause": {
+      "name": "blimp_distractor_agreement_relative_clause",
+      "alias": "blimp_distractor_agreement_relative_clause",
+      "sample_len": 1000,
+      "acc,none": 0.737,
+      "acc_stderr,none": 0.013929286594259776
+    },
+    "blimp_drop_argument": {
+      "name": "blimp_drop_argument",
+      "alias": "blimp_drop_argument",
+      "sample_len": 1000,
+      "acc,none": 0.744,
+      "acc_stderr,none": 0.013807775152234145
+    },
+    "blimp_ellipsis_n_bar_1": {
+      "name": "blimp_ellipsis_n_bar_1",
+      "alias": "blimp_ellipsis_n_bar_1",
+      "sample_len": 1000,
+      "acc,none": 0.816,
+      "acc_stderr,none": 0.01225945734093864
+    },
+    "blimp_ellipsis_n_bar_2": {
+      "name": "blimp_ellipsis_n_bar_2",
+      "alias": "blimp_ellipsis_n_bar_2",
+      "sample_len": 1000,
+      "acc,none": 0.839,
+      "acc_stderr,none": 0.011628164696727263
+    },
+    "blimp_existential_there_object_raising": {
+      "name": "blimp_existential_there_object_raising",
+      "alias": "blimp_existential_there_object_raising",
+      "sample_len": 1000,
+      "acc,none": 0.726,
+      "acc_stderr,none": 0.01411109928825969
+    },
+    "blimp_existential_there_quantifiers_1": {
+      "name": "blimp_existential_there_quantifiers_1",
+      "alias": "blimp_existential_there_quantifiers_1",
+      "sample_len": 1000,
+      "acc,none": 0.969,
+      "acc_stderr,none": 0.005483527064679181
+    },
+    "blimp_existential_there_quantifiers_2": {
+      "name": "blimp_existential_there_quantifiers_2",
+      "alias": "blimp_existential_there_quantifiers_2",
+      "sample_len": 1000,
+      "acc,none": 0.187,
+      "acc_stderr,none": 0.012336254828074166
+    },
+    "blimp_existential_there_subject_raising": {
+      "name": "blimp_existential_there_subject_raising",
+      "alias": "blimp_existential_there_subject_raising",
+      "sample_len": 1000,
+      "acc,none": 0.872,
+      "acc_stderr,none": 0.010570133761108595
+    },
+    "blimp_expletive_it_object_raising": {
+      "name": "blimp_expletive_it_object_raising",
+      "alias": "blimp_expletive_it_object_raising",
+      "sample_len": 1000,
+      "acc,none": 0.685,
+      "acc_stderr,none": 0.014696631960792617
+    },
+    "blimp_inchoative": {
+      "name": "blimp_inchoative",
+      "alias": "blimp_inchoative",
+      "sample_len": 1000,
+      "acc,none": 0.665,
+      "acc_stderr,none": 0.014933117490932629
+    },
+    "blimp_intransitive": {
+      "name": "blimp_intransitive",
+      "alias": "blimp_intransitive",
+      "sample_len": 1000,
+      "acc,none": 0.748,
+      "acc_stderr,none": 0.013736254390651213
+    },
+    "blimp_irregular_past_participle_adjectives": {
+      "name": "blimp_irregular_past_participle_adjectives",
+      "alias": "blimp_irregular_past_participle_adjectives",
+      "sample_len": 1000,
+      "acc,none": 0.723,
+      "acc_stderr,none": 0.014158794845306273
+    },
+    "blimp_irregular_past_participle_verbs": {
+      "name": "blimp_irregular_past_participle_verbs",
+      "alias": "blimp_irregular_past_participle_verbs",
+      "sample_len": 1000,
+      "acc,none": 0.839,
+      "acc_stderr,none": 0.011628164696727263
+    },
+    "blimp_irregular_plural_subject_verb_agreement_1": {
+      "name": "blimp_irregular_plural_subject_verb_agreement_1",
+      "alias": "blimp_irregular_plural_subject_verb_agreement_1",
+      "sample_len": 1000,
+      "acc,none": 0.905,
+      "acc_stderr,none": 0.009276910103103353
+    },
+    "blimp_irregular_plural_subject_verb_agreement_2": {
+      "name": "blimp_irregular_plural_subject_verb_agreement_2",
+      "alias": "blimp_irregular_plural_subject_verb_agreement_2",
+      "sample_len": 1000,
+      "acc,none": 0.82,
+      "acc_stderr,none": 0.012155153135512022
+    },
+    "blimp_left_branch_island_echo_question": {
+      "name": "blimp_left_branch_island_echo_question",
+      "alias": "blimp_left_branch_island_echo_question",
+      "sample_len": 1000,
+      "acc,none": 0.618,
+      "acc_stderr,none": 0.015372453034968502
+    },
+    "blimp_left_branch_island_simple_question": {
+      "name": "blimp_left_branch_island_simple_question",
+      "alias": "blimp_left_branch_island_simple_question",
+      "sample_len": 1000,
+      "acc,none": 0.595,
+      "acc_stderr,none": 0.01553113699045296
+    },
+    "blimp_matrix_question_npi_licensor_present": {
+      "name": "blimp_matrix_question_npi_licensor_present",
+      "alias": "blimp_matrix_question_npi_licensor_present",
+      "sample_len": 1000,
+      "acc,none": 0.532,
+      "acc_stderr,none": 0.015786868759359002
+    },
+    "blimp_npi_present_1": {
+      "name": "blimp_npi_present_1",
+      "alias": "blimp_npi_present_1",
+      "sample_len": 1000,
+      "acc,none": 0.665,
+      "acc_stderr,none": 0.014933117490932629
+    },
+    "blimp_npi_present_2": {
+      "name": "blimp_npi_present_2",
+      "alias": "blimp_npi_present_2",
+      "sample_len": 1000,
+      "acc,none": 0.723,
+      "acc_stderr,none": 0.014158794845306273
+    },
+    "blimp_only_npi_licensor_present": {
+      "name": "blimp_only_npi_licensor_present",
+      "alias": "blimp_only_npi_licensor_present",
+      "sample_len": 1000,
+      "acc,none": 0.993,
+      "acc_stderr,none": 0.002637794146243781
+    },
+    "blimp_only_npi_scope": {
+      "name": "blimp_only_npi_scope",
+      "alias": "blimp_only_npi_scope",
+      "sample_len": 1000,
+      "acc,none": 0.712,
+      "acc_stderr,none": 0.01432694179723158
+    },
+    "blimp_passive_1": {
+      "name": "blimp_passive_1",
+      "alias": "blimp_passive_1",
+      "sample_len": 1000,
+      "acc,none": 0.862,
+      "acc_stderr,none": 0.010912152632504508
+    },
+    "blimp_passive_2": {
+      "name": "blimp_passive_2",
+      "alias": "blimp_passive_2",
+      "sample_len": 1000,
+      "acc,none": 0.842,
+      "acc_stderr,none": 0.011539894677559635
+    },
+    "blimp_principle_A_c_command": {
+      "name": "blimp_principle_A_c_command",
+      "alias": "blimp_principle_A_c_command",
+      "sample_len": 1000,
+      "acc,none": 0.706,
+      "acc_stderr,none": 0.014414290540008265
+    },
+    "blimp_principle_A_case_1": {
+      "name": "blimp_principle_A_case_1",
+      "alias": "blimp_principle_A_case_1",
+      "sample_len": 1000,
+      "acc,none": 0.998,
+      "acc_stderr,none": 0.0014135055705578234
+    },
+    "blimp_principle_A_case_2": {
+      "name": "blimp_principle_A_case_2",
+      "alias": "blimp_principle_A_case_2",
+      "sample_len": 1000,
+      "acc,none": 0.908,
+      "acc_stderr,none": 0.009144376393151129
+    },
+    "blimp_principle_A_domain_1": {
+      "name": "blimp_principle_A_domain_1",
+      "alias": "blimp_principle_A_domain_1",
+      "sample_len": 1000,
+      "acc,none": 0.905,
+      "acc_stderr,none": 0.009276910103103353
+    },
+    "blimp_principle_A_domain_2": {
+      "name": "blimp_principle_A_domain_2",
+      "alias": "blimp_principle_A_domain_2",
+      "sample_len": 1000,
+      "acc,none": 0.704,
+      "acc_stderr,none": 0.01444273494157504
+    },
+    "blimp_principle_A_domain_3": {
+      "name": "blimp_principle_A_domain_3",
+      "alias": "blimp_principle_A_domain_3",
+      "sample_len": 1000,
+      "acc,none": 0.647,
+      "acc_stderr,none": 0.015120172605483696
+    },
+    "blimp_principle_A_reconstruction": {
+      "name": "blimp_principle_A_reconstruction",
+      "alias": "blimp_principle_A_reconstruction",
+      "sample_len": 1000,
+      "acc,none": 0.425,
+      "acc_stderr,none": 0.01564032031704017
+    },
+    "blimp_regular_plural_subject_verb_agreement_1": {
+      "name": "blimp_regular_plural_subject_verb_agreement_1",
+      "alias": "blimp_regular_plural_subject_verb_agreement_1",
+      "sample_len": 1000,
+      "acc,none": 0.864,
+      "acc_stderr,none": 0.01084535023047304
+    },
+    "blimp_regular_plural_subject_verb_agreement_2": {
+      "name": "blimp_regular_plural_subject_verb_agreement_2",
+      "alias": "blimp_regular_plural_subject_verb_agreement_2",
+      "sample_len": 1000,
+      "acc,none": 0.869,
+      "acc_stderr,none": 0.0106748748448379
+    },
+    "blimp_sentential_negation_npi_licensor_present": {
+      "name": "blimp_sentential_negation_npi_licensor_present",
+      "alias": "blimp_sentential_negation_npi_licensor_present",
+      "sample_len": 1000,
+      "acc,none": 0.985,
+      "acc_stderr,none": 0.003845749574502972
+    },
+    "blimp_sentential_negation_npi_scope": {
+      "name": "blimp_sentential_negation_npi_scope",
+      "alias": "blimp_sentential_negation_npi_scope",
+      "sample_len": 1000,
+      "acc,none": 0.632,
+      "acc_stderr,none": 0.015258073561521743
+    },
+    "blimp_sentential_subject_island": {
+      "name": "blimp_sentential_subject_island",
+      "alias": "blimp_sentential_subject_island",
+      "sample_len": 1000,
+      "acc,none": 0.528,
+      "acc_stderr,none": 0.015794475789511517
+    },
+    "blimp_superlative_quantifiers_1": {
+      "name": "blimp_superlative_quantifiers_1",
+      "alias": "blimp_superlative_quantifiers_1",
+      "sample_len": 1000,
+      "acc,none": 0.832,
+      "acc_stderr,none": 0.011828605831454352
+    },
+    "blimp_superlative_quantifiers_2": {
+      "name": "blimp_superlative_quantifiers_2",
+      "alias": "blimp_superlative_quantifiers_2",
+      "sample_len": 1000,
+      "acc,none": 0.663,
+      "acc_stderr,none": 0.01495508791865346
+    },
+    "blimp_tough_vs_raising_1": {
+      "name": "blimp_tough_vs_raising_1",
+      "alias": "blimp_tough_vs_raising_1",
+      "sample_len": 1000,
+      "acc,none": 0.66,
+      "acc_stderr,none": 0.014987482264363972
+    },
+    "blimp_tough_vs_raising_2": {
+      "name": "blimp_tough_vs_raising_2",
+      "alias": "blimp_tough_vs_raising_2",
+      "sample_len": 1000,
+      "acc,none": 0.848,
+      "acc_stderr,none": 0.01135891830347525
+    },
+    "blimp_transitive": {
+      "name": "blimp_transitive",
+      "alias": "blimp_transitive",
+      "sample_len": 1000,
+      "acc,none": 0.797,
+      "acc_stderr,none": 0.01272607374459825
+    },
+    "blimp_wh_island": {
+      "name": "blimp_wh_island",
+      "alias": "blimp_wh_island",
+      "sample_len": 1000,
+      "acc,none": 0.685,
+      "acc_stderr,none": 0.014696631960792617
+    },
+    "blimp_wh_questions_object_gap": {
+      "name": "blimp_wh_questions_object_gap",
+      "alias": "blimp_wh_questions_object_gap",
+      "sample_len": 1000,
+      "acc,none": 0.795,
+      "acc_stderr,none": 0.012772554096113201
+    },
+    "blimp_wh_questions_subject_gap": {
+      "name": "blimp_wh_questions_subject_gap",
+      "alias": "blimp_wh_questions_subject_gap",
+      "sample_len": 1000,
+      "acc,none": 0.931,
+      "acc_stderr,none": 0.008018934050315124
+    },
+    "blimp_wh_questions_subject_gap_long_distance": {
+      "name": "blimp_wh_questions_subject_gap_long_distance",
+      "alias": "blimp_wh_questions_subject_gap_long_distance",
+      "sample_len": 1000,
+      "acc,none": 0.828,
+      "acc_stderr,none": 0.011939788882495308
+    },
+    "blimp_wh_vs_that_no_gap": {
+      "name": "blimp_wh_vs_that_no_gap",
+      "alias": "blimp_wh_vs_that_no_gap",
+      "sample_len": 1000,
+      "acc,none": 0.968,
+      "acc_stderr,none": 0.005568393575081328
+    },
+    "blimp_wh_vs_that_no_gap_long_distance": {
+      "name": "blimp_wh_vs_that_no_gap_long_distance",
+      "alias": "blimp_wh_vs_that_no_gap_long_distance",
+      "sample_len": 1000,
+      "acc,none": 0.969,
+      "acc_stderr,none": 0.005483527064679181
+    },
+    "blimp_wh_vs_that_with_gap": {
+      "name": "blimp_wh_vs_that_with_gap",
+      "alias": "blimp_wh_vs_that_with_gap",
+      "sample_len": 1000,
+      "acc,none": 0.486,
+      "acc_stderr,none": 0.01581309754773093
+    },
+    "blimp_wh_vs_that_with_gap_long_distance": {
+      "name": "blimp_wh_vs_that_with_gap_long_distance",
+      "alias": "blimp_wh_vs_that_with_gap_long_distance",
+      "sample_len": 1000,
+      "acc,none": 0.258,
+      "acc_stderr,none": 0.013842963108656601
+    },
+    "blimp": {
+      "alias": "blimp",
+      "name": "blimp",
+      "sample_len": 67000,
+      "acc,none": 0.7696865671641792,
+      "acc_stderr,none": 0.0014888534910081543,
+      "sample_count": {
+        "acc,none": 67000
+      }
+    }
+  }
+}

m15/config.json

@@ -0,0 +1,51 @@
+{
+  "model_id": "m15",
+  "run_name": "m15-mha-rope-shared-moe-muon",
+  "data_dir": "data/fineweb-edu",
+  "vocab_size": 50257,
+  "block_size": 1024,
+  "d_model": 512,
+  "num_layers": 12,
+  "num_heads": 8,
+  "ffn_mult": 4,
+  "dropout": 0.0,
+  "bias": false,
+  "norm_type": "rmsnorm",
+  "tie_embeddings": true,
+  "attention_type": "mha",
+  "gqa_num_kv_heads": 2,
+  "mla_latent_dim": 64,
+  "pos_type": "rope",
+  "rope_base": 10000,
+  "ffn_type": "shared_moe",
+  "moe_num_experts": 8,
+  "moe_top_k": 2,
+  "optimizer": "muon",
+  "lr": 0.0006,
+  "min_lr": 6e-05,
+  "weight_decay": 0.1,
+  "beta1": 0.9,
+  "beta2": 0.95,
+  "grad_clip": 1.0,
+  "muon_lr": 0.02,
+  "muon_momentum": 0.95,
+  "training_tokens": 1000000000,
+  "warmup_tokens": 6000000,
+  "lr_schedule": "cosine",
+  "batch_size": 2,
+  "grad_accum_steps": 32,
+  "device": "cuda",
+  "dtype": "bfloat16",
+  "compile": true,
+  "seed": 1337,
+  "eval_interval": 250,
+  "eval_iters": 100,
+  "log_interval": 10,
+  "checkpoint_interval": 1000,
+  "results_dir": "results",
+  "mlflow_experiment": "slm-ablation",
+  "mlflow_tracking_uri": "sqlite:///mlflow.db",
+  "mlflow_artifact_location": "./mlartifacts",
+  "moe_num_shared_experts": 1,
+  "moe_aux_loss_coef": 0.01
+}

m15/eval_results.json

@@ -0,0 +1,223 @@
+{
+  "model_id": "m15",
+  "checkpoint": "results/m15/final.pt",
+  "trained_step": 15258,
+  "total_params": 264868864,
+  "active_params": 113873920,
+  "flops_per_forward": 227747840.0,
+  "val_loss": 3.4677844196558,
+  "val_perplexity": 32.06561972800647,
+  "bits_per_token": 5.002955385109434,
+  "ppl_tokens_evaluated": 655360,
+  "prefill_tok_per_sec": 61298.66085312115,
+  "inference_tok_per_sec": 41.6292336332186,
+  "inference_tok_per_sec_kvcache": 67.26176681140403,
+  "kvcache_speedup": 1.6157339672410307,
+  "gen_tokens": 128,
+  "peak_infer_vram_gb": 1.599,
+  "peak_train_vram_gb": 3.981,
+  "closeness_answer_ppl": 12.372,
+  "closeness_hit_rate": 0.55,
+  "closeness_token_f1": 0.029,
+  "closeness_n": 20,
+  "closeness_items": [
+    {
+      "prompt": "The capital of France is",
+      "answer": " Paris",
+      "generated": " Paris.\nThe city is",
+      "answer_nll": 2.0906,
+      "answer_ppl": 8.09,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The capital of Japan is",
+      "answer": " Tokyo",
+      "generated": " the city of Tokyo.\n",
+      "answer_nll": 3.1428,
+      "answer_ppl": 23.169,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The largest planet in the solar system is",
+      "answer": " Jupiter",
+      "generated": " the Sun, which is the",
+      "answer_nll": 1.6507,
+      "answer_ppl": 5.211,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "Water is made of hydrogen and",
+      "answer": " oxygen",
+      "generated": " oxygen, which are both carbon",
+      "answer_nll": 0.2688,
+      "answer_ppl": 1.308,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The opposite of hot is",
+      "answer": " cold",
+      "generated": " the fact that the temperature of",
+      "answer_nll": 3.2884,
+      "answer_ppl": 26.799,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The opposite of up is",
+      "answer": " down",
+      "generated": " the word \u201cfoss",
+      "answer_nll": 6.3577,
+      "answer_ppl": 576.902,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "Two plus two equals",
+      "answer": " four",
+      "generated": " the sum of the two.",
+      "answer_nll": 3.4839,
+      "answer_ppl": 32.588,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The sun rises in the",
+      "answer": " east",
+      "generated": " east and sets in the west",
+      "answer_nll": 0.7555,
+      "answer_ppl": 2.129,
+      "hit": true,
+      "token_f1": 0.286
+    },
+    {
+      "prompt": "The first president of the United States was George",
+      "answer": " Washington",
+      "generated": " Washington, who was a member",
+      "answer_nll": 0.4636,
+      "answer_ppl": 1.59,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The speed of light is approximately 300,000 kilometers per",
+      "answer": " second",
+      "generated": " second. The speed of light",
+      "answer_nll": 0.1067,
+      "answer_ppl": 1.113,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "A triangle has three",
+      "answer": " sides",
+      "generated": " sides. The first is the",
+      "answer_nll": 1.2746,
+      "answer_ppl": 3.577,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The chemical symbol for gold is",
+      "answer": " Au",
+      "generated": " the gold leaf.\nThe",
+      "answer_nll": 6.585,
+      "answer_ppl": 724.124,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The freezing point of water in Celsius is",
+      "answer": " zero",
+      "generated": " -2.5\u00b0C",
+      "answer_nll": 5.1008,
+      "answer_ppl": 164.146,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The author of Romeo and Juliet is William",
+      "answer": " Shakespeare",
+      "generated": " Shakespeare, a professor of English",
+      "answer_nll": 0.989,
+      "answer_ppl": 2.688,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The Earth revolves around the",
+      "answer": " Sun",
+      "generated": " sun, and the moon revolves",
+      "answer_nll": 1.0905,
+      "answer_ppl": 2.976,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The plural of mouse is",
+      "answer": " mice",
+      "generated": " mouse.\n- mouse =",
+      "answer_nll": 4.5976,
+      "answer_ppl": 99.242,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "Photosynthesis occurs in the",
+      "answer": " leaves",
+      "generated": " chloroplast, which is",
+      "answer_nll": 3.1152,
+      "answer_ppl": 22.538,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The human heart pumps",
+      "answer": " blood",
+      "generated": " blood through the heart and lungs",
+      "answer_nll": 1.616,
+      "answer_ppl": 5.033,
+      "hit": true,
+      "token_f1": 0.286
+    },
+    {
+      "prompt": "The currency used in the United States is the",
+      "answer": " dollar",
+      "generated": " U.S. dollar.",
+      "answer_nll": 2.0529,
+      "answer_ppl": 7.79,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The tallest animal in the world is the",
+      "answer": " giraffe",
+      "generated": " African elephant.\nThe elephant",
+      "answer_nll": 2.2793,
+      "answer_ppl": 9.77,
+      "hit": false,
+      "token_f1": 0.0
+    }
+  ],
+  "moe/aux_loss": 0.12051594359800219,
+  "moe/routing_entropy": 0.9923138906558353,
+  "moe/max_load_frac": 0.14490839640299483,
+  "moe/expert_0_prob": 0.13816578015685083,
+  "moe/expert_1_prob": 0.12321405634284019,
+  "moe/expert_2_prob": 0.12382426597177983,
+  "moe/expert_3_prob": 0.11399608328938485,
+  "moe/expert_4_prob": 0.11689293012022972,
+  "moe/expert_5_prob": 0.1181701436638832,
+  "moe/expert_6_prob": 0.13688957169651986,
+  "moe/expert_7_prob": 0.12884716577827932,
+  "generation_samples": {
+    "The most important thing about sleep is": "The most important thing about sleep is that it\u2019s not just about eating. It is also about maintaining a healthy body and a healthy personality.\n- Sleep is a huge focus for all of us. Let\u2019s face it: sleep is our first resort.\n- Sleep is the best thing to have for you.",
+    "Scientists have discovered that": "Scientists have discovered that, when in fact the earth was not the same size as Earth in the past.\nBut when you put the earth in its place, with all its surface, the earth could have a circumference of a little over one-quarter of its entire surface. So the moon, the moon, this huge",
+    "In the history of mathematics,": "In the history of mathematics, the term has been used to describe a set of principles and practices which are in some sense similar to traditional methods of research. It is a special case of the mathematical sciences of the 19th Century, but the term has been used in different ways by different researchers. The traditional methods of research have been",
+    "Python code for prime number": "Python code for prime number in the above example. However, the string is not a string itself but a string in the string itself.\nA simple example would be a prime number that is also the prime number of the base $n$N$n$N$n$n\nTo check if a prime number is",
+    "The best way to learn a new language is": "The best way to learn a new language is to learn about its culture, geography, economy, technology, history, philosophy, culture, values, etc.\n- If you have some knowledge of the language, you need to think of the culture of that language, and that culture.\n- If you are new to the language, you need"
+  }
+}

m15/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:150978df2850d9eae1ab6d89c412c4189b4f5dc70446ab6edf1a5f12850c9ba3
+size 1059507864