Add m29

m29/benchmark_results.json

@@ -0,0 +1,540 @@
+{
+  "model_id": "m29",
+  "checkpoint": "results/m29/final.pt",
+  "trained_step": 15258,
+  "num_fewshot": 0,
+  "limit": null,
+  "results": {
+    "lambada_openai": {
+      "name": "lambada_openai",
+      "alias": "lambada_openai",
+      "sample_len": 5153,
+      "perplexity,none": 580.781945907576,
+      "perplexity_stderr,none": 32.536453548312494,
+      "acc,none": 0.1841645643314574,
+      "acc_stderr,none": 0.005400281393204285
+    },
+    "hellaswag": {
+      "name": "hellaswag",
+      "alias": "hellaswag",
+      "sample_len": 10042,
+      "acc,none": 0.2727544313881697,
+      "acc_stderr,none": 0.004444654076551038,
+      "acc_norm,none": 0.28201553475403307,
+      "acc_norm_stderr,none": 0.004490612245335525
+    },
+    "piqa": {
+      "name": "piqa",
+      "alias": "piqa",
+      "sample_len": 1838,
+      "acc,none": 0.5919477693144722,
+      "acc_stderr,none": 0.011466872778651143,
+      "acc_norm,none": 0.5919477693144722,
+      "acc_norm_stderr,none": 0.011466872778651143
+    },
+    "sciq": {
+      "name": "sciq",
+      "alias": "sciq",
+      "sample_len": 1000,
+      "acc,none": 0.694,
+      "acc_stderr,none": 0.014580006055436898,
+      "acc_norm,none": 0.599,
+      "acc_norm_stderr,none": 0.015506109745498368
+    },
+    "arc_easy": {
+      "name": "arc_easy",
+      "alias": "arc_easy",
+      "sample_len": 2376,
+      "acc,none": 0.4701178451178451,
+      "acc_stderr,none": 0.010241444322886225,
+      "acc_norm,none": 0.4158249158249158,
+      "acc_norm_stderr,none": 0.01011334824464792
+    },
+    "winogrande": {
+      "name": "winogrande",
+      "alias": "winogrande",
+      "sample_len": 1267,
+      "acc,none": 0.5272296764009471,
+      "acc_stderr,none": 0.01403163162982771
+    },
+    "blimp_adjunct_island": {
+      "name": "blimp_adjunct_island",
+      "alias": "blimp_adjunct_island",
+      "sample_len": 1000,
+      "acc,none": 0.768,
+      "acc_stderr,none": 0.013354937452281623
+    },
+    "blimp_anaphor_gender_agreement": {
+      "name": "blimp_anaphor_gender_agreement",
+      "alias": "blimp_anaphor_gender_agreement",
+      "sample_len": 1000,
+      "acc,none": 0.887,
+      "acc_stderr,none": 0.010016552866696773
+    },
+    "blimp_anaphor_number_agreement": {
+      "name": "blimp_anaphor_number_agreement",
+      "alias": "blimp_anaphor_number_agreement",
+      "sample_len": 1000,
+      "acc,none": 0.978,
+      "acc_stderr,none": 0.004640855259274689
+    },
+    "blimp_animate_subject_passive": {
+      "name": "blimp_animate_subject_passive",
+      "alias": "blimp_animate_subject_passive",
+      "sample_len": 1000,
+      "acc,none": 0.715,
+      "acc_stderr,none": 0.014282120955200562
+    },
+    "blimp_animate_subject_trans": {
+      "name": "blimp_animate_subject_trans",
+      "alias": "blimp_animate_subject_trans",
+      "sample_len": 1000,
+      "acc,none": 0.901,
+      "acc_stderr,none": 0.009449248027662746
+    },
+    "blimp_causative": {
+      "name": "blimp_causative",
+      "alias": "blimp_causative",
+      "sample_len": 1000,
+      "acc,none": 0.708,
+      "acc_stderr,none": 0.014385511563477433
+    },
+    "blimp_complex_NP_island": {
+      "name": "blimp_complex_NP_island",
+      "alias": "blimp_complex_NP_island",
+      "sample_len": 1000,
+      "acc,none": 0.506,
+      "acc_stderr,none": 0.015818160898606836
+    },
+    "blimp_coordinate_structure_constraint_complex_left_branch": {
+      "name": "blimp_coordinate_structure_constraint_complex_left_branch",
+      "alias": "blimp_coordinate_structure_constraint_complex_left_branch",
+      "sample_len": 1000,
+      "acc,none": 0.393,
+      "acc_stderr,none": 0.015452824654081525
+    },
+    "blimp_coordinate_structure_constraint_object_extraction": {
+      "name": "blimp_coordinate_structure_constraint_object_extraction",
+      "alias": "blimp_coordinate_structure_constraint_object_extraction",
+      "sample_len": 1000,
+      "acc,none": 0.786,
+      "acc_stderr,none": 0.012975838021968847
+    },
+    "blimp_determiner_noun_agreement_1": {
+      "name": "blimp_determiner_noun_agreement_1",
+      "alias": "blimp_determiner_noun_agreement_1",
+      "sample_len": 1000,
+      "acc,none": 0.973,
+      "acc_stderr,none": 0.005128089049275305
+    },
+    "blimp_determiner_noun_agreement_2": {
+      "name": "blimp_determiner_noun_agreement_2",
+      "alias": "blimp_determiner_noun_agreement_2",
+      "sample_len": 1000,
+      "acc,none": 0.97,
+      "acc_stderr,none": 0.0053971408290991564
+    },
+    "blimp_determiner_noun_agreement_irregular_1": {
+      "name": "blimp_determiner_noun_agreement_irregular_1",
+      "alias": "blimp_determiner_noun_agreement_irregular_1",
+      "sample_len": 1000,
+      "acc,none": 0.916,
+      "acc_stderr,none": 0.008776162089491113
+    },
+    "blimp_determiner_noun_agreement_irregular_2": {
+      "name": "blimp_determiner_noun_agreement_irregular_2",
+      "alias": "blimp_determiner_noun_agreement_irregular_2",
+      "sample_len": 1000,
+      "acc,none": 0.953,
+      "acc_stderr,none": 0.006695956678163013
+    },
+    "blimp_determiner_noun_agreement_with_adj_2": {
+      "name": "blimp_determiner_noun_agreement_with_adj_2",
+      "alias": "blimp_determiner_noun_agreement_with_adj_2",
+      "sample_len": 1000,
+      "acc,none": 0.932,
+      "acc_stderr,none": 0.007964887911291624
+    },
+    "blimp_determiner_noun_agreement_with_adj_irregular_1": {
+      "name": "blimp_determiner_noun_agreement_with_adj_irregular_1",
+      "alias": "blimp_determiner_noun_agreement_with_adj_irregular_1",
+      "sample_len": 1000,
+      "acc,none": 0.885,
+      "acc_stderr,none": 0.010093407594904551
+    },
+    "blimp_determiner_noun_agreement_with_adj_irregular_2": {
+      "name": "blimp_determiner_noun_agreement_with_adj_irregular_2",
+      "alias": "blimp_determiner_noun_agreement_with_adj_irregular_2",
+      "sample_len": 1000,
+      "acc,none": 0.909,
+      "acc_stderr,none": 0.009099549538400337
+    },
+    "blimp_determiner_noun_agreement_with_adjective_1": {
+      "name": "blimp_determiner_noun_agreement_with_adjective_1",
+      "alias": "blimp_determiner_noun_agreement_with_adjective_1",
+      "sample_len": 1000,
+      "acc,none": 0.943,
+      "acc_stderr,none": 0.007335175853706905
+    },
+    "blimp_distractor_agreement_relational_noun": {
+      "name": "blimp_distractor_agreement_relational_noun",
+      "alias": "blimp_distractor_agreement_relational_noun",
+      "sample_len": 1000,
+      "acc,none": 0.863,
+      "acc_stderr,none": 0.01087884871433321
+    },
+    "blimp_distractor_agreement_relative_clause": {
+      "name": "blimp_distractor_agreement_relative_clause",
+      "alias": "blimp_distractor_agreement_relative_clause",
+      "sample_len": 1000,
+      "acc,none": 0.713,
+      "acc_stderr,none": 0.014312087053810001
+    },
+    "blimp_drop_argument": {
+      "name": "blimp_drop_argument",
+      "alias": "blimp_drop_argument",
+      "sample_len": 1000,
+      "acc,none": 0.766,
+      "acc_stderr,none": 0.01339490288966006
+    },
+    "blimp_ellipsis_n_bar_1": {
+      "name": "blimp_ellipsis_n_bar_1",
+      "alias": "blimp_ellipsis_n_bar_1",
+      "sample_len": 1000,
+      "acc,none": 0.709,
+      "acc_stderr,none": 0.014370995982378032
+    },
+    "blimp_ellipsis_n_bar_2": {
+      "name": "blimp_ellipsis_n_bar_2",
+      "alias": "blimp_ellipsis_n_bar_2",
+      "sample_len": 1000,
+      "acc,none": 0.853,
+      "acc_stderr,none": 0.011203415395160376
+    },
+    "blimp_existential_there_object_raising": {
+      "name": "blimp_existential_there_object_raising",
+      "alias": "blimp_existential_there_object_raising",
+      "sample_len": 1000,
+      "acc,none": 0.703,
+      "acc_stderr,none": 0.01445683229480096
+    },
+    "blimp_existential_there_quantifiers_1": {
+      "name": "blimp_existential_there_quantifiers_1",
+      "alias": "blimp_existential_there_quantifiers_1",
+      "sample_len": 1000,
+      "acc,none": 0.973,
+      "acc_stderr,none": 0.005128089049275305
+    },
+    "blimp_existential_there_quantifiers_2": {
+      "name": "blimp_existential_there_quantifiers_2",
+      "alias": "blimp_existential_there_quantifiers_2",
+      "sample_len": 1000,
+      "acc,none": 0.308,
+      "acc_stderr,none": 0.01460648312734278
+    },
+    "blimp_existential_there_subject_raising": {
+      "name": "blimp_existential_there_subject_raising",
+      "alias": "blimp_existential_there_subject_raising",
+      "sample_len": 1000,
+      "acc,none": 0.866,
+      "acc_stderr,none": 0.010777762298369738
+    },
+    "blimp_expletive_it_object_raising": {
+      "name": "blimp_expletive_it_object_raising",
+      "alias": "blimp_expletive_it_object_raising",
+      "sample_len": 1000,
+      "acc,none": 0.714,
+      "acc_stderr,none": 0.014297146862518003
+    },
+    "blimp_inchoative": {
+      "name": "blimp_inchoative",
+      "alias": "blimp_inchoative",
+      "sample_len": 1000,
+      "acc,none": 0.69,
+      "acc_stderr,none": 0.014632638658632766
+    },
+    "blimp_intransitive": {
+      "name": "blimp_intransitive",
+      "alias": "blimp_intransitive",
+      "sample_len": 1000,
+      "acc,none": 0.782,
+      "acc_stderr,none": 0.013063179040595235
+    },
+    "blimp_irregular_past_participle_adjectives": {
+      "name": "blimp_irregular_past_participle_adjectives",
+      "alias": "blimp_irregular_past_participle_adjectives",
+      "sample_len": 1000,
+      "acc,none": 0.746,
+      "acc_stderr,none": 0.013772206565168622
+    },
+    "blimp_irregular_past_participle_verbs": {
+      "name": "blimp_irregular_past_participle_verbs",
+      "alias": "blimp_irregular_past_participle_verbs",
+      "sample_len": 1000,
+      "acc,none": 0.867,
+      "acc_stderr,none": 0.010743669132397295
+    },
+    "blimp_irregular_plural_subject_verb_agreement_1": {
+      "name": "blimp_irregular_plural_subject_verb_agreement_1",
+      "alias": "blimp_irregular_plural_subject_verb_agreement_1",
+      "sample_len": 1000,
+      "acc,none": 0.913,
+      "acc_stderr,none": 0.008916866630745944
+    },
+    "blimp_irregular_plural_subject_verb_agreement_2": {
+      "name": "blimp_irregular_plural_subject_verb_agreement_2",
+      "alias": "blimp_irregular_plural_subject_verb_agreement_2",
+      "sample_len": 1000,
+      "acc,none": 0.903,
+      "acc_stderr,none": 0.009363689373248062
+    },
+    "blimp_left_branch_island_echo_question": {
+      "name": "blimp_left_branch_island_echo_question",
+      "alias": "blimp_left_branch_island_echo_question",
+      "sample_len": 1000,
+      "acc,none": 0.42,
+      "acc_stderr,none": 0.015615500115072968
+    },
+    "blimp_left_branch_island_simple_question": {
+      "name": "blimp_left_branch_island_simple_question",
+      "alias": "blimp_left_branch_island_simple_question",
+      "sample_len": 1000,
+      "acc,none": 0.502,
+      "acc_stderr,none": 0.01581917337430266
+    },
+    "blimp_matrix_question_npi_licensor_present": {
+      "name": "blimp_matrix_question_npi_licensor_present",
+      "alias": "blimp_matrix_question_npi_licensor_present",
+      "sample_len": 1000,
+      "acc,none": 0.52,
+      "acc_stderr,none": 0.015806639423035177
+    },
+    "blimp_npi_present_1": {
+      "name": "blimp_npi_present_1",
+      "alias": "blimp_npi_present_1",
+      "sample_len": 1000,
+      "acc,none": 0.499,
+      "acc_stderr,none": 0.015819268290576817
+    },
+    "blimp_npi_present_2": {
+      "name": "blimp_npi_present_2",
+      "alias": "blimp_npi_present_2",
+      "sample_len": 1000,
+      "acc,none": 0.616,
+      "acc_stderr,none": 0.015387682761896958
+    },
+    "blimp_only_npi_licensor_present": {
+      "name": "blimp_only_npi_licensor_present",
+      "alias": "blimp_only_npi_licensor_present",
+      "sample_len": 1000,
+      "acc,none": 0.913,
+      "acc_stderr,none": 0.008916866630745944
+    },
+    "blimp_only_npi_scope": {
+      "name": "blimp_only_npi_scope",
+      "alias": "blimp_only_npi_scope",
+      "sample_len": 1000,
+      "acc,none": 0.627,
+      "acc_stderr,none": 0.015300493622922927
+    },
+    "blimp_passive_1": {
+      "name": "blimp_passive_1",
+      "alias": "blimp_passive_1",
+      "sample_len": 1000,
+      "acc,none": 0.871,
+      "acc_stderr,none": 0.010605256784796466
+    },
+    "blimp_passive_2": {
+      "name": "blimp_passive_2",
+      "alias": "blimp_passive_2",
+      "sample_len": 1000,
+      "acc,none": 0.869,
+      "acc_stderr,none": 0.0106748748448379
+    },
+    "blimp_principle_A_c_command": {
+      "name": "blimp_principle_A_c_command",
+      "alias": "blimp_principle_A_c_command",
+      "sample_len": 1000,
+      "acc,none": 0.618,
+      "acc_stderr,none": 0.015372453034968502
+    },
+    "blimp_principle_A_case_1": {
+      "name": "blimp_principle_A_case_1",
+      "alias": "blimp_principle_A_case_1",
+      "sample_len": 1000,
+      "acc,none": 1.0,
+      "acc_stderr,none": 0.0
+    },
+    "blimp_principle_A_case_2": {
+      "name": "blimp_principle_A_case_2",
+      "alias": "blimp_principle_A_case_2",
+      "sample_len": 1000,
+      "acc,none": 0.925,
+      "acc_stderr,none": 0.008333333333333333
+    },
+    "blimp_principle_A_domain_1": {
+      "name": "blimp_principle_A_domain_1",
+      "alias": "blimp_principle_A_domain_1",
+      "sample_len": 1000,
+      "acc,none": 0.956,
+      "acc_stderr,none": 0.006488921798427387
+    },
+    "blimp_principle_A_domain_2": {
+      "name": "blimp_principle_A_domain_2",
+      "alias": "blimp_principle_A_domain_2",
+      "sample_len": 1000,
+      "acc,none": 0.765,
+      "acc_stderr,none": 0.013414729030247124
+    },
+    "blimp_principle_A_domain_3": {
+      "name": "blimp_principle_A_domain_3",
+      "alias": "blimp_principle_A_domain_3",
+      "sample_len": 1000,
+      "acc,none": 0.635,
+      "acc_stderr,none": 0.015231776226264848
+    },
+    "blimp_principle_A_reconstruction": {
+      "name": "blimp_principle_A_reconstruction",
+      "alias": "blimp_principle_A_reconstruction",
+      "sample_len": 1000,
+      "acc,none": 0.299,
+      "acc_stderr,none": 0.014484778521220435
+    },
+    "blimp_regular_plural_subject_verb_agreement_1": {
+      "name": "blimp_regular_plural_subject_verb_agreement_1",
+      "alias": "blimp_regular_plural_subject_verb_agreement_1",
+      "sample_len": 1000,
+      "acc,none": 0.9,
+      "acc_stderr,none": 0.00949157995752499
+    },
+    "blimp_regular_plural_subject_verb_agreement_2": {
+      "name": "blimp_regular_plural_subject_verb_agreement_2",
+      "alias": "blimp_regular_plural_subject_verb_agreement_2",
+      "sample_len": 1000,
+      "acc,none": 0.902,
+      "acc_stderr,none": 0.009406619184621247
+    },
+    "blimp_sentential_negation_npi_licensor_present": {
+      "name": "blimp_sentential_negation_npi_licensor_present",
+      "alias": "blimp_sentential_negation_npi_licensor_present",
+      "sample_len": 1000,
+      "acc,none": 0.98,
+      "acc_stderr,none": 0.004429403980178329
+    },
+    "blimp_sentential_negation_npi_scope": {
+      "name": "blimp_sentential_negation_npi_scope",
+      "alias": "blimp_sentential_negation_npi_scope",
+      "sample_len": 1000,
+      "acc,none": 0.555,
+      "acc_stderr,none": 0.015723301886761007
+    },
+    "blimp_sentential_subject_island": {
+      "name": "blimp_sentential_subject_island",
+      "alias": "blimp_sentential_subject_island",
+      "sample_len": 1000,
+      "acc,none": 0.453,
+      "acc_stderr,none": 0.015749255189977683
+    },
+    "blimp_superlative_quantifiers_1": {
+      "name": "blimp_superlative_quantifiers_1",
+      "alias": "blimp_superlative_quantifiers_1",
+      "sample_len": 1000,
+      "acc,none": 0.783,
+      "acc_stderr,none": 0.013041513757270706
+    },
+    "blimp_superlative_quantifiers_2": {
+      "name": "blimp_superlative_quantifiers_2",
+      "alias": "blimp_superlative_quantifiers_2",
+      "sample_len": 1000,
+      "acc,none": 0.693,
+      "acc_stderr,none": 0.01459328489285274
+    },
+    "blimp_tough_vs_raising_1": {
+      "name": "blimp_tough_vs_raising_1",
+      "alias": "blimp_tough_vs_raising_1",
+      "sample_len": 1000,
+      "acc,none": 0.6,
+      "acc_stderr,none": 0.015499685165842571
+    },
+    "blimp_tough_vs_raising_2": {
+      "name": "blimp_tough_vs_raising_2",
+      "alias": "blimp_tough_vs_raising_2",
+      "sample_len": 1000,
+      "acc,none": 0.879,
+      "acc_stderr,none": 0.010318210380946179
+    },
+    "blimp_transitive": {
+      "name": "blimp_transitive",
+      "alias": "blimp_transitive",
+      "sample_len": 1000,
+      "acc,none": 0.818,
+      "acc_stderr,none": 0.0122075806376622
+    },
+    "blimp_wh_island": {
+      "name": "blimp_wh_island",
+      "alias": "blimp_wh_island",
+      "sample_len": 1000,
+      "acc,none": 0.724,
+      "acc_stderr,none": 0.01414298497574059
+    },
+    "blimp_wh_questions_object_gap": {
+      "name": "blimp_wh_questions_object_gap",
+      "alias": "blimp_wh_questions_object_gap",
+      "sample_len": 1000,
+      "acc,none": 0.769,
+      "acc_stderr,none": 0.013334797216936478
+    },
+    "blimp_wh_questions_subject_gap": {
+      "name": "blimp_wh_questions_subject_gap",
+      "alias": "blimp_wh_questions_subject_gap",
+      "sample_len": 1000,
+      "acc,none": 0.945,
+      "acc_stderr,none": 0.0072129762946392
+    },
+    "blimp_wh_questions_subject_gap_long_distance": {
+      "name": "blimp_wh_questions_subject_gap_long_distance",
+      "alias": "blimp_wh_questions_subject_gap_long_distance",
+      "sample_len": 1000,
+      "acc,none": 0.909,
+      "acc_stderr,none": 0.009099549538400337
+    },
+    "blimp_wh_vs_that_no_gap": {
+      "name": "blimp_wh_vs_that_no_gap",
+      "alias": "blimp_wh_vs_that_no_gap",
+      "sample_len": 1000,
+      "acc,none": 0.967,
+      "acc_stderr,none": 0.005651808820452346
+    },
+    "blimp_wh_vs_that_no_gap_long_distance": {
+      "name": "blimp_wh_vs_that_no_gap_long_distance",
+      "alias": "blimp_wh_vs_that_no_gap_long_distance",
+      "sample_len": 1000,
+      "acc,none": 0.984,
+      "acc_stderr,none": 0.003969856390319398
+    },
+    "blimp_wh_vs_that_with_gap": {
+      "name": "blimp_wh_vs_that_with_gap",
+      "alias": "blimp_wh_vs_that_with_gap",
+      "sample_len": 1000,
+      "acc,none": 0.533,
+      "acc_stderr,none": 0.015784807891138876
+    },
+    "blimp_wh_vs_that_with_gap_long_distance": {
+      "name": "blimp_wh_vs_that_with_gap_long_distance",
+      "alias": "blimp_wh_vs_that_with_gap_long_distance",
+      "sample_len": 1000,
+      "acc,none": 0.177,
+      "acc_stderr,none": 0.012075463420375087
+    },
+    "blimp": {
+      "alias": "blimp",
+      "name": "blimp",
+      "sample_len": 67000,
+      "acc,none": 0.7626567164179104,
+      "acc_stderr,none": 0.0014674121164425231,
+      "sample_count": {
+        "acc,none": 67000
+      }
+    }
+  }
+}

m29/config.json

@@ -0,0 +1,50 @@
+{
+  "model_id": "m29",
+  "run_name": "m29-mla-rope-decoupled-dense-adamw",
+  "data_dir": "data/fineweb-edu",
+  "vocab_size": 50257,
+  "block_size": 1024,
+  "d_model": 512,
+  "num_layers": 12,
+  "num_heads": 8,
+  "ffn_mult": 4,
+  "dropout": 0.0,
+  "bias": false,
+  "norm_type": "rmsnorm",
+  "tie_embeddings": true,
+  "attention_type": "mla",
+  "gqa_num_kv_heads": 2,
+  "mla_latent_dim": 256,
+  "pos_type": "rope",
+  "rope_base": 10000,
+  "ffn_type": "dense",
+  "moe_num_experts": 8,
+  "moe_top_k": 2,
+  "optimizer": "adamw",
+  "lr": 0.0006,
+  "min_lr": 6e-05,
+  "weight_decay": 0.1,
+  "beta1": 0.9,
+  "beta2": 0.95,
+  "grad_clip": 1.0,
+  "muon_lr": 0.02,
+  "muon_momentum": 0.95,
+  "training_tokens": 1000000000,
+  "warmup_tokens": 6000000,
+  "lr_schedule": "cosine",
+  "batch_size": 4,
+  "grad_accum_steps": 16,
+  "device": "cuda",
+  "dtype": "bfloat16",
+  "compile": true,
+  "seed": 1337,
+  "eval_interval": 250,
+  "eval_iters": 100,
+  "log_interval": 10,
+  "checkpoint_interval": 1000,
+  "results_dir": "results",
+  "mlflow_experiment": "slm-ablation",
+  "mlflow_tracking_uri": "sqlite:///mlflow.db",
+  "mlflow_artifact_location": "./mlartifacts",
+  "mla_rope_head_dim": 128
+}

m29/eval_results.json

@@ -0,0 +1,212 @@
+{
+  "model_id": "m29",
+  "checkpoint": "/home/akshat/codemindv2/results/m29/final.pt",
+  "trained_step": 15258,
+  "total_params": 68998144,
+  "active_params": 68998144,
+  "flops_per_forward": 137996288.0,
+  "val_loss": 3.441070702672005,
+  "val_perplexity": 31.220368010563895,
+  "bits_per_token": 4.964415638093202,
+  "ppl_tokens_evaluated": 655360,
+  "prefill_tok_per_sec": 88178.8236224039,
+  "inference_tok_per_sec": 172.30087307262255,
+  "inference_tok_per_sec_kvcache": 182.03567943438054,
+  "kvcache_speedup": 1.056498880058808,
+  "gen_tokens": 128,
+  "peak_infer_vram_gb": 0.431,
+  "peak_train_vram_gb": 4.413,
+  "closeness_answer_ppl": 21.486,
+  "closeness_hit_rate": 0.5,
+  "closeness_token_f1": 0.037,
+  "closeness_n": 20,
+  "closeness_items": [
+    {
+      "prompt": "The capital of France is",
+      "answer": " Paris",
+      "generated": " the city of Paris. The",
+      "answer_nll": 2.5287,
+      "answer_ppl": 12.538,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The capital of Japan is",
+      "answer": " Tokyo",
+      "generated": " Tokyo.\nThe capital of",
+      "answer_nll": 1.7255,
+      "answer_ppl": 5.615,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The largest planet in the solar system is",
+      "answer": " Jupiter",
+      "generated": " the planet Mercury.\nThe",
+      "answer_nll": 3.6406,
+      "answer_ppl": 38.114,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "Water is made of hydrogen and",
+      "answer": " oxygen",
+      "generated": " oxygen. The hydrogen is then",
+      "answer_nll": 0.8387,
+      "answer_ppl": 2.313,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The opposite of hot is",
+      "answer": " cold",
+      "generated": " the hot air.\nThe",
+      "answer_nll": 4.313,
+      "answer_ppl": 74.66,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The opposite of up is",
+      "answer": " down",
+      "generated": " the difference between the two.",
+      "answer_nll": 2.7835,
+      "answer_ppl": 16.175,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "Two plus two equals",
+      "answer": " four",
+      "generated": " the number of times the number",
+      "answer_nll": 5.4009,
+      "answer_ppl": 221.596,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The sun rises in the",
+      "answer": " east",
+      "generated": " sky and the sun rises in",
+      "answer_nll": 3.1233,
+      "answer_ppl": 22.722,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The first president of the United States was George",
+      "answer": " Washington",
+      "generated": " Washington. He was the first",
+      "answer_nll": 1.0048,
+      "answer_ppl": 2.731,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The speed of light is approximately 300,000 kilometers per",
+      "answer": " second",
+      "generated": " second.\nThe speed of",
+      "answer_nll": 0.6529,
+      "answer_ppl": 1.921,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "A triangle has three",
+      "answer": " sides",
+      "generated": " sides and a half-circle",
+      "answer_nll": 1.4301,
+      "answer_ppl": 4.179,
+      "hit": true,
+      "token_f1": 0.4
+    },
+    {
+      "prompt": "The chemical symbol for gold is",
+      "answer": " Au",
+      "generated": " gold.\nThe word \ufffd",
+      "answer_nll": 7.6048,
+      "answer_ppl": 2007.901,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The freezing point of water in Celsius is",
+      "answer": " zero",
+      "generated": " about 1.5\u00b0C",
+      "answer_nll": 5.0617,
+      "answer_ppl": 157.856,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The author of Romeo and Juliet is William",
+      "answer": " Shakespeare",
+      "generated": " H. H. H.",
+      "answer_nll": 4.1658,
+      "answer_ppl": 64.444,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The Earth revolves around the",
+      "answer": " Sun",
+      "generated": " Sun.\nThe Earth is",
+      "answer_nll": 1.0321,
+      "answer_ppl": 2.807,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The plural of mouse is",
+      "answer": " mice",
+      "generated": " the word for the word \ufffd",
+      "answer_nll": 5.4251,
+      "answer_ppl": 227.029,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "Photosynthesis occurs in the",
+      "answer": " leaves",
+      "generated": " plant\u2019s leaves.",
+      "answer_nll": 3.7376,
+      "answer_ppl": 41.997,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The human heart pumps",
+      "answer": " blood",
+      "generated": " blood to the brain, and",
+      "answer_nll": 1.145,
+      "answer_ppl": 3.142,
+      "hit": true,
+      "token_f1": 0.333
+    },
+    {
+      "prompt": "The currency used in the United States is the",
+      "answer": " dollar",
+      "generated": " United States dollar.\nThe",
+      "answer_nll": 3.0361,
+      "answer_ppl": 20.824,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The tallest animal in the world is the",
+      "answer": " giraffe",
+      "generated": " elephant.\nThe elephant is",
+      "answer_nll": 2.6978,
+      "answer_ppl": 14.846,
+      "hit": false,
+      "token_f1": 0.0
+    }
+  ],
+  "generation_samples": {
+    "The most important thing about sleep is": "The most important thing about sleep is to feel rest. It doesn\u2019t just take the body and mind to do our things. It\u2019s also important to think about what you want to, what you want to expect. If you want to feel more calm, you can do a little bit more than talk to yourself,",
+    "Scientists have discovered that": "Scientists have discovered that, in many ways, the human brain is capable of processing information in only a few seconds, when it is constantly experiencing a change in the environment. This behavior is called the \"receiving of the brain,\" a process that takes place in the brain's sensory centers, said co-author John",
+    "In the history of mathematics,": "In the history of mathematics, the term \"curved\" means \"the sum of the elements of the whole\".\nThe word \"curved\" is sometimes used interchangeably with the Latin word \"curved.\" (This word is a derivative of the two \"curved\" words, \"curved\", and \"curved",
+    "Python code for prime number": "Python code for prime number of integers and integers. However, the main object of the Python code is to define prime numbers and to use the main object of the Python code to define the main object of the Python code.\nThe python code is a Python library that is used to develop Python functions. Like Python, Python,",
+    "The best way to learn a new language is": "The best way to learn a new language is to learn about it.\nRead the article on the new BBC article here.<|endoftext|>In 2011, the first mass-market energy system was born by the American Energy Agency. As the energy system entered the market in 2008, it received the first full-quality energy in the world, with a lower"
+  }
+}

m29/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37523b9ff3dce98feeb8f3b7e2137eaf9de2169923aed5189934f9ca5e7672d1
+size 276005816