{ "model_id": "m12", "checkpoint": "results/m12/final.pt", "trained_step": 15258, "total_params": 58774528, "active_params": 58774528, "flops_per_forward": 117549056.0, "val_loss": 3.4686521768569945, "val_perplexity": 32.09345697667592, "bits_per_token": 5.0042072941202935, "ppl_tokens_evaluated": 655360, "prefill_tok_per_sec": 193194.22812667364, "inference_tok_per_sec": 212.31968241677103, "inference_tok_per_sec_kvcache": 210.12757536665706, "kvcache_speedup": 0.9896754411783124, "gen_tokens": 128, "peak_infer_vram_gb": 0.366, "peak_train_vram_gb": 3.993, "closeness_answer_ppl": 42.281, "closeness_hit_rate": 0.4, "closeness_token_f1": 0.033, "closeness_n": 20, "closeness_items": [ { "prompt": "The capital of France is", "answer": " Paris", "generated": " the city of Paris.\n", "answer_nll": 2.863, "answer_ppl": 17.513, "hit": true, "token_f1": 0.0 }, { "prompt": "The capital of Japan is", "answer": " Tokyo", "generated": " the city of Sichuan", "answer_nll": 2.9536, "answer_ppl": 19.174, "hit": false, "token_f1": 0.0 }, { "prompt": "The largest planet in the solar system is", "answer": " Jupiter", "generated": " the Sun.\nThe Sun", "answer_nll": 3.8845, "answer_ppl": 48.642, "hit": false, "token_f1": 0.0 }, { "prompt": "Water is made of hydrogen and", "answer": " oxygen", "generated": " oxygen.\nThe water is", "answer_nll": 0.9818, "answer_ppl": 2.669, "hit": true, "token_f1": 0.0 }, { "prompt": "The opposite of hot is", "answer": " cold", "generated": " the heat of the sun.", "answer_nll": 4.4139, "answer_ppl": 82.587, "hit": false, "token_f1": 0.0 }, { "prompt": "The opposite of up is", "answer": " down", "generated": " the fact that the two are", "answer_nll": 8.6555, "answer_ppl": 5741.871, "hit": false, "token_f1": 0.0 }, { "prompt": "Two plus two equals", "answer": " four", "generated": " to the total number of people", "answer_nll": 5.2942, "answer_ppl": 199.172, "hit": false, "token_f1": 0.0 }, { "prompt": "The sun rises in the", "answer": " east", "generated": " sky and the sun is at", "answer_nll": 3.463, "answer_ppl": 31.914, "hit": false, "token_f1": 0.0 }, { "prompt": "The first president of the United States was George", "answer": " Washington", "generated": " Washington.\nThe second president", "answer_nll": 1.038, "answer_ppl": 2.824, "hit": true, "token_f1": 0.0 }, { "prompt": "The speed of light is approximately 300,000 kilometers per", "answer": " second", "generated": " second.\nThe speed of", "answer_nll": 0.1077, "answer_ppl": 1.114, "hit": true, "token_f1": 0.0 }, { "prompt": "A triangle has three", "answer": " sides", "generated": " sides. The two sides of", "answer_nll": 1.6775, "answer_ppl": 5.352, "hit": true, "token_f1": 0.333 }, { "prompt": "The chemical symbol for gold is", "answer": " Au", "generated": " the gold ion.\nGold", "answer_nll": 6.4909, "answer_ppl": 659.103, "hit": false, "token_f1": 0.0 }, { "prompt": "The freezing point of water in Celsius is", "answer": " zero", "generated": " about 0.5\u00b0C", "answer_nll": 4.9866, "answer_ppl": 146.444, "hit": false, "token_f1": 0.0 }, { "prompt": "The author of Romeo and Juliet is William", "answer": " Shakespeare", "generated": " Shakespeare\u2019s play Romeo", "answer_nll": 0.3631, "answer_ppl": 1.438, "hit": true, "token_f1": 0.0 }, { "prompt": "The Earth revolves around the", "answer": " Sun", "generated": " sun, and the sun is", "answer_nll": 2.4816, "answer_ppl": 11.961, "hit": true, "token_f1": 0.333 }, { "prompt": "The plural of mouse is", "answer": " mice", "generated": " the same as the human.", "answer_nll": 9.4341, "answer_ppl": 12507.574, "hit": false, "token_f1": 0.0 }, { "prompt": "Photosynthesis occurs in the", "answer": " leaves", "generated": " brain, and the brain is", "answer_nll": 5.3131, "answer_ppl": 202.985, "hit": false, "token_f1": 0.0 }, { "prompt": "The human heart pumps", "answer": " blood", "generated": " are not only the heart pumps", "answer_nll": 3.9707, "answer_ppl": 53.021, "hit": false, "token_f1": 0.0 }, { "prompt": "The currency used in the United States is the", "answer": " dollar", "generated": " United States dollar.\nThe", "answer_nll": 3.4802, "answer_ppl": 32.465, "hit": true, "token_f1": 0.0 }, { "prompt": "The tallest animal in the world is the", "answer": " giraffe", "generated": " largest in the world.\n", "answer_nll": 3.034, "answer_ppl": 20.781, "hit": false, "token_f1": 0.0 } ], "generation_samples": { "The most important thing about sleep is": "The most important thing about sleep is to look for your own specific medical conditions, such as:\n- Heart conditions such as coronary heart disease, chronic heart failure, diabetes, and cancer.\n- If you have had any of the following:\n- Acute respiratory syndrome\n- High blood pressure\n- Heart disease\n- O", "Scientists have discovered that": "Scientists have discovered that, in many ways, the human body is actually a computer system.\nThe first evidence for the existence of the human body is the presence of a long-lived entity. In a number of cases the body is not yet the subject of many other science studies, but in a much smaller number it", "In the history of mathematics,": "In the history of mathematics, the term \"curriculum\" is often used to refer to the process in which schools use the term to describe the process in which a curriculum has been created. In schools, it is most commonly referred to as a \"curriculum\" or \"curriculum\" as it is also referred", "Python code for prime number": "Python code for prime number of objects.\nHow to create a string using string\nTo create a string using string, you have to use a string you need. For example, we need the string:\n\nHow to create the string using string\nTo", "The best way to learn a new language is": "The best way to learn a new language is to learn it in a second language. It is especially important to study the language in a way that is pleasing to the senses and allows you to build the foundation for yourself.\nFor instance, if you can\u2019t speak a language in a sentence, you can write a new sentence in it" } }