| { |
| "model_id": "m12", |
| "checkpoint": "results/m12/final.pt", |
| "trained_step": 15258, |
| "total_params": 58774528, |
| "active_params": 58774528, |
| "flops_per_forward": 117549056.0, |
| "val_loss": 3.4686521768569945, |
| "val_perplexity": 32.09345697667592, |
| "bits_per_token": 5.0042072941202935, |
| "ppl_tokens_evaluated": 655360, |
| "prefill_tok_per_sec": 193194.22812667364, |
| "inference_tok_per_sec": 212.31968241677103, |
| "inference_tok_per_sec_kvcache": 210.12757536665706, |
| "kvcache_speedup": 0.9896754411783124, |
| "gen_tokens": 128, |
| "peak_infer_vram_gb": 0.366, |
| "peak_train_vram_gb": 3.993, |
| "closeness_answer_ppl": 42.281, |
| "closeness_hit_rate": 0.4, |
| "closeness_token_f1": 0.033, |
| "closeness_n": 20, |
| "closeness_items": [ |
| { |
| "prompt": "The capital of France is", |
| "answer": " Paris", |
| "generated": " the city of Paris.\n", |
| "answer_nll": 2.863, |
| "answer_ppl": 17.513, |
| "hit": true, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "The capital of Japan is", |
| "answer": " Tokyo", |
| "generated": " the city of Sichuan", |
| "answer_nll": 2.9536, |
| "answer_ppl": 19.174, |
| "hit": false, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "The largest planet in the solar system is", |
| "answer": " Jupiter", |
| "generated": " the Sun.\nThe Sun", |
| "answer_nll": 3.8845, |
| "answer_ppl": 48.642, |
| "hit": false, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "Water is made of hydrogen and", |
| "answer": " oxygen", |
| "generated": " oxygen.\nThe water is", |
| "answer_nll": 0.9818, |
| "answer_ppl": 2.669, |
| "hit": true, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "The opposite of hot is", |
| "answer": " cold", |
| "generated": " the heat of the sun.", |
| "answer_nll": 4.4139, |
| "answer_ppl": 82.587, |
| "hit": false, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "The opposite of up is", |
| "answer": " down", |
| "generated": " the fact that the two are", |
| "answer_nll": 8.6555, |
| "answer_ppl": 5741.871, |
| "hit": false, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "Two plus two equals", |
| "answer": " four", |
| "generated": " to the total number of people", |
| "answer_nll": 5.2942, |
| "answer_ppl": 199.172, |
| "hit": false, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "The sun rises in the", |
| "answer": " east", |
| "generated": " sky and the sun is at", |
| "answer_nll": 3.463, |
| "answer_ppl": 31.914, |
| "hit": false, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "The first president of the United States was George", |
| "answer": " Washington", |
| "generated": " Washington.\nThe second president", |
| "answer_nll": 1.038, |
| "answer_ppl": 2.824, |
| "hit": true, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "The speed of light is approximately 300,000 kilometers per", |
| "answer": " second", |
| "generated": " second.\nThe speed of", |
| "answer_nll": 0.1077, |
| "answer_ppl": 1.114, |
| "hit": true, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "A triangle has three", |
| "answer": " sides", |
| "generated": " sides. The two sides of", |
| "answer_nll": 1.6775, |
| "answer_ppl": 5.352, |
| "hit": true, |
| "token_f1": 0.333 |
| }, |
| { |
| "prompt": "The chemical symbol for gold is", |
| "answer": " Au", |
| "generated": " the gold ion.\nGold", |
| "answer_nll": 6.4909, |
| "answer_ppl": 659.103, |
| "hit": false, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "The freezing point of water in Celsius is", |
| "answer": " zero", |
| "generated": " about 0.5\u00b0C", |
| "answer_nll": 4.9866, |
| "answer_ppl": 146.444, |
| "hit": false, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "The author of Romeo and Juliet is William", |
| "answer": " Shakespeare", |
| "generated": " Shakespeare\u2019s play Romeo", |
| "answer_nll": 0.3631, |
| "answer_ppl": 1.438, |
| "hit": true, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "The Earth revolves around the", |
| "answer": " Sun", |
| "generated": " sun, and the sun is", |
| "answer_nll": 2.4816, |
| "answer_ppl": 11.961, |
| "hit": true, |
| "token_f1": 0.333 |
| }, |
| { |
| "prompt": "The plural of mouse is", |
| "answer": " mice", |
| "generated": " the same as the human.", |
| "answer_nll": 9.4341, |
| "answer_ppl": 12507.574, |
| "hit": false, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "Photosynthesis occurs in the", |
| "answer": " leaves", |
| "generated": " brain, and the brain is", |
| "answer_nll": 5.3131, |
| "answer_ppl": 202.985, |
| "hit": false, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "The human heart pumps", |
| "answer": " blood", |
| "generated": " are not only the heart pumps", |
| "answer_nll": 3.9707, |
| "answer_ppl": 53.021, |
| "hit": false, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "The currency used in the United States is the", |
| "answer": " dollar", |
| "generated": " United States dollar.\nThe", |
| "answer_nll": 3.4802, |
| "answer_ppl": 32.465, |
| "hit": true, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "The tallest animal in the world is the", |
| "answer": " giraffe", |
| "generated": " largest in the world.\n", |
| "answer_nll": 3.034, |
| "answer_ppl": 20.781, |
| "hit": false, |
| "token_f1": 0.0 |
| } |
| ], |
| "generation_samples": { |
| "The most important thing about sleep is": "The most important thing about sleep is to look for your own specific medical conditions, such as:\n- Heart conditions such as coronary heart disease, chronic heart failure, diabetes, and cancer.\n- If you have had any of the following:\n- Acute respiratory syndrome\n- High blood pressure\n- Heart disease\n- O", |
| "Scientists have discovered that": "Scientists have discovered that, in many ways, the human body is actually a computer system.\nThe first evidence for the existence of the human body is the presence of a long-lived entity. In a number of cases the body is not yet the subject of many other science studies, but in a much smaller number it", |
| "In the history of mathematics,": "In the history of mathematics, the term \"curriculum\" is often used to refer to the process in which schools use the term to describe the process in which a curriculum has been created. In schools, it is most commonly referred to as a \"curriculum\" or \"curriculum\" as it is also referred", |
| "Python code for prime number": "Python code for prime number of objects.\nHow to create a string using string\nTo create a string using string, you have to use a string you need. For example, we need the string:\n<stringname=\u201d\u201d\u201d </string>\nHow to create the string using string\nTo", |
| "The best way to learn a new language is": "The best way to learn a new language is to learn it in a second language. It is especially important to study the language in a way that is pleasing to the senses and allows you to build the foundation for yourself.\nFor instance, if you can\u2019t speak a language in a sentence, you can write a new sentence in it" |
| } |
| } |
