| { |
| "schema": "lfm25-conversion-gate/1", |
| "result": "PASS", |
| "bundle": "exports/lfm2_5_2_6b_decode_int8hu_attnfp16_block32_sym", |
| "mode": "int8hu", |
| "cases_exact": 5, |
| "cases": 5, |
| "word_prefix": 65, |
| "expected_words": 65, |
| "quality": { |
| "teacher_forced_top1_agree": 122, |
| "positions": 125, |
| "sequences": 5, |
| "cosine_min": 0.9970501333100017, |
| "cosine_mean": 0.9997748702116448, |
| "per_sequence": [ |
| { |
| "case": 0, |
| "agree": 20, |
| "positions": 21, |
| "cosine_min": 0.9970501333100017 |
| }, |
| { |
| "case": 1, |
| "agree": 20, |
| "positions": 20, |
| "cosine_min": 0.9988472272394007 |
| }, |
| { |
| "case": 2, |
| "agree": 24, |
| "positions": 24, |
| "cosine_min": 0.9997840496708345 |
| }, |
| { |
| "case": 3, |
| "agree": 27, |
| "positions": 28, |
| "cosine_min": 0.9995996541432354 |
| }, |
| { |
| "case": 4, |
| "agree": 31, |
| "positions": 32, |
| "cosine_min": 0.9985697415741762 |
| } |
| ] |
| }, |
| "detail": [ |
| { |
| "case": 0, |
| "prompt": "The capital of France is", |
| "exact": true, |
| "expected_text": " Paris. (A)\n* \"The capital of France is Paris.\" (", |
| "bundle_text": " Paris. (A)\n* \"The capital of France is Paris.\" (", |
| "word_prefix": 10, |
| "expected_words": 10 |
| }, |
| { |
| "case": 1, |
| "prompt": "def fibonacci(n):", |
| "exact": true, |
| "expected_text": " function that takes an integer n and returns the nth Fibonacci number. fibonacci(0", |
| "bundle_text": " function that takes an integer n and returns the nth Fibonacci number. fibonacci(0", |
| "word_prefix": 13, |
| "expected_words": 13 |
| }, |
| { |
| "case": 2, |
| "prompt": "The second law of thermodynamics states that", |
| "exact": true, |
| "expected_text": " the total entropy of an isolated system can never decrease over time. This is a", |
| "bundle_text": " the total entropy of an isolated system can never decrease over time. This is a", |
| "word_prefix": 15, |
| "expected_words": 15 |
| }, |
| { |
| "case": 3, |
| "prompt": "Q: What is 17 * 23?\nA:", |
| "exact": true, |
| "expected_text": " 391\n\nThe user is asking for a simple multiplication. I should provide the", |
| "bundle_text": " 391\n\nThe user is asking for a simple multiplication. I should provide the", |
| "word_prefix": 13, |
| "expected_words": 13 |
| }, |
| { |
| "case": 4, |
| "prompt": "Once upon a time, in a small village at the edge of a forest,", |
| "exact": true, |
| "expected_text": " there lived a young woman named Lila. She was known for her kindness and", |
| "bundle_text": " there lived a young woman named Lila. She was known for her kindness and", |
| "word_prefix": 14, |
| "expected_words": 14 |
| } |
| ] |
| } |
|
|
