File size: 2,844 Bytes
f9da83b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 | {
"schema": "lfm25-conversion-gate/1",
"result": "PASS",
"bundle": "exports/lfm2_5_2_6b_decode_int8hu_attnfp16_block32_sym",
"mode": "int8hu",
"cases_exact": 5,
"cases": 5,
"word_prefix": 65,
"expected_words": 65,
"quality": {
"teacher_forced_top1_agree": 122,
"positions": 125,
"sequences": 5,
"cosine_min": 0.9970501333100017,
"cosine_mean": 0.9997748702116448,
"per_sequence": [
{
"case": 0,
"agree": 20,
"positions": 21,
"cosine_min": 0.9970501333100017
},
{
"case": 1,
"agree": 20,
"positions": 20,
"cosine_min": 0.9988472272394007
},
{
"case": 2,
"agree": 24,
"positions": 24,
"cosine_min": 0.9997840496708345
},
{
"case": 3,
"agree": 27,
"positions": 28,
"cosine_min": 0.9995996541432354
},
{
"case": 4,
"agree": 31,
"positions": 32,
"cosine_min": 0.9985697415741762
}
]
},
"detail": [
{
"case": 0,
"prompt": "The capital of France is",
"exact": true,
"expected_text": " Paris. (A)\n* \"The capital of France is Paris.\" (",
"bundle_text": " Paris. (A)\n* \"The capital of France is Paris.\" (",
"word_prefix": 10,
"expected_words": 10
},
{
"case": 1,
"prompt": "def fibonacci(n):",
"exact": true,
"expected_text": " function that takes an integer n and returns the nth Fibonacci number. fibonacci(0",
"bundle_text": " function that takes an integer n and returns the nth Fibonacci number. fibonacci(0",
"word_prefix": 13,
"expected_words": 13
},
{
"case": 2,
"prompt": "The second law of thermodynamics states that",
"exact": true,
"expected_text": " the total entropy of an isolated system can never decrease over time. This is a",
"bundle_text": " the total entropy of an isolated system can never decrease over time. This is a",
"word_prefix": 15,
"expected_words": 15
},
{
"case": 3,
"prompt": "Q: What is 17 * 23?\nA:",
"exact": true,
"expected_text": " 391\n\nThe user is asking for a simple multiplication. I should provide the",
"bundle_text": " 391\n\nThe user is asking for a simple multiplication. I should provide the",
"word_prefix": 13,
"expected_words": 13
},
{
"case": 4,
"prompt": "Once upon a time, in a small village at the edge of a forest,",
"exact": true,
"expected_text": " there lived a young woman named Lila. She was known for her kindness and",
"bundle_text": " there lived a young woman named Lila. She was known for her kindness and",
"word_prefix": 14,
"expected_words": 14
}
]
}
|