| { |
| "scope": "CPU CLI smoke/conformance, synthetic waveforms through real pretrained MERT; not retrieval evaluation", |
| "mert_preserved_in_quantized": true, |
| "clamp3_397_source_audited_tensors_preserved": true, |
| "cases": { |
| "f32": { |
| "English": { |
| "max_abs": 0.0, |
| "rmse": 0.0, |
| "relative_l2": 0.0, |
| "cosine": 1.0000000000000002, |
| "norm": 2.875889854969512, |
| "finite_768": true, |
| "vs_independent_original_f32_reference": { |
| "max_abs": 2.384185791015625e-07, |
| "rmse": 8.135369907297125e-08, |
| "relative_l2": 7.839452562324725e-07, |
| "cosine": 0.9999999999997062, |
| "norm": 2.875889854969512 |
| } |
| }, |
| "Chinese": { |
| "max_abs": 0.0, |
| "rmse": 0.0, |
| "relative_l2": 0.0, |
| "cosine": 0.9999999999999999, |
| "norm": 3.5457470949243235, |
| "finite_768": true, |
| "vs_independent_original_f32_reference": { |
| "max_abs": 2.942979335784912e-07, |
| "rmse": 9.953604191894943e-08, |
| "relative_l2": 7.779527056134522e-07, |
| "cosine": 0.9999999999996992, |
| "norm": 3.5457470949243235 |
| } |
| }, |
| "broadband": { |
| "max_abs": 0.0, |
| "rmse": 0.0, |
| "relative_l2": 0.0, |
| "cosine": 1.0000000000000002, |
| "norm": 16.45825506212605, |
| "finite_768": true |
| }, |
| "two_chunks": { |
| "max_abs": 0.0, |
| "rmse": 0.0, |
| "relative_l2": 0.0, |
| "cosine": 1.0000000000000002, |
| "norm": 16.52023833601139, |
| "finite_768": true |
| }, |
| "resampled": { |
| "max_abs": 0.0, |
| "rmse": 0.0, |
| "relative_l2": 0.0, |
| "cosine": 1.0, |
| "norm": 16.411079182039764, |
| "finite_768": true |
| }, |
| "resampled-input": { |
| "max_abs": 0.0, |
| "rmse": 0.0, |
| "relative_l2": 0.0, |
| "cosine": 1.0, |
| "norm": 16.411079182039764, |
| "finite_768": true |
| } |
| }, |
| "f16": { |
| "English": { |
| "max_abs": 0.00022810325026512146, |
| "rmse": 7.088559774306416e-05, |
| "relative_l2": 0.0006830718171141, |
| "cosine": 0.9999997667319563, |
| "norm": 2.875868631349859, |
| "finite_768": true, |
| "vs_independent_original_f32_reference": { |
| "max_abs": 0.00022817403078079224, |
| "rmse": 7.089716082045669e-05, |
| "relative_l2": 0.0006831833529252924, |
| "cosine": 0.9999997666546141, |
| "norm": 2.875868631349859 |
| } |
| }, |
| "Chinese": { |
| "max_abs": 0.0002489835023880005, |
| "rmse": 7.447915636778748e-05, |
| "relative_l2": 0.0005821134084539248, |
| "cosine": 0.9999998308906621, |
| "norm": 3.5456569785249474, |
| "finite_768": true, |
| "vs_independent_original_f32_reference": { |
| "max_abs": 0.0002489835023880005, |
| "rmse": 7.447880227569815e-05, |
| "relative_l2": 0.0005821106066123175, |
| "cosine": 0.9999998308937837, |
| "norm": 3.5456569785249474 |
| } |
| }, |
| "broadband": { |
| "max_abs": 0.0012890994548797607, |
| "rmse": 0.00032566747492797815, |
| "relative_l2": 0.0005483668695799694, |
| "cosine": 0.9999998505157217, |
| "norm": 16.45756652969884, |
| "finite_768": true |
| }, |
| "two_chunks": { |
| "max_abs": 0.00105363130569458, |
| "rmse": 0.000330052133283634, |
| "relative_l2": 0.0005536647134177084, |
| "cosine": 0.9999998469812418, |
| "norm": 16.520607833229146, |
| "finite_768": true |
| }, |
| "resampled": { |
| "max_abs": 0.001208961009979248, |
| "rmse": 0.00033640600577405206, |
| "relative_l2": 0.0005680770045734879, |
| "cosine": 0.9999998386835246, |
| "norm": 16.410931078233933, |
| "finite_768": true |
| }, |
| "resampled-input": { |
| "max_abs": 0.001208961009979248, |
| "rmse": 0.00033640600577405206, |
| "relative_l2": 0.0005680770045734879, |
| "cosine": 0.9999998386835246, |
| "norm": 16.410931078233933, |
| "finite_768": true |
| } |
| }, |
| "q8_0": { |
| "English": { |
| "max_abs": 0.015449784812517464, |
| "rmse": 0.003922133034160892, |
| "relative_l2": 0.03779468078012584, |
| "cosine": 0.9992860618049164, |
| "norm": 2.8708601182372364, |
| "finite_768": true, |
| "vs_independent_original_f32_reference": { |
| "max_abs": 0.015449812752194703, |
| "rmse": 0.00392213394523595, |
| "relative_l2": 0.037794695701767037, |
| "cosine": 0.9992860610717784, |
| "norm": 2.8708601182372364 |
| } |
| }, |
| "Chinese": { |
| "max_abs": 0.013699561357498169, |
| "rmse": 0.004458963924226846, |
| "relative_l2": 0.03485032342857464, |
| "cosine": 0.9994129382139674, |
| "norm": 3.5210265136557255, |
| "finite_768": true, |
| "vs_independent_original_f32_reference": { |
| "max_abs": 0.0136994868516922, |
| "rmse": 0.004458938652337973, |
| "relative_l2": 0.034850123853381154, |
| "cosine": 0.9994129455946132, |
| "norm": 3.5210265136557255 |
| } |
| }, |
| "broadband": { |
| "max_abs": 0.04030671343207359, |
| "rmse": 0.008613078940262394, |
| "relative_l2": 0.014502913245976976, |
| "cosine": 0.9998948286547856, |
| "norm": 16.457404788498625, |
| "finite_768": true |
| }, |
| "two_chunks": { |
| "max_abs": 0.029031038284301758, |
| "rmse": 0.008333094256655871, |
| "relative_l2": 0.013978822671414926, |
| "cosine": 0.9999023794545989, |
| "norm": 16.511696523537914, |
| "finite_768": true |
| }, |
| "resampled": { |
| "max_abs": 0.03081924468278885, |
| "rmse": 0.008371378748487108, |
| "relative_l2": 0.014136453220115258, |
| "cosine": 0.9999002635076021, |
| "norm": 16.39937567329061, |
| "finite_768": true |
| }, |
| "resampled-input": { |
| "max_abs": 0.03081924468278885, |
| "rmse": 0.008371378748487108, |
| "relative_l2": 0.014136453220115258, |
| "cosine": 0.9999002635076021, |
| "norm": 16.39937567329061, |
| "finite_768": true |
| } |
| }, |
| "q4_k_m": { |
| "English": { |
| "max_abs": 0.08015982061624527, |
| "rmse": 0.020998184602702766, |
| "relative_l2": 0.20234389734082361, |
| "cosine": 0.9797742320216475, |
| "norm": 2.73141739496616, |
| "finite_768": true, |
| "vs_independent_original_f32_reference": { |
| "max_abs": 0.08015970140695572, |
| "rmse": 0.020998165788392677, |
| "relative_l2": 0.20234374892564083, |
| "cosine": 0.9797742587638331, |
| "norm": 2.73141739496616 |
| } |
| }, |
| "Chinese": { |
| "max_abs": 0.06565920449793339, |
| "rmse": 0.020778767113661906, |
| "relative_l2": 0.16240246987055623, |
| "cosine": 0.9867267070631845, |
| "norm": 3.5059123775977254, |
| "finite_768": true, |
| "vs_independent_original_f32_reference": { |
| "max_abs": 0.06565918028354645, |
| "rmse": 0.020778765756177996, |
| "relative_l2": 0.16240244968239376, |
| "cosine": 0.9867267102588001, |
| "norm": 3.5059123775977254 |
| } |
| }, |
| "broadband": { |
| "max_abs": 0.18925083428621292, |
| "rmse": 0.05830344263772219, |
| "relative_l2": 0.09817276451095897, |
| "cosine": 0.9951840840091468, |
| "norm": 16.468009306816164, |
| "finite_768": true |
| }, |
| "two_chunks": { |
| "max_abs": 0.2002243995666504, |
| "rmse": 0.05890692826046174, |
| "relative_l2": 0.09881677548685394, |
| "cosine": 0.995115805901144, |
| "norm": 16.51384044329767, |
| "finite_768": true |
| }, |
| "resampled": { |
| "max_abs": 0.18772372230887413, |
| "rmse": 0.05838769136718791, |
| "relative_l2": 0.09859724335037957, |
| "cosine": 0.9951285778564337, |
| "norm": 16.35603800628563, |
| "finite_768": true |
| }, |
| "resampled-input": { |
| "max_abs": 0.18772372230887413, |
| "rmse": 0.05838769136718791, |
| "relative_l2": 0.09859724335037957, |
| "cosine": 0.9951285778564337, |
| "norm": 16.35603800628563, |
| "finite_768": true |
| } |
| } |
| } |
| } |
|
|
