| { |
| "all_tokens_match": true, |
| "dtype": "int8-affine-all-linear", |
| "fixtures": [ |
| { |
| "beam_decode_first_seconds_after_parity": 0.007718416978605092, |
| "beam_decode_repeat_seconds": 0.005875291011761874, |
| "encoder_layers": [ |
| { |
| "cosine_similarity": 0.9999974126549085, |
| "maximum_absolute_error": 0.03477287292480469, |
| "mean_absolute_error": 0.0026948835317564257 |
| }, |
| { |
| "cosine_similarity": 0.999994178014735, |
| "maximum_absolute_error": 0.03160285949707031, |
| "mean_absolute_error": 0.003565792698381719 |
| }, |
| { |
| "cosine_similarity": 0.9999876302727856, |
| "maximum_absolute_error": 0.03735613822937012, |
| "mean_absolute_error": 0.005018145615454037 |
| }, |
| { |
| "cosine_similarity": 0.9999829393139568, |
| "maximum_absolute_error": 0.046927452087402344, |
| "mean_absolute_error": 0.0058425431963007365 |
| }, |
| { |
| "cosine_similarity": 0.9999812689294654, |
| "maximum_absolute_error": 0.04162287712097168, |
| "mean_absolute_error": 0.006123722175210344 |
| }, |
| { |
| "cosine_similarity": 0.9999797710902483, |
| "maximum_absolute_error": 0.06708478927612305, |
| "mean_absolute_error": 0.006552451898656924 |
| }, |
| { |
| "cosine_similarity": 0.9999771930246442, |
| "maximum_absolute_error": 0.0812063217163086, |
| "mean_absolute_error": 0.006925176367794724 |
| }, |
| { |
| "cosine_similarity": 0.9999738426320705, |
| "maximum_absolute_error": 0.09817290306091309, |
| "mean_absolute_error": 0.007099670186972936 |
| }, |
| { |
| "cosine_similarity": 0.9999683083186475, |
| "maximum_absolute_error": 0.12683701515197754, |
| "mean_absolute_error": 0.00763576833017694 |
| }, |
| { |
| "cosine_similarity": 0.9999677488408045, |
| "maximum_absolute_error": 0.12433314323425293, |
| "mean_absolute_error": 0.007547555017847141 |
| }, |
| { |
| "cosine_similarity": 0.9999720646262313, |
| "maximum_absolute_error": 0.15894126892089844, |
| "mean_absolute_error": 0.006999035167991924 |
| }, |
| { |
| "cosine_similarity": 0.9999940886428628, |
| "maximum_absolute_error": 0.09621238708496094, |
| "mean_absolute_error": 0.003924053548773274 |
| }, |
| { |
| "cosine_similarity": 0.9999879951948528, |
| "maximum_absolute_error": 0.06196737289428711, |
| "mean_absolute_error": 0.004866445243693011 |
| }, |
| { |
| "cosine_similarity": 0.9999814145240172, |
| "maximum_absolute_error": 0.0691235363483429, |
| "mean_absolute_error": 0.00629057279581527 |
| }, |
| { |
| "cosine_similarity": 0.9999845548930014, |
| "maximum_absolute_error": 0.10037994384765625, |
| "mean_absolute_error": 0.006365957403597742 |
| }, |
| { |
| "cosine_similarity": 0.9999585913853329, |
| "maximum_absolute_error": 0.02571159601211548, |
| "mean_absolute_error": 0.0024021906953625937 |
| } |
| ], |
| "first_step_logits": { |
| "cosine_similarity": 0.9999605972620904, |
| "maximum_absolute_error": 0.2038872241973877, |
| "mean_absolute_error": 0.11693925559520721 |
| }, |
| "fixture": "hello_en_1_5s.wav", |
| "frontend": { |
| "cosine_similarity": 1.0000000000000002, |
| "maximum_absolute_error": 0.0, |
| "mean_absolute_error": 0.0 |
| }, |
| "maximum_encoder_layer_absolute_error": 0.15894126892089844, |
| "minimum_encoder_layer_cosine": 0.9999585913853329, |
| "mlx_confidence": 0.9977218508720398, |
| "mlx_token_ids": [ |
| 5 |
| ], |
| "official_confidence": 0.9978703260421753, |
| "official_token_ids": [ |
| 5 |
| ], |
| "second_step_logits": { |
| "cosine_similarity": 0.999998828530253, |
| "maximum_absolute_error": 0.08715057373046875, |
| "mean_absolute_error": 0.03150423069794973 |
| }, |
| "subsampling": { |
| "cosine_similarity": 0.9999998613939332, |
| "maximum_absolute_error": 0.002836465835571289, |
| "mean_absolute_error": 0.00010501035792001372 |
| }, |
| "text": "en", |
| "token_match": true, |
| "warm_repeat_match": true |
| }, |
| { |
| "beam_decode_first_seconds_after_parity": 0.00900483300210908, |
| "beam_decode_repeat_seconds": 0.008772417029831558, |
| "encoder_layers": [ |
| { |
| "cosine_similarity": 0.9999974935504741, |
| "maximum_absolute_error": 0.033333778381347656, |
| "mean_absolute_error": 0.0026803150121683765 |
| }, |
| { |
| "cosine_similarity": 0.9999935244078233, |
| "maximum_absolute_error": 0.051532745361328125, |
| "mean_absolute_error": 0.00374875220180338 |
| }, |
| { |
| "cosine_similarity": 0.9999883498367391, |
| "maximum_absolute_error": 0.04737663269042969, |
| "mean_absolute_error": 0.0048553833886984124 |
| }, |
| { |
| "cosine_similarity": 0.9999823744619646, |
| "maximum_absolute_error": 0.048705101013183594, |
| "mean_absolute_error": 0.005919573505089951 |
| }, |
| { |
| "cosine_similarity": 0.9999831795662019, |
| "maximum_absolute_error": 0.04788041114807129, |
| "mean_absolute_error": 0.005786362481257546 |
| }, |
| { |
| "cosine_similarity": 0.9999839517784685, |
| "maximum_absolute_error": 0.06499552726745605, |
| "mean_absolute_error": 0.005804667324823945 |
| }, |
| { |
| "cosine_similarity": 0.9999849617964995, |
| "maximum_absolute_error": 0.08105182647705078, |
| "mean_absolute_error": 0.005640252763750066 |
| }, |
| { |
| "cosine_similarity": 0.9999830088797715, |
| "maximum_absolute_error": 0.07564067840576172, |
| "mean_absolute_error": 0.0057925679361637405 |
| }, |
| { |
| "cosine_similarity": 0.9999811645957883, |
| "maximum_absolute_error": 0.08871650695800781, |
| "mean_absolute_error": 0.006029855238833632 |
| }, |
| { |
| "cosine_similarity": 0.9999822326545446, |
| "maximum_absolute_error": 0.12425971031188965, |
| "mean_absolute_error": 0.005776002957197444 |
| }, |
| { |
| "cosine_similarity": 0.9999857079204477, |
| "maximum_absolute_error": 0.09299468994140625, |
| "mean_absolute_error": 0.005288105543541033 |
| }, |
| { |
| "cosine_similarity": 0.999997093081792, |
| "maximum_absolute_error": 0.064422607421875, |
| "mean_absolute_error": 0.00290668242098145 |
| }, |
| { |
| "cosine_similarity": 0.9999944434031225, |
| "maximum_absolute_error": 0.05225181579589844, |
| "mean_absolute_error": 0.003416959429003031 |
| }, |
| { |
| "cosine_similarity": 0.9999917854381818, |
| "maximum_absolute_error": 0.061490222811698914, |
| "mean_absolute_error": 0.004365877812463473 |
| }, |
| { |
| "cosine_similarity": 0.999991063061805, |
| "maximum_absolute_error": 0.08561515808105469, |
| "mean_absolute_error": 0.004931087964572041 |
| }, |
| { |
| "cosine_similarity": 0.9999811389617621, |
| "maximum_absolute_error": 0.02193164825439453, |
| "mean_absolute_error": 0.001859259993716009 |
| } |
| ], |
| "first_step_logits": { |
| "cosine_similarity": 0.9999972948978457, |
| "maximum_absolute_error": 0.15688323974609375, |
| "mean_absolute_error": 0.03872461318969726 |
| }, |
| "fixture": "hello_en.wav", |
| "frontend": { |
| "cosine_similarity": 1.0000000000000002, |
| "maximum_absolute_error": 0.0, |
| "mean_absolute_error": 0.0 |
| }, |
| "maximum_encoder_layer_absolute_error": 0.12425971031188965, |
| "minimum_encoder_layer_cosine": 0.9999811389617621, |
| "mlx_confidence": 0.9993755221366882, |
| "mlx_token_ids": [ |
| 5 |
| ], |
| "official_confidence": 0.9993982315063477, |
| "official_token_ids": [ |
| 5 |
| ], |
| "second_step_logits": { |
| "cosine_similarity": 0.9999984379603233, |
| "maximum_absolute_error": 0.08304190635681152, |
| "mean_absolute_error": 0.01872244824965795 |
| }, |
| "subsampling": { |
| "cosine_similarity": 0.999999885708639, |
| "maximum_absolute_error": 0.003216266632080078, |
| "mean_absolute_error": 0.00010146924228612561 |
| }, |
| "text": "en", |
| "token_match": true, |
| "warm_repeat_match": true |
| }, |
| { |
| "beam_decode_first_seconds_after_parity": 0.009719291003420949, |
| "beam_decode_repeat_seconds": 0.009087582991924137, |
| "encoder_layers": [ |
| { |
| "cosine_similarity": 0.9999972922084077, |
| "maximum_absolute_error": 0.028516769409179688, |
| "mean_absolute_error": 0.0027094290655879337 |
| }, |
| { |
| "cosine_similarity": 0.9999939202528341, |
| "maximum_absolute_error": 0.046573638916015625, |
| "mean_absolute_error": 0.0036505912165029166 |
| }, |
| { |
| "cosine_similarity": 0.999991309707319, |
| "maximum_absolute_error": 0.04838156700134277, |
| "mean_absolute_error": 0.00425935310722207 |
| }, |
| { |
| "cosine_similarity": 0.9999886882830208, |
| "maximum_absolute_error": 0.041106224060058594, |
| "mean_absolute_error": 0.004784165055859463 |
| }, |
| { |
| "cosine_similarity": 0.9999885341710141, |
| "maximum_absolute_error": 0.04517030715942383, |
| "mean_absolute_error": 0.004878931270808032 |
| }, |
| { |
| "cosine_similarity": 0.9999896078643795, |
| "maximum_absolute_error": 0.04493267834186554, |
| "mean_absolute_error": 0.004798952600995061 |
| }, |
| { |
| "cosine_similarity": 0.9999901949893468, |
| "maximum_absolute_error": 0.06097888946533203, |
| "mean_absolute_error": 0.0046641416468206675 |
| }, |
| { |
| "cosine_similarity": 0.9999901206158142, |
| "maximum_absolute_error": 0.07126331329345703, |
| "mean_absolute_error": 0.004515139120300142 |
| }, |
| { |
| "cosine_similarity": 0.9999883426901348, |
| "maximum_absolute_error": 0.05031013488769531, |
| "mean_absolute_error": 0.004868494294667376 |
| }, |
| { |
| "cosine_similarity": 0.9999887730848651, |
| "maximum_absolute_error": 0.06040763854980469, |
| "mean_absolute_error": 0.0046648543612886634 |
| }, |
| { |
| "cosine_similarity": 0.9999906585084845, |
| "maximum_absolute_error": 0.07903194427490234, |
| "mean_absolute_error": 0.004394933146108778 |
| }, |
| { |
| "cosine_similarity": 0.9999983121053144, |
| "maximum_absolute_error": 0.07384300231933594, |
| "mean_absolute_error": 0.002289940727397192 |
| }, |
| { |
| "cosine_similarity": 0.9999959708925714, |
| "maximum_absolute_error": 0.07651710510253906, |
| "mean_absolute_error": 0.0029475082223536565 |
| }, |
| { |
| "cosine_similarity": 0.999993553575914, |
| "maximum_absolute_error": 0.05951118469238281, |
| "mean_absolute_error": 0.003950899197058892 |
| }, |
| { |
| "cosine_similarity": 0.9999943130674037, |
| "maximum_absolute_error": 0.07894468307495117, |
| "mean_absolute_error": 0.004051757462801972 |
| }, |
| { |
| "cosine_similarity": 0.9999892834933597, |
| "maximum_absolute_error": 0.01026219129562378, |
| "mean_absolute_error": 0.0013744048593582296 |
| } |
| ], |
| "first_step_logits": { |
| "cosine_similarity": 0.9999993527115524, |
| "maximum_absolute_error": 0.13068771362304688, |
| "mean_absolute_error": 0.04440820962190628 |
| }, |
| "fixture": "hello_zh.wav", |
| "frontend": { |
| "cosine_similarity": 1.0, |
| "maximum_absolute_error": 0.0, |
| "mean_absolute_error": 0.0 |
| }, |
| "maximum_encoder_layer_absolute_error": 0.07903194427490234, |
| "minimum_encoder_layer_cosine": 0.9999883426901348, |
| "mlx_confidence": 0.9997503161430359, |
| "mlx_token_ids": [ |
| 8, |
| 14 |
| ], |
| "official_confidence": 0.9997549057006836, |
| "official_token_ids": [ |
| 8, |
| 14 |
| ], |
| "second_step_logits": { |
| "cosine_similarity": 0.9999982286853081, |
| "maximum_absolute_error": 0.08087924122810364, |
| "mean_absolute_error": 0.017943078776200612 |
| }, |
| "subsampling": { |
| "cosine_similarity": 0.9999998655302096, |
| "maximum_absolute_error": 0.00445103645324707, |
| "mean_absolute_error": 9.755159161937613e-05 |
| }, |
| "text": "zh mandarin", |
| "token_match": true, |
| "warm_repeat_match": true |
| }, |
| { |
| "beam_decode_first_seconds_after_parity": 0.015697707945946604, |
| "beam_decode_repeat_seconds": 0.015121500007808208, |
| "encoder_layers": [ |
| { |
| "cosine_similarity": 0.9999974753041487, |
| "maximum_absolute_error": 0.04085540771484375, |
| "mean_absolute_error": 0.0026753888418655303 |
| }, |
| { |
| "cosine_similarity": 0.999993524505242, |
| "maximum_absolute_error": 0.05753898620605469, |
| "mean_absolute_error": 0.0037407932574813913 |
| }, |
| { |
| "cosine_similarity": 0.9999883443769058, |
| "maximum_absolute_error": 0.04660797119140625, |
| "mean_absolute_error": 0.00486449089398839 |
| }, |
| { |
| "cosine_similarity": 0.9999819825261375, |
| "maximum_absolute_error": 0.049437910318374634, |
| "mean_absolute_error": 0.005960059630000918 |
| }, |
| { |
| "cosine_similarity": 0.9999831096485101, |
| "maximum_absolute_error": 0.06641387939453125, |
| "mean_absolute_error": 0.005792712864395101 |
| }, |
| { |
| "cosine_similarity": 0.9999841877241425, |
| "maximum_absolute_error": 0.058662623167037964, |
| "mean_absolute_error": 0.005784846874545626 |
| }, |
| { |
| "cosine_similarity": 0.9999852832779774, |
| "maximum_absolute_error": 0.05826053023338318, |
| "mean_absolute_error": 0.005617633600221018 |
| }, |
| { |
| "cosine_similarity": 0.999983699812909, |
| "maximum_absolute_error": 0.08028793334960938, |
| "mean_absolute_error": 0.005713151763618771 |
| }, |
| { |
| "cosine_similarity": 0.9999820278934909, |
| "maximum_absolute_error": 0.0823819637298584, |
| "mean_absolute_error": 0.005916724377417874 |
| }, |
| { |
| "cosine_similarity": 0.9999831747886415, |
| "maximum_absolute_error": 0.0947113037109375, |
| "mean_absolute_error": 0.005652063742734814 |
| }, |
| { |
| "cosine_similarity": 0.9999865261979987, |
| "maximum_absolute_error": 0.09869718551635742, |
| "mean_absolute_error": 0.005163557694032564 |
| }, |
| { |
| "cosine_similarity": 0.9999972361836617, |
| "maximum_absolute_error": 0.07796669006347656, |
| "mean_absolute_error": 0.002831356242392955 |
| }, |
| { |
| "cosine_similarity": 0.999994856878261, |
| "maximum_absolute_error": 0.0582275390625, |
| "mean_absolute_error": 0.0033022663774843674 |
| }, |
| { |
| "cosine_similarity": 0.9999914981830116, |
| "maximum_absolute_error": 0.053646087646484375, |
| "mean_absolute_error": 0.004413037563174233 |
| }, |
| { |
| "cosine_similarity": 0.9999922027637055, |
| "maximum_absolute_error": 0.07339668273925781, |
| "mean_absolute_error": 0.004643678797878647 |
| }, |
| { |
| "cosine_similarity": 0.9999865593903893, |
| "maximum_absolute_error": 0.015595287084579468, |
| "mean_absolute_error": 0.001871918864069668 |
| } |
| ], |
| "first_step_logits": { |
| "cosine_similarity": 0.9999989197783602, |
| "maximum_absolute_error": 0.07843208312988281, |
| "mean_absolute_error": 0.021767886479695638 |
| }, |
| "fixture": "hello_en_30s.wav", |
| "frontend": { |
| "cosine_similarity": 0.9999999999999998, |
| "maximum_absolute_error": 0.0, |
| "mean_absolute_error": 0.0 |
| }, |
| "maximum_encoder_layer_absolute_error": 0.09869718551635742, |
| "minimum_encoder_layer_cosine": 0.9999819825261375, |
| "mlx_confidence": 1.0, |
| "mlx_token_ids": [ |
| 5 |
| ], |
| "official_confidence": 0.9999996423721313, |
| "official_token_ids": [ |
| 5 |
| ], |
| "second_step_logits": { |
| "cosine_similarity": 0.999998785095425, |
| "maximum_absolute_error": 0.1163177490234375, |
| "mean_absolute_error": 0.02773282987376054 |
| }, |
| "subsampling": { |
| "cosine_similarity": 0.9999998846548594, |
| "maximum_absolute_error": 0.0032448768615722656, |
| "mean_absolute_error": 0.00010198331100249356 |
| }, |
| "text": "en", |
| "token_match": true, |
| "warm_repeat_match": true |
| } |
| ], |
| "format": "mlx", |
| "minimum_decoder_logit_cosine": 0.9999605972620904, |
| "minimum_encoder_layer_cosine": 0.9999585913853329, |
| "peak_memory_bytes": 1725457036 |
| } |
|
|
