FrozenDict({ classifier: { bias: Array([ 0.13959047, -0.15330061, 0.23963794, 0.26991433, 0.14193657, -0.07303967, -0.15755221, -0.00429095, -0.1597117 , 0.01030699], dtype=float32), kernel: Array([[-0.21733038, -0.04500001, 0.6502397 , ..., -0.16637883, -0.19785312, -0.19238858], [-0.05325412, 0.08365323, 0.04513923, ..., 0.04858669, -0.2313862 , 0.15936573], [ 0.15941077, -0.5403379 , 0.17727152, ..., 0.23605543, -0.5477616 , 0.400377 ], ..., [ 0.14339526, -0.40216786, -0.11720804, ..., -0.4565188 , 0.55370927, -0.3628201 ], [ 0.1330832 , -0.05788376, 0.05994943, ..., -0.05280861, -0.35830075, -0.07267395], [ 0.07371017, 0.0797717 , 0.05526711, ..., -0.15986513, -0.19333237, -0.11412273]], dtype=float32), }, vit: { embeddings: { cls_token: Array([[[ 4.21749912e-02, -7.63344541e-02, -9.38936546e-02, 3.23799951e-03, 1.69068575e-01, 2.25809757e-02, 3.44559725e-04, -3.61712719e-03, -8.03779662e-02, 3.50343175e-02, -2.25252569e-01, 8.41705650e-02, -1.40322876e+00, -6.44051209e-02, 3.56393009e-02, 1.55636281e-01, 1.44712580e-02, -1.67391926e-01, 1.13313071e-01, 2.03049585e-01, -1.21489443e-01, 3.01832557e-01, -6.39758036e-02, 2.43316695e-01, 8.19623619e-02, 3.64291817e-01, 2.20934585e-01, 4.44887951e-02, -1.20811634e-01, 7.45030791e-02, 2.88756996e-01, 2.45113447e-01, -1.02346905e-01, -1.44495424e-02, -2.92423535e-02, 1.27275869e-01, -4.68343794e-02, 6.18743040e-02, 1.62268832e-01, 7.68043697e-02, 8.22397470e-02, 4.15349565e-02, 4.29975651e-02, 3.34680080e-01, 1.23265073e-01, -1.82213470e-01, -2.61384398e-01, -2.86995154e-02, 4.78795953e-02, -7.12198839e-02, -9.28327814e-02, -1.04730994e-01, -2.08872005e-01, 5.08915447e-02, -2.92177856e-01, -4.63173315e-02, -1.39276087e-01, -6.45741746e-02, 4.31298822e-01, -2.79950589e-01, -3.42224717e-01, -2.57990032e-01, 7.88487419e-02, 1.75719306e-01, -1.91833928e-01, -1.60736710e-01, 2.24118829e-01, -1.42223332e-02, -1.56884298e-01, -1.34610429e-01, -7.02320859e-02, -5.77486083e-02, -2.67209888e-01, -1.95046023e-01, 1.72840059e-01, 5.77186942e-02, 1.49908558e-01, 1.20706670e-01, 2.60736823e-01, -4.85861525e-02, 1.50602728e-01, -1.42758638e-01, 2.74297837e-02, 2.43728027e-01, 2.83344448e-01, -3.49160563e-03, -1.21281445e-01, -1.72878504e-01, -1.20296307e-01, -5.02357781e-02, 8.33122134e-02, -1.96546650e+00, 1.50719002e-01, 2.28078086e-02, -1.70492530e-01, 4.20110017e-01, -9.01997983e-02, -3.95133868e-02, 1.77724794e-01, 1.20884374e-01, -3.86571921e-02, -7.12739974e-02, 1.50522381e-01, 1.34570688e-01, 3.98968756e-01, -1.69036195e-01, 7.98634291e-02, -2.08218116e-02, -1.60120368e-01, -2.48059444e-02, -1.43607566e-02, 3.75894368e-01, 8.90320763e-02, -1.83609471e-01, 3.58941674e-01, -1.86423570e-01, 2.57358737e-02, 1.14504859e-01, -1.51107222e-01, -1.81861663e+00, -1.02884192e-02, -3.45049873e-02, 7.34117031e-02, 6.87620267e-02, 6.22403771e-02, 3.00719798e-01, 6.51065707e-02, 3.67412001e-01]]], dtype=float32), patch_embeddings: { projection: { bias: Array([-0.23706354, -1.6540319 , -0.56307954, 0.26430997, 0.45925793, 0.10770348, -0.64220166, -0.256316 , -0.70986354, 0.00280434, -0.8621466 , 0.33027068, -0.09428401, -0.18285578, 0.48031166, 0.31876177, 0.1595892 , -0.7937292 , 0.41048172, 0.7553594 , -0.11688971, 1.0029606 , -0.19344132, 0.9868722 , 0.33903363, 1.4018133 , 0.6315837 , 0.15693058, -0.43901056, -0.03160467, 1.4104275 , 0.8133011 , -0.8685446 , 0.09167413, -0.38324404, 0.4127237 , -0.0992351 , -0.8858117 , 0.49157575, 1.1184502 , 0.29291674, 0.09339268, 0.10110624, 1.3492149 , 0.23373768, -0.62969214, -1.1953127 , -0.32950965, 0.16054444, -0.04147059, -0.32538444, -0.44066548, -0.81780577, 0.3041742 , -0.88412094, -0.2175792 , -0.7703008 , -0.29075882, 1.4584533 , -0.90483326, -1.0301116 , -1.9739118 , 0.30468434, 0.47119713, -2.507297 , -0.4836628 , 0.7882562 , 0.13242652, -0.64575255, -0.5046555 , -0.2928836 , -0.19443981, -1.8653952 , -0.6447174 , 0.53096473, 0.53206563, 0.53241354, 0.5417067 , 0.87263006, -0.22854666, 0.57786745, -0.3563041 , -0.03160271, 0.5421835 , 1.0412699 , -0.1696869 , -0.4523725 , -1.6923577 , -0.46177727, -0.20988323, 0.2240901 , -0.9807764 , 0.61122894, -1.4953612 , -0.6922171 , 1.6215447 , -0.32785478, -0.6734319 , 0.5917648 , 0.33969748, -0.27886963, -0.50457156, 0.5443144 , 0.5749297 , 2.0102262 , 0.15956564, 0.03995752, 0.18428732, 0.1132739 , 1.1407704 , 0.10695937, 0.6835315 , 0.15768398, -0.5695771 , 0.45638716, -0.6475178 , -0.24609064, 0.36820015, -0.7532235 , -0.60841304, -0.13496049, -0.49064922, 0.31223625, 0.27192888, 0.27554023, 1.1431948 , 0.30906013, 1.071581 ], dtype=float32), kernel: Array([[[[ 0.1773554 , -0.0076261 , 0.39377195, ..., 0.09380915, -0.01814741, 0.2283536 ], [-0.2979896 , 0.02483123, 0.03855339, ..., 0.09292422, 0.0840548 , 0.02128509], [-0.11292487, -0.09941795, -0.21962844, ..., -0.00123785, 0.14886236, 0.08236729]], [[ 0.32237047, -0.10110885, 0.5543121 , ..., 0.20536377, -0.33312383, 0.189913 ], [-0.23798265, -0.02816244, 0.15939426, ..., 0.1943017 , -0.09701379, -0.03432931], [ 0.10851432, -0.19066858, 0.04841118, ..., -0.01213947, -0.04722768, 0.09524001]], [[ 0.2550473 , -0.04364953, -0.47635067, ..., 0.23234586, -0.365624 , -0.01943434], [-0.2977616 , 0.00419303, -0.8656642 , ..., 0.30416554, -0.04954646, -0.1717201 ], [ 0.11049867, -0.17219873, -0.6994334 , ..., 0.08241771, 0.03823497, 0.0737647 ]], [[ 0.32358578, 0.17112552, 0.14320694, ..., -0.16366176, -0.2593694 , 0.07770303], [-0.1454706 , 0.12558867, -0.0796358 , ..., -0.1381298 , -0.02053097, 0.09441353], [ 0.05893492, -0.00259268, 0.02598703, ..., -0.20935844, 0.1816749 , 0.30294153]]], [[[ 0.14045075, -0.10724663, 0.1179781 , ..., 0.03702212, 0.30327222, 0.0930538 ], [-0.30190122, 0.11180247, -0.158114 , ..., 0.0602173 , 0.28998795, -0.29604238], [ 0.15810204, -0.02881167, -0.5261955 , ..., 0.16892877, -0.02681437, -0.18120374]], [[ 0.1801053 , -0.17351657, 0.55432343, ..., -0.04310204, 0.20605011, 0.15464272], [-0.35629642, 0.1292508 , 0.4196015 , ..., -0.00466895, 0.40400398, -0.22713828], [ 0.3233468 , -0.09626149, 0.11579885, ..., 0.07394816, -0.05250774, 0.03030762]], [[ 0.00362034, -0.27520794, -0.2702903 , ..., -0.0462058 , 0.11103056, -0.17394325], [-0.5339498 , 0.0243352 , -0.31235537, ..., 0.11845686, 0.4861063 , -0.42657182], [ 0.18858455, -0.18266024, -0.27887428, ..., 0.160679 , 0.1009312 , -0.00895783]], [[ 0.23181222, -0.16900635, 0.17172067, ..., -0.2783022 , 0.0531309 , -0.14735658], [-0.17012398, 0.0160596 , 0.26340246, ..., -0.15997212, 0.34474295, -0.18495837], [ 0.25120145, -0.10030471, 0.4254995 , ..., -0.14183548, 0.27963412, 0.15028709]]], [[[ 0.04808923, -0.0793573 , -0.05729656, ..., -0.06481453, -0.08433866, 0.25621825], [-0.21680164, 0.12764114, -0.13568583, ..., -0.07495075, -0.22111697, -0.15526403], [ 0.3291076 , -0.05329537, -0.51923084, ..., 0.05456135, -0.5494265 , -0.08629624]], [[ 0.00642845, -0.02851976, 0.48139414, ..., -0.3939782 , -0.02460362, 0.4114907 ], [-0.33011124, 0.21700603, 0.6363917 , ..., -0.34058657, 0.07507597, -0.00235403], [ 0.4467702 , -0.08816491, 0.338514 , ..., -0.20894945, -0.44596782, 0.22598238]], [[-0.12848432, -0.17561927, -0.29103017, ..., -0.3294456 , 0.0409772 , 0.05005151], [-0.47656342, 0.09599458, 0.01909353, ..., -0.15514864, 0.3270731 , -0.2028388 ], [ 0.28001264, -0.17020059, 0.09817386, ..., -0.0626152 , -0.07981837, 0.1589364 ]], [[ 0.18718387, -0.22170076, -0.12480265, ..., -0.2592998 , -0.3301834 , 0.06356978], [-0.05257114, -0.01859191, 0.32042652, ..., -0.23058891, -0.06912984, 0.0211558 ], [ 0.38780808, -0.15261608, 0.6172939 , ..., -0.2062019 , -0.08611296, 0.31126183]]], [[[-0.17952503, -0.05265503, -0.25038588, ..., 0.2504428 , -0.02090847, 0.28380206], [-0.2667295 , 0.07449401, -0.06040205, ..., 0.07079565, -0.10550356, 0.09134477], [ 0.09433349, -0.00351469, -0.22650217, ..., 0.02396235, -0.11385609, -0.00377663]], [[-0.0575775 , 0.03152074, -0.16277562, ..., 0.3681371 , 0.01227211, 0.08380491], [-0.17844254, 0.10643482, 0.21974468, ..., 0.26805398, 0.11603563, -0.11773686], [ 0.38371345, -0.10176778, 0.20368902, ..., 0.1509468 , -0.02080609, -0.06920536]], [[-0.18148401, -0.06984355, -0.85413545, ..., 0.47789526, 0.04577804, -0.23575552], [-0.3161512 , 0.02919929, -0.4008322 , ..., 0.44068208, 0.26976907, -0.33418196], [ 0.20420933, -0.1428987 , -0.11891793, ..., 0.30973166, 0.22936389, -0.15018184]], [[-0.08356225, -0.13858244, -0.32757443, ..., 0.142595 , -0.351965 , -0.0224065 ], [-0.17430148, -0.03812229, 0.26883033, ..., -0.06516638, -0.17817669, 0.04958347], [ 0.09899019, -0.05975993, 0.6693031 , ..., -0.12235346, 0.04438766, 0.1941795 ]]]], dtype=float32), }, }, position_embeddings: Array([[[ 0.033918 , -0.0929377 , -0.10530281, ..., 0.28548247, 0.08699364, 0.3533416 ], [-1.1857578 , -0.05044684, 0.04907949, ..., 0.22999725, 0.11194737, -0.5982713 ], [-0.8083621 , -0.45285803, -0.35289305, ..., 0.1278877 , -0.05470289, -0.3990702 ], ..., [ 0.01943148, -0.21790016, 0.20034477, ..., 0.13447031, -0.12896743, 0.67419606], [-0.5797325 , 0.09393977, -0.10601121, ..., 0.5656874 , -0.4588539 , 0.3854702 ], [-0.5565649 , 0.41388488, 0.21124075, ..., 0.8872757 , -0.52099276, -0.30215293]]], dtype=float32), }, encoder: { layer: { 0: { attention: { attention: { key: { bias: Array([-0.03415372, 0.45620176, -0.6126655 , -0.51253253, 0.4352034 , 0.649004 , 0.68406653, -0.46161497, -0.42964068, 0.8848562 , 0.61135024, -0.67950135, -0.08493523, -0.29988992, 0.17360933, 0.6337579 , 0.08232927, -0.59048873, -0.31515497, 0.09602205, 0.14608672, -0.06844742, 0.5664715 , -0.6412251 , 0.44181392, -0.40813568, 0.6399837 , 0.41530073, -0.66080755, -0.01420364, -0.19360113, 0.14690916, -0.9573473 , -0.3561616 , -0.18794109, 0.5325726 , -0.5846602 , -0.73184437, -0.35172847, -0.27912554, 0.33543518, -0.21184462, 0.35979107, -0.12768705, 0.3412426 , 0.35638675, 0.02133832, -0.08022975, -0.15245748, -0.20536591, -0.11586879, -1.1425315 , -0.02779106, 0.32926312, -0.26967174, 0.1869237 , -0.04547051, -1.0505885 , 0.9415589 , -0.5067646 , -0.4434777 , 0.31517163, -0.48155847, 0.53517747, -0.40972438, 0.16880448, 0.17367116, 0.29588723, 0.05554008, -0.48776305, -0.31501368, -0.10502674, 0.13945673, 0.55460685, 0.51986885, -0.39894032, -0.2494641 , 0.05012343, -0.04834368, 0.29485628, 0.19806208, 0.3374114 , 0.11331975, 0.36502162, -0.01847783, -0.39011198, 0.3790581 , 0.02779981, 0.651827 , 0.46813282, 0.29877353, -0.43409863, 0.15295495, -0.35798907, -0.43806 , -0.26218414, -0.45440933, 0.11940431, -0.4274145 , 0.18455578, 0.00287167, 0.33210322, 0.32568368, -0.33072832, 0.08617178, -0.24881381, -0.07380535, 0.23978767, -0.12747152, -0.7031561 , 0.10753731, 0.27640837, 0.0658379 , -0.00729593, 0.15286936, 0.14529614, 0.20602514, -0.18859431, 0.08264172, 0.09762905, -0.1777643 , 0.21521233, 0.68952733, 0.35572144, -0.25402895, -0.12086129, 0.33329782, -0.27421075], dtype=float32), kernel: Array([[-0.13037314, -0.3519865 , -0.7311013 , ..., -0.14157563, -0.098704 , 0.08183759], [ 0.36583012, 0.46603498, 0.07223602, ..., 0.3872796 , -0.01522054, 0.1815025 ], [ 0.4379942 , -0.2856487 , -0.14937577, ..., 0.2555367 , 0.143062 , -0.10283173], ..., [-0.08320192, -0.2315665 , 0.183584 , ..., -0.08580804, -0.13162506, 0.07868645], [ 0.42390886, -0.22044583, 0.4916073 , ..., 0.57441944, -0.04296773, 0.02346938], [ 0.09392057, -0.64890575, -0.21604928, ..., 0.09232247, 0.13443615, 0.11167435]], dtype=float32), }, query: { bias: Array([-0.06905509, -0.44198722, 0.30581513, 0.14314957, -0.15857255, 0.15910107, -0.14257118, 0.11895534, -0.13231936, -0.28292307, -0.1729494 , 0.1297792 , 0.29567292, 0.1645278 , -0.07054784, -0.1081412 , 0.2331662 , -0.02096415, 0.00768806, -0.13316594, -0.25273684, -0.09793813, 0.04979343, 0.23559354, -0.19012289, 0.08803113, -0.03696692, -0.29770622, 0.39657098, 0.1723672 , 0.12946679, 0.12689376, 0.3535053 , -0.02396139, 0.01831555, -0.45482886, 0.0035749 , 0.0489781 , -0.19142908, 0.04616706, -0.10055473, 0.00193944, -0.0123636 , -0.1297169 , 0.06566181, -0.12390063, 0.01915232, -0.10427503, -0.26933163, 0.01553594, -0.02893843, 0.49531803, 0.30729276, 0.01837071, 0.172419 , -0.05943993, -0.29665995, 0.46334326, -0.13110274, 0.49487928, 0.21740474, -0.25742924, 0.03184005, -0.16928338, 0.04680282, -0.10523521, 0.07657482, 0.147745 , -0.09269993, 0.01065696, 0.05368982, -0.18941729, 0.10891443, 0.20821781, 0.15085766, 0.51667964, -0.15096618, 0.34021842, -0.03054576, 0.26622248, -0.36788234, 0.29284194, -0.18616207, 0.09047277, 0.3589835 , -0.09647398, -0.10208427, 0.09295312, 0.03126433, -0.27378327, 0.08303533, -0.17912042, -0.06608632, -0.05499709, -0.09980894, 0.15720649, 0.37134722, 0.39746282, 0.172588 , 0.04140972, 0.58430547, -0.250469 , -0.3090034 , 0.35163316, -0.09326122, 0.18299101, -0.02789885, -0.16457352, 0.21470276, 0.20549582, -0.27671963, -0.4622744 , 0.19656613, 0.31474093, -0.0242331 , -0.01637263, 0.08560522, 0.40495184, -0.14052592, -0.24214344, -0.27252468, 0.15204628, -0.19363014, -0.13963734, -0.02372985, -0.13916686, 0.21864863, 0.13198575], dtype=float32), kernel: Array([[-0.02407378, -0.08052919, -0.78489935, ..., 0.0214601 , -0.21603775, 0.07814599], [ 0.1441562 , 0.39637634, -0.01425371, ..., 0.44336745, 0.02438617, 0.04486706], [ 0.54582417, -0.38129556, -0.09409017, ..., 0.07300002, 0.08039478, -0.01730129], ..., [-0.40244198, 0.23577765, -0.09792052, ..., 0.06759664, 0.0144011 , -0.21112448], [ 0.52981406, 0.07420167, 0.320484 , ..., 0.3295426 , -0.08885895, -0.11461516], [ 0.2322218 , -0.26398233, -0.4191451 , ..., 0.24738975, -0.05001807, 0.01805569]], dtype=float32), }, value: { bias: Array([-2.94787940e-02, -2.72796545e-02, 6.14140369e-02, -2.86302455e-02, 7.17062429e-02, -3.11819911e-02, -4.36154753e-02, 3.26400585e-02, -2.82624215e-02, 1.26899080e-02, -7.74416178e-02, 4.88068024e-03, 2.13063625e-03, -7.00014010e-02, 6.05225153e-02, -4.30300795e-02, 1.79692768e-02, -2.71751042e-02, 3.53089422e-02, -1.95429940e-02, -1.21456282e-02, -2.05201493e-03, -3.62878200e-05, -1.08139282e-02, 7.40849450e-02, 1.50353080e-02, -7.06956685e-02, 8.49624351e-02, -7.26162344e-02, -5.55856004e-02, -7.41069913e-02, 7.11530168e-03, -2.33412739e-02, 6.24122694e-02, 1.00617625e-01, 4.78886962e-02, 6.93868250e-02, 8.44263211e-02, 6.89995065e-02, -1.49511993e-02, 6.76077977e-02, 2.99593173e-02, -6.50391206e-02, -3.10420450e-02, -4.87265997e-02, -7.07542598e-02, 3.83699648e-02, -1.07949436e-01, 3.88861708e-02, 1.07101694e-01, -7.05710873e-02, 6.53532194e-03, 5.76992594e-02, 8.30659941e-02, -1.04475126e-01, 8.40004310e-02, -9.89107862e-02, 8.94425064e-02, -3.12575921e-02, -5.92782833e-02, -8.53339434e-02, -5.87267503e-02, 3.41176279e-02, -6.32282570e-02, -4.08555977e-02, -7.24948272e-02, 4.16365713e-02, -2.20883206e-01, -4.90566827e-02, -1.89475138e-02, -1.50290072e-01, 2.09986553e-01, 3.20502929e-02, 8.28777160e-03, 6.13699071e-02, -2.50448077e-03, -2.15128988e-01, -3.07804886e-02, -1.82573318e-01, 9.74187329e-02, 8.86107534e-02, -6.78683072e-02, -6.89225420e-02, -3.00914403e-02, -1.05735838e-01, -2.17407808e-01, 1.63544074e-01, 1.00356489e-01, -6.03550719e-03, 3.53857167e-02, -7.28843063e-02, 1.15036063e-01, -1.29502550e-01, 5.06461645e-03, 1.20808132e-01, 1.27421441e-02, -3.18243355e-02, -5.57740144e-02, -6.90745190e-02, -1.28188869e-02, 1.20238643e-02, -4.42795120e-02, -2.79449914e-02, -3.33387293e-02, 7.23095536e-02, -2.58487016e-02, -1.62459761e-02, 1.07147522e-01, -6.11629374e-02, 7.07705468e-02, -5.56649491e-02, 3.85964476e-02, 9.34836396e-04, -1.59531720e-02, -7.20297545e-02, 8.05353299e-02, 1.81769580e-02, -3.95291820e-02, -7.38437474e-02, -9.54268277e-02, 1.34773731e-01, 9.29158255e-02, -5.70693836e-02, 2.48423647e-02, -8.44348520e-02, -7.13706389e-02, -7.08770528e-02, 7.93647394e-02], dtype=float32), kernel: Array([[-0.19559653, 0.10921917, -0.24855891, ..., -0.3095954 , -0.11501747, 0.3495971 ], [ 0.2759333 , -0.0968355 , 0.25864232, ..., -0.07446828, -0.01653024, 0.04277551], [-0.53325766, -0.17772451, -0.19101648, ..., -0.09837775, 0.07115 , 0.39424804], ..., [ 0.36891153, -0.11530672, 0.24386573, ..., 0.05493925, -0.2263004 , -0.10157143], [ 0.06181217, 0.21771231, -0.52629954, ..., -0.0283559 , -0.11670491, -0.19776788], [-0.20297265, -0.24884045, 0.21022551, ..., 0.13422379, 0.22147001, 0.18063086]], dtype=float32), }, }, output: { dense: { bias: Array([-0.08976492, 0.35623267, -0.28853896, 0.1584903 , -0.04931528, 0.41930285, -0.12198956, -0.1676379 , -0.07629202, 0.44505563, -0.13279814, -0.2606873 , -0.38957864, 0.2600788 , -0.15301487, -0.7227959 , 0.06558075, -0.02327237, -0.01965788, 0.3900318 , 0.14463983, 0.33909434, -0.02560525, 0.25102413, 0.26336563, 0.07151037, -0.01791747, -0.32692116, -0.29931375, 0.26871207, -0.45228213, -0.07384168, -0.2372598 , 0.16236512, 0.20462725, -0.04373658, 0.18107706, 0.5327047 , 0.11542407, -0.20473395, -0.29505363, -0.1296326 , -0.50376815, 0.23728877, 0.16542588, 0.20167176, 0.05926127, 0.1154462 , -0.14579555, -0.08487356, 0.07978057, -0.27115968, -0.5287936 , 0.07883674, -0.07367402, -0.04930678, -0.5660592 , 0.14252122, -0.05342535, -0.03159878, 0.18108739, -0.06234799, 0.13124788, 0.00128543, -0.24720064, 0.13804607, -0.5026876 , 0.53043497, 0.07013435, 0.22524473, 0.17431633, 0.22486666, 0.01429994, -0.02526889, 0.65786517, -0.12362218, -0.124548 , 0.02234903, -0.16374543, -0.03648356, -0.79906535, -0.21115243, -0.20314318, 0.24117418, -0.42399853, 0.37275848, 0.11345251, -0.07611627, 0.09416302, -0.16610427, 0.22861136, -0.22463125, 0.21705595, 0.09253941, -0.2043051 , -0.06828043, -0.27955458, -0.00722787, -0.13780643, -0.44847608, 0.18490234, 0.09589677, 0.2274974 , -0.04555204, -0.13882257, -0.17222178, -0.24243575, 0.09366386, 0.16359329, 0.15961671, 0.08576287, 0.09707799, 0.31846833, -0.27176753, 0.12482687, -0.35437414, -0.04145558, -0.4648349 , 0.1269856 , -0.09939937, 0.00209725, 0.00080232, 0.20968738, -0.18917999, -0.12978964, 0.13607804, 0.00752292, -0.00256601], dtype=float32), kernel: Array([[ 0.01726927, 0.12866902, -0.1439863 , ..., -0.26570368, -0.0510007 , 0.47683123], [ 0.15968257, -0.1014777 , -0.26105717, ..., 0.30918548, 0.48335168, 0.01941068], [-0.05697384, 0.05050297, -0.3797851 , ..., -0.70562077, 0.11496508, 0.02426186], ..., [-0.37025422, -0.41289684, -0.09372032, ..., 0.3905355 , -0.44608384, -0.10514601], [ 0.03965069, -0.21772908, -0.06686966, ..., 0.5678079 , -0.5982024 , 0.03439448], [ 0.31467485, 0.03231593, 0.6761289 , ..., 0.3369738 , -0.16577064, 0.09921929]], dtype=float32), }, }, }, layernorm_after: { bias: Array([ 7.65252188e-02, 6.29107952e-01, 8.02198052e-02, 8.70596059e-03, 4.38350849e-02, -1.07495952e-02, -5.31466417e-02, -1.02623366e-01, 2.12361347e-02, 6.80099875e-02, -2.28481810e-03, 8.53950828e-02, -5.73644787e-02, 2.06265952e-02, -7.53715783e-02, -1.43298656e-01, -2.74074376e-02, -1.09807447e-01, -6.17579147e-02, -7.90846869e-02, -9.07771438e-02, 8.38840082e-02, -1.48849720e-02, -2.24489588e-02, -9.19756591e-02, -1.33614585e-01, 8.40165392e-02, -2.04390988e-01, 7.47799221e-03, 1.38857782e-01, -5.69530539e-02, 9.17964336e-03, -1.55208111e-01, 2.73925383e-02, -1.95245608e-03, 5.41600175e-02, -2.93773953e-02, 3.53689879e-01, -1.23494409e-01, -1.56066343e-01, 8.71012285e-02, 2.55195685e-02, -3.10201854e-01, -8.39867163e-03, -3.08605190e-02, -5.28353490e-02, -1.40786827e-01, -5.47004975e-02, -1.13185709e-02, 1.86418276e-02, -7.84755684e-03, -4.15783050e-03, -7.23032653e-02, -1.61183357e-01, 1.90100819e-02, -4.55883006e-03, -2.70947274e-02, 1.69784650e-02, -2.28463719e-03, 4.21662033e-02, -9.98298172e-03, 7.62146562e-02, -1.09199071e-02, 7.20275193e-02, 1.14038222e-01, 2.85849470e-04, -2.99424119e-02, 1.37919337e-01, -1.45379841e-01, 6.62748562e-03, 2.23827194e-02, -5.66348387e-03, 5.01772054e-02, 1.41351158e-03, 5.96981421e-02, -2.08293200e-01, -5.95040731e-02, 5.53276669e-03, -8.04201216e-02, -1.67969614e-01, 3.61396512e-03, 5.54506434e-03, -2.92952433e-02, 4.65981402e-02, 1.59924058e-03, -2.43443865e-02, -1.17824182e-01, -3.83222140e-02, -3.87748890e-02, -6.59047961e-02, -8.08075536e-04, -3.35978210e-01, -4.24195863e-02, 3.55881304e-01, 3.02827614e-03, -5.55533350e-01, 1.54737878e-04, -1.47938222e-01, -2.13265121e-02, -1.14879206e-01, -3.36717785e-04, 5.57862930e-02, 1.95077602e-02, -8.35356787e-02, 2.89721284e-02, -1.48645073e-01, -2.94460828e-04, 8.76041055e-02, -1.90158889e-01, 1.32130921e-01, 1.11547068e-01, -8.59021768e-02, -4.04770300e-03, -2.04068184e-01, 2.25875482e-01, -7.98009858e-02, 1.03451028e-01, -6.95376145e-03, -8.43686834e-02, -1.78507604e-02, -9.83250737e-02, -7.10767433e-02, -6.19554799e-03, -5.51361814e-02, -8.06146394e-03, 4.24989313e-01, -2.02743262e-02, -7.10928217e-02], dtype=float32), scale: Array([ 4.92835701e-01, 7.25533962e-01, 1.97780564e-01, 7.19367981e-01, 3.85366410e-01, 1.70830980e-01, 4.03196029e-02, 3.59184593e-01, 1.64954796e-01, 5.99162340e-01, 1.94611892e-01, 4.47059900e-01, 1.33249581e-01, 1.17341705e-01, 2.57417858e-01, 2.13419840e-01, 1.17401130e-01, 1.44011736e-01, 4.55426246e-01, 3.37985694e-01, 3.68331194e-01, 1.58645034e-01, 4.11303312e-01, 2.80974478e-01, 2.08782077e-01, 3.16983044e-01, 2.54319280e-01, 5.07932603e-01, 6.26261353e-01, 3.31194192e-01, 5.77102304e-01, 5.91307692e-03, 2.17876673e-01, 1.83340341e-01, 7.28062959e-03, 1.43880785e-01, 3.79934996e-01, 8.64668906e-01, 5.03845096e-01, 3.30553919e-01, 3.66227090e-01, 1.23913087e-01, 4.04930204e-01, -5.25120134e-03, 2.16952696e-01, 2.82419056e-01, 2.41874039e-01, 2.18594924e-01, 2.72880584e-01, 1.63890943e-01, 5.69228344e-02, 1.57258492e-02, 3.11221033e-01, 3.75806093e-01, 3.45861703e-01, 3.33030730e-01, 1.86815023e-01, 2.87897736e-01, 1.79882497e-02, 3.26225400e-01, 3.56188752e-02, 3.17295760e-01, 5.93220115e-01, 3.10718954e-01, 1.92407712e-01, 9.49200336e-03, 2.15880141e-01, 4.15735185e-01, 3.10675055e-01, 5.28817117e-01, 1.04173638e-01, 1.57576382e-01, 3.02691996e-01, 3.79474498e-02, 7.07599940e-03, 5.08240998e-01, 3.50080490e-01, 3.61189306e-01, 1.87464565e-01, 4.86818105e-01, 3.34662758e-03, -1.36662647e-03, 1.20725386e-01, 5.04414022e-01, 3.98086966e-04, 9.34311152e-02, 1.42500699e-01, 3.84391755e-01, 1.07679166e-01, 3.01436037e-01, -1.05553423e-03, 4.41559106e-01, 1.26928464e-01, 4.32251424e-01, -6.94631459e-03, 5.02396822e-01, -3.20078339e-04, 1.94142312e-01, 8.77228081e-02, 1.18769050e-01, 4.47774172e-01, 3.87618184e-01, 4.12855119e-01, 3.15822661e-01, 4.97932345e-01, 6.33932471e-01, 6.15001982e-03, 2.09667221e-01, 2.26229250e-01, 1.05634868e+00, 5.23372293e-01, 4.29101616e-01, 2.27578685e-01, 6.58651769e-01, 2.97994912e-01, 1.07822992e-01, 4.83628422e-01, 4.72095191e-01, 3.23185503e-01, 1.00680351e-01, 4.04555202e-01, 1.64712802e-01, 6.41667992e-02, 1.81472510e-01, 5.06515145e-01, 4.70199496e-01, 2.70896405e-01, 4.36434001e-01], dtype=float32), }, layernorm_before: { bias: Array([-0.05890432, 0.07051883, 0.09445145, -0.07912993, -0.14302658, -0.01429262, 0.03426607, 0.00171352, 0.11276561, -0.05662137, 0.13313785, -0.04794319, 0.00357568, 0.01197018, -0.07405796, -0.06490814, -0.0245945 , 0.14022568, -0.05081064, -0.12518148, 0.00789185, -0.24005446, 0.02322139, -0.17626956, -0.03338344, -0.33740795, -0.13774505, -0.07997898, 0.04349966, -0.01520896, -0.39623648, -0.150468 , 0.11885046, -0.01181321, 0.00808169, -0.20219389, 0.0043777 , -0.08942726, -0.16421181, -0.18974544, -0.07742172, -0.01755824, -0.02627599, -0.4378331 , -0.19054742, 0.2720697 , 0.25448796, 0.01773962, -0.01873611, 0.0169231 , 0.03406947, 0.0349858 , 0.09072868, -0.05627641, 0.14739008, 0.01965971, 0.09791738, 0.03601414, -0.28183195, 0.15014632, 0.22542529, 0.3072184 , -0.09403451, -0.08351854, 0.25167593, 0.04495538, -0.11564947, -0.00397131, 0.10662062, 0.05204451, 0.01583207, 0.01414629, 0.23943776, 0.06893958, -0.08743864, -0.06693614, -0.07541402, -0.19011301, -0.17251883, 0.01803088, -0.09193094, 0.04894969, -0.00502513, -0.09108073, -0.1768744 , -0.01100582, 0.10178719, 0.22854435, 0.04437195, 0.00859537, -0.03013818, 0.26522186, -0.0835686 , 0.0909194 , 0.08486014, -0.39586094, 0.03491055, 0.02060359, -0.14571564, -0.18893251, 0.04928007, 0.05034845, -0.15555385, -0.15786135, -0.56710386, 0.01834173, -0.04088113, -0.0075858 , 0.04828192, -0.1228609 , -0.00563954, -0.3955512 , -0.06544469, 0.09136705, -0.10234332, 0.09786685, 0.00981475, -0.05797027, 0.08497368, 0.18863663, -0.00070434, 0.03551389, -0.04786862, -0.03968854, -0.02606042, -0.2953443 , -0.03344315, -0.2232286 ], dtype=float32), scale: Array([0.45335642, 0.28725365, 0.25888288, 0.51014036, 0.25393826, 0.14649442, 0.45231742, 0.07753127, 0.3569898 , 0.67538893, 0.2085993 , 0.17448373, 0.01005436, 0.07782084, 0.26946452, 0.14676209, 0.22900493, 0.29659525, 0.13212137, 0.1925922 , 0.04533805, 0.26069018, 0.15647988, 0.20724292, 0.08556735, 0.296176 , 0.20237958, 0.6578139 , 0.11472861, 0.3509395 , 0.38030067, 0.19699657, 0.30957213, 0.1079342 , 0.12253352, 0.54774934, 0.06868377, 0.49437425, 0.34611228, 0.33752495, 0.24485146, 0.11372902, 0.13047746, 0.40855378, 0.49101192, 0.5738504 , 0.32902393, 0.2079778 , 0.09577054, 0.11233962, 0.17488489, 0.10529607, 0.13811632, 0.19155163, 0.16904642, 0.10060844, 0.2304541 , 0.20629783, 0.21513213, 0.1885996 , 0.2502167 , 0.3126915 , 0.3413607 , 0.1449984 , 0.41228867, 0.09932974, 0.14989796, 0.13950281, 0.24986736, 0.15920106, 0.06112578, 0.10076807, 0.25724828, 0.11337321, 0.1927693 , 0.24177718, 0.14134662, 0.4839469 , 0.20330684, 0.1124863 , 0.17910473, 0.1306797 , 0.09654108, 0.13909315, 0.19018278, 0.12305983, 0.3541507 , 0.31824607, 0.13942654, 0.08698463, 0.08841392, 0.17479059, 0.15011418, 0.45322958, 0.17411204, 0.29569057, 0.10512718, 0.3171436 , 0.21283199, 0.51158214, 0.42643854, 0.18322334, 0.3302157 , 0.37901062, 0.44037428, 0.50349325, 0.1318383 , 0.13000993, 0.14093749, 0.5986485 , 0.1306848 , 0.36673996, 0.2426187 , 0.19750752, 0.14503735, 0.22176953, 0.1925638 , 0.15475766, 0.17507745, 0.22627683, 0.10930963, 0.37122646, 0.12475897, 0.12880948, 0.11971601, 0.2903781 , 0.1446715 , 0.20750394], dtype=float32), }, moe: { shared_experts: { intermediate: { bias: Array([-0.4722571 , -0.43928552, -0.5569621 , -0.3837599 , -0.47573015, -0.4316669 , -0.762032 , -0.5649565 , -0.31526604, -0.42409667, -0.35039356, -0.5466656 , -0.08653939, -0.50116634, -0.65188706, -0.47724712, -0.29790252, 0.32602024, -0.04504937, -0.40849677, -0.11292349, -0.25168103, -0.45442316, -0.34808317, -0.38236308, -0.3788954 , -0.19183853, -0.30315536, -0.16824003, -0.4485127 , -0.30100617, -0.39639288, -0.16540907, -0.5450078 , -0.4645357 , -0.36592278, -0.5585429 , -0.34257036, -0.19063933, -0.44469953, -0.42737913, -0.36188826, -0.2907807 , -0.02146477, -0.23154494, -0.20253362, -0.69757545, -0.39887667, -0.44025323, -0.39642838, -0.387052 , -0.24472845, -0.09177764, 0.05788963, -0.04718702, -0.21287683, -0.29563203, -0.3253591 , -0.43122077, -0.1849469 , -0.5141688 , -0.29721844, -0.47572163, -0.3446094 , -0.26447332, -0.47591433, -0.19857107, -0.6629566 , -0.36350504, -0.60418856, -0.4550368 , -0.40432703, -0.35896167, -0.33915696, -0.41339654, -0.23302272, -0.05349919, -0.47071078, -0.32239798, -0.3542714 , -0.25527537, -0.3346329 , -0.49991933, -0.22825265, -0.45629248, 0.01691399, -0.34854415, -0.44188562, -0.03925477, -0.3642511 , -0.40468022, -0.33773935, -0.49415597, -0.5592706 , -0.2977098 , -0.32126102, -0.16516496, -0.40793797, -0.8022191 , -0.44280693, -0.1008477 , -0.22637095, -0.16032524, -0.45757398, -0.4884214 , -0.48628846, -0.4652781 , -0.51417524, -0.25536695, -0.33155087, -0.5587205 , -0.37767413, -0.30986476, -0.47982594, -0.51307243, -0.38967907, -0.24242653, -0.25831246, -0.1458375 , -0.58483136, -0.3381596 , 0.27927315, -0.7453308 , -0.4533877 , -0.15844217, -0.45824784, -0.468726 , -0.4619875 , -0.4371935 , 0.00825728, -0.31616744, -0.4558102 , 0.0993996 , -0.08013049, -0.17536967, -0.52361816, -0.44187388, -0.25537458, -0.19660369, -0.30101493, -0.09137805, -0.41055298, -0.46738333, -0.57409215, -0.34877914, -0.52286893, -0.55365384, -0.33294562, -0.47346646, -0.3105509 , -0.44117743, -0.3502154 , -0.39879954, -0.3967916 , -0.47747064, -0.24431925, -0.39359137, -0.22136432, -0.35870054, -0.29925522, -0.34952432, -0.49339256, -0.42442355, -0.383239 , -0.3228079 , -0.59250957, -0.3766234 , -0.48536432, -0.49891275, -0.10365842, -0.21082799, -0.5129332 , -0.39634183, -0.4714169 , -0.3876767 , -0.2642752 , -0.1780093 , -0.08995035, -0.5045371 , -0.31991875, -0.3657458 , -0.51370513, -0.5282006 , -0.43932983, -0.44844607, -0.4636747 , -0.541842 , -0.28016394, -0.5640383 , -0.28513128, -0.44549647, -0.43519762, -0.42721725, -0.57123166, -0.40650263, -0.12434424, -0.2684522 , -0.13034356, -0.41602165, -0.2656361 , -0.63166195, -0.24252081, -0.49084774, -0.66598415, -0.21980134, -0.20069678, -0.3857888 , -0.5750186 , -0.28702593, -0.42044863, -0.43332833, -0.05599573, -0.17978549, -0.33969852, -0.42026928, -0.19741155, -0.2731103 , -0.5647213 , -0.61824954, -0.44372314, -0.16089505, -0.6141717 , -0.27717924, -0.39058053, -0.30320698, -0.3017265 , -0.36256546, -0.29453766, -0.7215134 , -0.36851448, -0.45460737, -0.25023326, -0.24756469, 0.5198105 , -0.49648827, -0.7883031 , -0.40549907, -0.44536465, -0.34158024, -0.4282217 , -0.45166638, -0.285461 , -0.5351079 , -0.5246095 , -0.38692436, -0.4037612 , -0.68792176, 0.09384751, -0.3728323 , -0.54188645, -0.14435874, -0.13926642, -0.48824954, 0.103251 , -0.5347266 , -0.12768686, -0.2596208 , -0.33724293, 0.41711676, -0.46974605, -0.31142974, -0.46029773, -0.38704032, -0.4837437 , -0.4762127 , -0.48931426, -0.27646464, -0.7709319 , -0.11419559, -0.3580196 , -0.41691482, -0.523539 , -0.7911689 , -0.14548191, -0.32703182, -0.22812605, -0.5490665 , -0.27162734, -0.47042787, -0.27687344, -0.29244688, -0.14437722, -0.42082107, -0.46603575, -0.3757466 , -0.43942484, -0.35704526, -0.50712955, -0.06310245, -0.51886773, -0.36505163, -0.29938784, -0.47636163, -0.798845 , -0.39453545, -0.42888972, -0.34777957, -0.42616373, -0.37013018, -0.2707561 , -0.22923009, -0.4484783 , 0.0781149 , -0.54366326, -0.31296468, -0.46913856, -0.26365936, -0.05434812, -0.43664017, -0.262903 , -0.21987678, -0.29727325, -0.53905886, -0.2881995 , -0.2697021 , -0.56999284, -0.23401402, -0.25733256, -0.48550016, -0.53915155, -0.29186913, -0.41333848, -0.11731004, -0.1876421 , -0.38366327, -0.69185966, -0.54369533, -0.13784738, -0.45933396, -0.47800148, -0.42821413, -0.37634867, -0.42712605, -0.06954894, -0.445592 , -0.40655133, -0.3772172 , -0.7594351 , -0.5565112 , -0.25855637, -0.13048875, -0.3732955 , -0.30561906, -0.5940574 , -0.41309616, -0.314704 , -0.3010439 , -0.5837942 , 0.40323326, -0.24861278, -0.4630232 , -0.538137 , -0.37435848, -0.46322653, -0.48135036, -0.26899594, -0.17752594, -0.46429896, -0.62554 , -0.4089612 , -0.34856302, -0.07396294, -0.4918398 , -0.44181666, -0.21933144, -0.41612074, -0.46780694, -0.25013933, -0.3507257 , -0.31338322, -0.49207342, -0.573551 , -0.25965518, -0.27235296, -0.46362886, -0.572643 , -0.36132613, -0.5891415 , -0.43028024, -0.58089757, -0.5628406 , -0.23266539, -0.48305756, -0.4171379 , -0.30668163, -0.800837 , 0.16001745, -0.32928717, -0.7041144 , -0.28114614, -0.1339794 , -0.1539137 , -0.48452973, -0.2916893 , -0.31748015, -0.41399214, -0.5892567 , -0.6904999 , -0.39504105, -0.05394559, -0.4220559 , -0.08093858, -0.55065817, -0.42636526, -0.36694276, -0.6491974 , -0.16998939, -0.6064266 , -0.49446252, -0.25082645, -0.3862861 , -0.5665641 , -0.12244188, -0.59443516, -0.43923312, -0.03984782, -0.4539737 , -0.2804616 , -0.28886226, -0.41679534, -0.7231176 , -0.44628784, -0.17072082, -0.55711204, -0.33631635, -0.3513167 , -0.44696072, -0.09838888, -0.5331628 , -0.3244455 , -0.22034623, -0.39987078, -0.52567166, -0.18180668, -0.1563768 , -0.15176131, -0.34147656, -0.39736265, -0.22072102, -0.16304936, -0.4934699 , -0.35189024, -0.38949656, 0.13233016, -0.2961783 , -0.03038953, -0.19008778, -0.43870625, -0.14041252, -0.46124077, -0.6101991 , -0.78746367, -0.41358054, -0.0965796 , -0.17454897, -0.46205354, -0.5193903 , -0.43612164, -0.338933 , -0.2509465 , -0.25011972, -0.32999432, -0.5068436 , -0.40737084, -0.59392023, -0.08752135, -0.6887468 , -0.02706476, -0.34656492, -0.45208535, 0.12221402, -0.07045163, -0.6397726 , -0.32004473, -0.2506177 , -0.41133684, -0.20769219, -0.3077098 , -0.48824766, -0.59370387, -0.3106976 , -0.09882683, -0.23733851, -0.5397758 , -0.384874 , -0.48532894, -0.35305282, -0.44897798, -0.57191414, -0.23201816, -0.40165707, -0.45165244, -0.30947968, -0.57236695, -0.40093747, -0.610941 , -0.4310733 , 0.00399699, -0.17124976, -0.33956885, -0.59238917, -0.4834356 , -0.23509572, -0.3706481 , -0.46995562, -0.30272713, -0.42109653, -0.16173697, -0.3515516 , -0.46951428, -0.24467751, -0.37127313], dtype=float32), kernel: Array([[-0.16517928, -0.06662413, -0.32296306, ..., 0.17395326, -0.00140762, -0.21535204], [-0.2497043 , -0.32800463, -0.54542446, ..., -0.37079784, -0.4540639 , -0.2762963 ], [ 0.20276228, -0.2351077 , -0.4339836 , ..., -0.07039763, 0.09680251, -0.00943627], ..., [-0.0865024 , -0.07723805, -1.0022317 , ..., -0.22161727, 0.2631157 , -0.2520337 ], [-0.06232752, 0.41687462, 0.43454677, ..., 0.17987384, -0.22243185, 0.2732002 ], [-0.1515276 , 0.08726911, -0.16446725, ..., -0.1428107 , -0.12378144, -0.24571244]], dtype=float32), }, output: { bias: Array([-2.27567896e-01, 3.73193085e-01, 1.41862273e-01, 8.60335305e-02, 6.99756369e-02, -1.08260542e-01, -7.94294663e-03, 1.12044938e-01, -1.97965074e-02, 1.03512242e-01, 1.25780692e-02, 2.14173347e-02, -1.95132956e-01, -1.66688226e-02, -5.57698980e-02, 2.21790038e-02, 8.23988672e-03, -1.56303987e-01, 2.77717095e-02, -6.28630817e-02, -1.10570341e-01, 1.39788479e-01, -9.08712447e-02, -7.11519644e-03, -1.21463031e-01, 1.24595746e-01, 6.99105812e-03, 6.04419075e-02, 1.16523303e-01, -3.08515457e-03, 1.71046168e-01, -4.62258421e-02, 1.37897981e-02, -9.23820660e-02, 1.22136414e-01, -5.12558557e-02, -4.16860264e-03, 5.67508861e-02, -8.22613947e-04, -1.72597811e-01, -1.73500534e-02, -1.02644004e-01, -7.16797352e-01, 6.50356859e-02, -9.02397409e-02, -7.97297060e-02, -3.20093259e-02, 7.92354718e-02, -5.91707882e-03, -5.96906338e-03, 2.83249887e-03, 4.21253145e-02, -1.67518064e-01, -2.99427286e-02, -5.09011634e-02, -3.20116989e-02, -1.33028731e-01, 2.27317363e-02, 1.29082099e-01, 1.18409574e-01, 1.57172829e-02, -2.19231278e-01, -2.51264381e-03, -4.79261391e-02, 4.22019213e-02, -7.78303593e-02, -7.12049752e-02, -8.38479102e-02, -3.88463922e-02, -3.15606929e-02, -6.03214689e-02, 4.97607961e-02, -3.79606709e-02, 7.08331587e-04, 9.65211987e-01, 1.56220675e-01, -2.14344375e-02, -2.15863492e-02, -1.17833465e-01, 1.19182810e-01, -2.71626860e-01, 1.97731983e-03, 3.98320071e-02, 2.02119559e-01, 6.09681681e-02, -3.32502718e-03, 1.39364108e-01, 5.88153526e-02, -4.45023738e-02, -6.55459762e-02, 4.42181528e-02, -8.78035650e-02, -4.80993986e-02, -1.09087467e-01, -9.66937765e-02, -1.30174085e-01, 2.12008785e-03, 2.63354164e-02, -9.37493294e-02, 7.85378218e-02, 1.73281968e-01, -7.77669847e-02, 1.33560196e-01, -1.09741293e-01, 1.85674503e-01, 1.03994340e-01, -1.89410765e-02, -1.18796170e-01, -1.62400916e-01, 2.00676285e-02, -1.06416032e-01, 1.79987997e-02, 1.19247243e-01, -2.42678579e-02, 1.64977908e-01, -7.41590792e-03, -1.76154330e-01, 2.93482076e-02, -6.35338202e-02, 2.07892321e-02, 8.92439187e-02, 2.59270482e-02, 2.71555986e-02, -7.91539550e-02, 9.35807526e-02, 5.84603362e-02, 1.93760265e-02, 1.72683045e-01], dtype=float32), kernel: Array([[ 6.6307461e-01, -2.6014799e-01, -8.3183780e-02, ..., -2.6699215e-01, -1.0033758e-01, -2.1524315e-01], [-1.4059058e-01, -4.3626988e-01, -8.1478050e-03, ..., -3.1092042e-01, 3.1286913e-01, -3.7179798e-02], [ 7.0444113e-01, -1.0556934e-05, -2.0829076e-02, ..., -9.0309918e-01, 3.2471472e-01, 5.0782524e-02], ..., [-7.5270250e-02, 3.7796192e-02, -5.1213449e-01, ..., 3.4989411e-01, -1.0396782e-01, -5.7098109e-01], [ 9.8383605e-02, -1.5289782e-01, 3.3700535e-01, ..., 5.9063655e-01, -9.2052268e-03, -1.1524218e-01], [ 1.1272128e-01, -6.6285774e-02, -2.5734550e-01, ..., -1.2502365e-01, 9.1840588e-02, 2.3722002e-01]], dtype=float32), }, }, }, }, 1: { attention: { attention: { key: { bias: Array([ 0.08042084, 0.17798816, 0.05449522, 0.09923156, 0.02864883, -0.06706466, 0.12987332, 0.05871709, 0.05153077, 0.05436414, 0.05116336, -0.14068684, -0.11056656, -0.00991776, 0.00717054, 0.0770971 , 0.11563851, 0.02029003, 0.02723394, -0.12056217, -0.06265891, -0.05970899, 0.14154014, -0.03163632, 0.0481686 , 0.02711394, 0.04412205, 0.02307438, -0.05139432, 0.03842278, -0.07357287, -0.08965195, -0.08623472, 0.03718801, 0.84410733, 0.4109549 , 0.3907228 , -0.44974568, -0.3517945 , 0.03430397, -0.07853877, -0.30442524, 0.5081588 , 0.15563154, 0.17845473, 0.14964798, 0.01935043, 0.78861445, 0.13657925, 0.9571119 , -0.15077294, 0.04331086, -0.23139133, -0.09555604, 0.06358018, -0.59850585, -0.37260893, -0.9738547 , -0.01462804, 0.12405673, 0.07412421, -0.01379338, 0.05929387, 0.7805736 , -0.36394337, 0.47818112, -0.43866792, 0.63309664, 0.1147754 , 0.45001298, 0.3213029 , 0.44093522, 0.51016414, -0.18746474, -0.13791506, 0.04778931, -0.49275073, 0.04839996, -0.38444966, -0.4882343 , -0.6615728 , 0.09630081, -0.29930323, -0.34863642, 0.77798736, -0.55292934, -0.56388927, 0.09340773, -0.4621951 , 0.26494777, 0.31161246, 0.44366807, 0.6213311 , 0.01337955, 0.14917652, -0.4876238 , 0.06336308, -0.12776731, -0.07288839, -0.18272795, 0.23383974, -0.059283 , -0.08504926, -0.12375498, 0.3620917 , 0.04436552, 0.15867496, 0.12362888, -0.09505215, 0.18897517, 0.08316641, -0.18332016, -0.26461446, 0.17972536, 0.23397446, -0.04752706, -0.04319599, 0.12071403, 0.25849587, 0.06566656, 0.19910227, 0.22742352, -0.22645345, -0.14635295, -0.2507506 , -0.18631294, -0.05247986, 0.12347726], dtype=float32), kernel: Array([[-0.06023952, 0.297179 , -0.02699618, ..., -0.1145321 , -0.08048212, -0.10587901], [ 0.6934901 , 0.47458088, 0.61580837, ..., -0.13140583, -0.14733373, 0.26471645], [ 0.09990517, -0.17357504, 0.05461919, ..., -0.11821933, -0.24399522, -0.15253429], ..., [-0.2205276 , -0.23215702, -0.24366312, ..., -0.03175567, 0.22550093, 0.27891138], [-0.24111255, -0.27693352, -0.21988523, ..., 0.30499363, 0.37258562, -0.13957877], [-0.103215 , -0.14666593, -0.04492585, ..., -0.6473914 , -0.23865867, 0.19981082]], dtype=float32), }, query: { bias: Array([-0.11081476, -0.09144565, -0.06050301, -0.03483055, 0.03535065, -0.1849559 , -0.04721399, -0.07624152, -0.10357637, 0.22812082, -0.3447222 , 0.0331145 , 0.05764522, -0.13265188, 0.1981749 , -0.03139308, -0.07606025, 0.13599226, 0.02934942, 0.12725802, 0.07556148, 0.07594807, -0.04966472, -0.15265177, -0.35979733, 0.01576409, -0.07025045, -0.19974303, -0.22990641, 0.0952646 , -0.33106107, 0.04587565, -0.41658264, 0.33157024, -0.51854265, 0.34642565, -0.38225 , 0.21841964, 0.4542027 , -0.3708585 , -0.17867757, 0.38574484, -0.40515402, -0.18270488, 0.15764216, -0.09051255, -0.47678202, -0.5485276 , 0.17880292, -0.42515096, 0.11444188, 0.27703342, -0.23375264, -0.17498691, 0.13194664, 0.06895838, 0.32982063, 0.17495483, -0.16514482, 0.16266893, 0.24224894, 0.49473688, -0.7105683 , -0.22717172, -0.4628581 , -0.02517562, 0.13951837, 0.04581514, -0.02814059, 0.06052845, 0.89534795, -0.01086807, -0.2937633 , 0.4263962 , 0.01343843, 0.9783011 , 0.23017581, 0.7163039 , 0.41632223, 0.17262603, -0.15068051, 0.850449 , -0.16111681, -0.02732765, -0.18101065, 0.47148392, 0.18522678, -0.06222672, 0.15925601, -0.07961265, 0.71117395, 0.02433823, -0.35966122, -0.09352748, -0.06117439, -0.13183966, -0.08041007, -0.27098748, -0.71445733, -0.5658294 , 0.51616377, -0.56782836, -0.68260497, -0.5732899 , 0.5585706 , -0.14336808, 0.59159863, -0.21151459, 0.42952493, 0.46055183, 0.46279824, -0.4195606 , -0.5513547 , 0.34171253, 0.69069153, -0.25621027, -0.46899697, 0.588077 , 0.6255269 , 0.5819053 , 0.54691887, 0.59542644, -0.5749214 , -0.57993734, 0.42467687, -0.70540833, -0.34917867, 0.3106283 ], dtype=float32), kernel: Array([[-0.11731137, -0.1564569 , -0.11458346, ..., 0.02826559, -0.03335454, 0.14976554], [-0.04609706, -0.04719561, -0.02786452, ..., -0.30554634, -0.20783764, -0.11992735], [ 0.0489803 , 0.12078558, 0.0699265 , ..., 0.26866522, -0.40512353, -0.5030211 ], ..., [-0.00423968, -0.00703923, -0.03773196, ..., -0.10513086, -0.145968 , 0.07945077], [ 0.07510082, -0.13710845, 0.03422854, ..., 0.15533802, 0.8349194 , -0.10965192], [-0.07718561, 0.09582806, -0.06439573, ..., -0.12381844, -0.9082412 , 0.04665521]], dtype=float32), }, value: { bias: Array([-0.11375186, 0.01737056, -0.11132707, -0.01888963, 0.02005741, -0.00092973, -0.11860004, 0.14434907, -0.06270761, 0.14292052, 0.14930397, 0.06040736, 0.1552559 , -0.06054825, 0.13105005, 0.05230521, 0.03080181, -0.14906318, 0.04209505, -0.11033 , -0.02365249, -0.14162254, -0.00175851, 0.03008844, 0.04629046, 0.10597341, 0.02212737, 0.13431458, -0.03534039, 0.00220095, -0.05828169, 0.01887055, 0.07417873, -0.18334955, 0.05120145, 0.03194632, 0.09620124, 0.07511445, 0.14970838, -0.3099061 , -0.00785524, 0.21368372, -0.24593364, -0.20003676, 0.29439747, 0.02457003, -0.0362221 , -0.04669455, -0.11111037, -0.09324432, -0.007548 , -0.05471316, -0.1160496 , -0.08286496, 0.10508952, -0.29490575, 0.00155351, 0.13688038, 0.055782 , -0.241635 , 0.19978897, 0.12800941, -0.08591252, -0.17519936, 0.0762416 , 0.03798862, -0.04639675, -0.1959964 , -0.25277418, 0.169555 , -0.06163629, -0.11278319, -0.01159456, -0.06765992, 0.0005254 , -0.06396206, -0.00113279, 0.1249745 , -0.02429713, -0.01882967, -0.04579956, -0.02704 , 0.00707008, 0.02282375, -0.10874218, -0.01254413, 0.05641572, -0.06401654, 0.17181516, 0.0763936 , -0.07932342, -0.12056571, 0.0820564 , -0.01728764, 0.04256084, -0.05678854, 0.11785327, 0.2377933 , -0.19259618, -0.08974701, -0.19949402, -0.09832341, 0.12747605, 0.01983653, 0.00941677, 0.11338363, -0.20597233, -0.04107258, 0.02300076, -0.04743035, 0.11633945, 0.12718917, 0.23441505, 0.01317911, -0.19377421, -0.06671913, 0.04473834, 0.00467375, -0.15131958, -0.07110944, -0.16257668, -0.08920506, 0.03026637, -0.0935519 , 0.06253995, -0.19525133, -0.21808124, -0.19407861], dtype=float32), kernel: Array([[-0.01158661, 0.04303771, -0.05590956, ..., -0.05333043, -0.00572822, -0.01724763], [ 0.2719801 , 0.14965016, -0.02898863, ..., -0.06263707, -0.35550225, -0.28345698], [-0.05632893, -0.26717782, -0.07165924, ..., -0.43430653, 0.0922191 , 0.10500429], ..., [-0.04451123, -0.00324844, 0.15866533, ..., -0.15973136, -0.06277192, -0.10269681], [-0.28674033, 0.40732047, -0.23387024, ..., 0.5245572 , -0.20356956, -0.19302332], [-0.02055671, -0.35077375, 0.13905095, ..., 0.08351879, 0.4497927 , 0.44072792]], dtype=float32), }, }, output: { dense: { bias: Array([-2.29397908e-01, 3.42801064e-01, 1.12294301e-01, -1.40454136e-02, 3.03802863e-02, -1.64602742e-01, 6.72679991e-02, 9.77464989e-02, -2.96701267e-02, 7.89122283e-02, 5.59587255e-02, 1.48079200e-02, -1.85802028e-01, -3.27114910e-02, -2.40307394e-02, 6.82050884e-02, 5.04146097e-04, -1.56787038e-01, 6.75788894e-02, -1.31789781e-02, -4.55502272e-02, 1.27612054e-01, -7.01643378e-02, -2.48179883e-02, -1.31983072e-01, 1.38503700e-01, 1.17344763e-02, 7.56531805e-02, 6.57847598e-02, 2.69352719e-02, 2.21325293e-01, -5.05222380e-02, 3.61935459e-02, -1.04897447e-01, 5.24883904e-02, -8.26861486e-02, 1.44010643e-02, 8.25399533e-02, -6.71743648e-03, -1.94458306e-01, -2.49852352e-02, -8.23389366e-02, -6.81263685e-01, 8.11951086e-02, -8.49441960e-02, -1.40838427e-02, -1.44380983e-02, 6.43714517e-02, -1.57576092e-02, -3.63134891e-02, -1.25548737e-02, 1.41190467e-02, -1.39931515e-01, 1.46125898e-01, -4.10604011e-03, -3.28917056e-02, -1.47544041e-01, -7.30510205e-02, 1.31265923e-01, 3.08194682e-02, 5.00784516e-02, -2.18488395e-01, 2.87581724e-03, -3.31245400e-02, 2.70526409e-02, -1.38555646e-01, -1.16920797e-02, -7.85453171e-02, 1.44379027e-02, -8.10271129e-02, -7.89678767e-02, 6.87262276e-03, 2.95094848e-02, -2.96989549e-02, 8.52790475e-01, 1.68656245e-01, -4.95392680e-02, 1.01274252e-03, -1.17389508e-01, 1.42783284e-01, -2.95234203e-01, -3.66869681e-02, 2.10161004e-02, 1.61923394e-01, 1.00733116e-01, 9.56858695e-02, 1.62279665e-01, 6.97396919e-02, -6.76168874e-02, -1.21831730e-01, 2.96244565e-02, -8.43596309e-02, -4.85159941e-02, -1.54231772e-01, -8.46877992e-02, -9.01621133e-02, -7.94285350e-03, 9.12197307e-03, -5.20062111e-02, 2.70500332e-02, 1.75196305e-01, -7.51989707e-02, 1.59212708e-01, -8.15638751e-02, 1.15850195e-01, -1.62196823e-03, 6.47357618e-03, -1.16396412e-01, -1.10036433e-01, -1.06435921e-02, -1.69102713e-01, 2.04750933e-02, 1.00554541e-01, -3.42080109e-02, 1.30484760e-01, 1.22546069e-02, -1.68031573e-01, 3.20073776e-02, -6.57493770e-02, -7.96901435e-03, 1.29959419e-01, 8.22199881e-02, 4.45892625e-02, -4.60687764e-02, 6.53466284e-02, 2.84257364e-02, -7.09084561e-03, 8.53643045e-02], dtype=float32), kernel: Array([[-4.33085859e-01, -5.61205745e-01, 2.93745458e-01, ..., -1.06319696e-01, -3.59772116e-01, -5.50748073e-02], [ 1.29990995e-01, -1.94321834e-02, 3.05669427e-01, ..., 1.81368724e-01, 9.14699510e-02, -6.21524341e-02], [-2.22445935e-01, 1.21082552e-01, -5.40154229e-04, ..., -1.46840677e-01, -1.43376559e-01, 9.97718945e-02], ..., [ 6.94694817e-01, 8.00504535e-03, 7.26794839e-01, ..., -2.44273514e-01, -3.41350943e-01, -1.44506514e-01], [ 6.64600372e-01, -1.74710259e-01, 2.86397934e-01, ..., -4.65431027e-02, -2.18760711e-03, -2.81117141e-01], [ 5.85957587e-01, -1.68455601e-01, 3.16843003e-01, ..., -4.89029028e-02, -3.35241156e-03, -2.94326484e-01]], dtype=float32), }, }, }, layernorm_after: { bias: Array([ 6.88922033e-02, 5.80978803e-02, 3.85873437e-01, -3.11470646e-02, 1.69555983e-03, -2.32034996e-01, -1.58923380e-02, -3.62700596e-02, 5.75650949e-04, -3.82705927e-01, 2.31035739e-01, 6.68624789e-02, -1.72114998e-01, -5.66142239e-02, 2.09674109e-02, -2.80936039e-03, -2.27225497e-02, -3.59404653e-01, 1.31325588e-01, -3.94533155e-03, -1.76445603e-01, 4.06457812e-01, -3.71316727e-03, -2.46738438e-02, -1.81620270e-01, 4.64766249e-02, 7.05432147e-02, 7.28050619e-02, -5.19782789e-02, -4.45098169e-02, 3.48823667e-01, 6.42984435e-02, -3.04025542e-02, -9.68929306e-02, -5.14351986e-02, -3.97855928e-03, -3.20420787e-02, 2.67196506e-01, -7.22298101e-02, -2.07284018e-01, 4.90039028e-02, -2.19781891e-01, -1.35466352e-01, 8.54400694e-02, 1.19789563e-01, -1.32402414e-02, -2.06071258e-01, 3.45563710e-01, 1.00530580e-01, 8.49714726e-02, 3.91122419e-03, 2.15060711e-02, 8.78436714e-02, 2.07931370e-01, 3.97193991e-02, -1.13610223e-01, 3.73787954e-02, 1.23064490e-02, -9.43080038e-02, -1.86623499e-01, -3.85700941e-01, -1.63304899e-02, 1.10437632e-01, -1.03100136e-01, -6.19614311e-03, -3.90111618e-02, 1.53294563e-01, -1.00014314e-01, 8.98484327e-03, -1.33218288e-01, 1.19500153e-01, -1.21143341e-01, 2.10582778e-01, 1.27065793e-01, -1.65820807e-01, 2.54214227e-01, 5.12329042e-02, -9.36500803e-02, 5.69578595e-02, 3.65398824e-01, -1.66889839e-02, -8.60542282e-02, -2.17619090e-04, 7.38569945e-02, 1.74466029e-01, 7.53093883e-02, -1.50982007e-01, -1.61455601e-01, -4.24303383e-01, -1.41928092e-01, 1.67686343e-02, -2.84632683e-01, 1.45200789e-01, 5.28667821e-03, -2.16334872e-02, 4.74276505e-02, 1.93778221e-02, -1.02245426e-02, 8.27698484e-02, -4.25329745e-01, -9.56311524e-02, 2.45452020e-02, 5.58077872e-01, 8.93760026e-02, -3.42835188e-02, -7.27823749e-03, -4.62576747e-02, -1.11996412e-01, -2.86399245e-01, 3.93988378e-03, -5.40265851e-02, 1.06320893e-02, -1.11137971e-01, 9.71371382e-02, 7.98522159e-02, -7.13857710e-02, -4.83553410e-01, 1.70313239e-01, -1.70816466e-01, 8.59201849e-02, -3.64008136e-02, -2.24960689e-02, -3.12829800e-02, 1.58451021e-01, 9.86519852e-04, -3.24616969e-01, -2.68537730e-01, -1.29793763e-01], dtype=float32), scale: Array([ 0.16587867, 0.62729955, 0.78539467, 0.33312872, 0.02448054, 0.52365166, 0.09352965, 0.57490396, -0.00181935, 0.62166315, 0.5622861 , 0.632528 , 0.32434413, 0.33608988, 0.09364542, 0.00295282, 0.20884101, 0.4969771 , 0.49006283, 0.3390643 , 0.76553243, 0.75735354, 0.64736444, 0.20501752, 0.5099314 , 0.39772737, 0.4760952 , 0.2673762 , 0.5248422 , 0.34622046, 0.67513955, 0.12790139, 0.78136003, 0.46376252, 0.17142247, 0.02035132, 0.35290378, 0.79245657, 0.43180364, 0.58383363, 0.28102416, 0.6248626 , 0.09244277, 0.38200957, 0.36086422, 0.44281945, 0.62160456, 0.49688524, 0.47265998, 0.4368852 , 0.00439632, 0.24748251, 0.2077039 , 0.71143705, 0.55966383, 0.76935214, 0.2611507 , 0.3572642 , 0.4780699 , 0.5534745 , 0.5650403 , 0.49476576, 0.6574258 , 0.38081974, 0.15224549, 0.2871514 , 0.5269491 , 0.5443608 , 0.47709846, 0.6731636 , 0.48256594, 0.57757723, 0.52946705, 0.27599993, 0.11646109, 0.7459179 , 0.6025415 , 0.4426929 , 0.150529 , 0.6974664 , 0.07860193, 0.63189274, 0.6666647 , 0.8469711 , 0.4019909 , 0.45879367, 0.22790541, 0.5310602 , 0.46144342, 0.46353984, 0.3125676 , 0.6395497 , 0.5902147 , -0.00431166, 0.51785195, 0.44832194, 0.2982607 , -0.02292351, 0.48743743, 0.42023692, 0.5205952 , 0.08753084, 0.76318944, 0.42676944, 0.6018406 , 0.7163417 , 0.55152804, 0.8593687 , 0.4082306 , 0.4901796 , 0.64518034, 0.5525667 , 0.34930795, 0.46910506, 0.6457558 , 0.54612505, 0.61357117, 0.49350467, 0.41991618, 0.55927193, 0.4409146 , 0.15714881, 0.31889114, 0.26255667, 0.45123503, 0.56282735, 0.50075245, 0.34856102], dtype=float32), }, layernorm_before: { bias: Array([ 2.82420870e-02, -2.29402691e-01, -2.96053607e-02, -1.30823404e-02, -2.65389867e-03, 2.99529568e-03, -3.38511716e-04, -4.37224060e-02, 2.60172427e-01, -2.05475893e-02, 1.60881188e-02, -1.69427078e-02, -5.21531154e-04, 7.41502224e-03, -1.61934067e-02, 3.54598239e-02, -1.85820572e-02, 2.71367803e-02, -4.88283485e-03, -6.32894114e-02, 3.68127339e-02, -6.04641587e-02, 1.19915558e-03, 1.77571445e-03, -7.46919066e-02, -1.53914511e-01, -3.68062826e-03, 1.60319414e-02, 6.28713518e-02, 3.06911282e-02, -2.31577735e-02, -6.57155272e-03, 6.15396686e-02, 2.63886862e-02, -4.85325791e-02, 7.88616296e-03, 8.01346614e-04, -1.55313745e-01, -2.11913697e-03, -6.54404704e-03, 5.55190295e-02, -7.31622195e-03, 1.96012199e-01, 5.94329368e-03, -2.78697144e-02, -4.93782759e-02, 3.01586390e-02, 6.05354905e-02, -1.21998333e-03, 3.04064993e-02, 1.04687298e-02, 1.09375557e-02, -2.71287002e-03, -2.32986733e-01, 1.44290384e-02, -2.30109710e-02, 4.33547981e-02, 5.82204685e-02, -2.50215475e-02, 1.38790637e-01, -6.68904185e-02, 1.24691881e-01, -4.73514060e-03, -3.82121443e-03, 3.83599102e-02, 8.80813003e-02, -1.65379103e-02, -7.19220552e-04, 3.60673331e-02, 1.00804691e-03, -1.36750117e-02, -2.13624742e-02, 3.65178101e-02, 2.25900616e-02, -4.44582880e-01, -2.65965145e-02, -5.42465337e-02, -7.62177110e-02, 6.57133060e-04, -1.34424558e-02, 3.63113172e-02, 7.55314380e-02, 8.09766643e-04, -2.51245238e-02, -2.74020876e-03, -4.82323244e-02, -7.02568442e-02, 1.68840366e-03, 1.02340458e-02, 1.46568000e-01, -2.30909362e-02, 4.56218012e-02, -6.14786409e-02, -2.31754649e-02, 1.12327132e-02, 8.98616686e-02, 3.33565250e-02, 3.43459705e-03, -1.15024485e-02, -9.24066082e-02, -3.95509861e-02, -2.51406804e-03, -9.44837704e-02, -6.91622263e-04, -3.78696583e-02, -7.13908523e-02, -6.70393333e-02, 2.14871876e-02, -3.32830548e-02, -9.49056670e-02, 3.31703201e-02, 5.51439868e-03, -1.63990129e-02, 1.68498412e-01, -1.52823254e-01, 1.28563598e-01, 2.69770622e-02, -2.52538477e-03, -6.20844103e-02, -2.31217127e-03, -1.12146828e-02, 2.15321942e-03, -1.07783070e-02, 7.04224780e-03, -5.70042757e-04, -1.27387140e-02, -1.50637852e-03, -1.26768842e-01], dtype=float32), scale: Array([ 0.02225583, 0.3582427 , 0.18810613, 0.2897796 , 0.01008445, 0.20093079, 0.00262845, 0.18910973, 0.20207141, 0.05451531, 0.13650979, 0.22285123, 0.04530312, 0.00609608, 0.10985828, 0.09655713, 0.13278736, 0.08787937, 0.02501552, 0.16963154, 0.22782324, 0.20419084, 0.23176458, 0.04793768, 0.12091924, 0.15503103, 0.12642743, 0.07698647, 0.37561363, 0.1734934 , 0.22910334, 0.00436596, 0.25682634, 0.24328624, 0.2714059 , 0.06567708, 0.02274263, 0.567844 , -0.00143492, 0.16284508, 0.18691777, 0.3501844 , 0.11066452, 0.01387185, 0.17211044, 0.31377098, 0.1559427 , 0.31400132, 0.03055096, 0.1480449 , 0.06916483, 0.05761943, -0.00319004, 0.38499779, 0.0325634 , 0.3114719 , 0.07862399, 0.19379751, 0.12478653, 0.41179135, 0.26195255, 0.3847175 , 0.16452418, 0.00272222, 0.15891619, 0.11142635, 0.14923933, 0.10400768, 0.21697171, 0.12220434, 0.24840005, 0.16391885, 0.09268359, 0.12174353, 0.12482438, 0.2716236 , 0.22829536, 0.2231431 , -0.0092565 , 0.03284183, 0.06326266, 0.15956563, 0.06667496, 0.22676413, 0.01954163, 0.2572471 , 0.14380816, 0.3025213 , 0.13769881, 0.38677344, 0.05833767, 0.11660744, 0.1784911 , 0.31127843, 0.0378647 , 0.11842007, 0.13203348, 0.08775622, 0.22929348, 0.1685133 , 0.17730162, 0.14410907, 0.3452061 , -0.01377543, 0.44221473, 0.43522978, 0.29845122, 0.13120209, 0.45809656, 0.19538262, 0.46633697, 0.1401029 , 0.05973453, 0.4215374 , 0.16866335, 0.27390364, 0.03285838, 0.01355824, 0.18128489, 0.2254643 , 0.03559769, 0.00747606, 0.18407959, 0.02931737, -0.00183865, 0.0155194 , 0.2910443 , 0.2715314 ], dtype=float32), }, moe: { shared_experts: { intermediate: { bias: Array([-2.87663639e-01, -2.04487324e-01, -3.44864547e-01, -2.00529769e-01, -5.94829321e-01, -1.03927039e-01, -2.77930766e-01, -4.94999826e-01, -2.08718747e-01, -3.45035255e-01, -2.67087758e-01, -4.77189213e-01, -1.66359320e-01, -4.48330075e-01, -4.78699297e-01, -2.21549183e-01, -2.41500542e-01, -3.03187490e-01, -4.03688133e-01, -3.62672597e-01, 3.11330687e-02, -4.75372255e-01, -1.33399740e-01, -1.58057332e-01, -1.38183832e-01, -3.53402674e-01, -4.83101279e-01, -3.49856049e-01, -4.62018728e-01, -2.20034316e-01, -1.45977482e-01, -1.44924179e-01, -3.43084693e-01, -2.04671875e-01, -2.16533989e-01, -5.23977339e-01, -2.12659031e-01, -2.74191111e-01, -2.65450865e-01, -4.38724220e-01, -4.05285835e-01, -4.25657749e-01, -2.83261627e-01, -1.07905425e-01, -3.10509741e-01, -3.69468153e-01, -5.25859118e-01, -3.74539018e-01, -4.44122553e-01, -2.76227951e-01, -3.62340897e-01, -3.20163608e-01, -5.22766829e-01, -3.68136406e-01, -6.11795008e-01, -1.88745782e-01, -4.54058141e-01, -1.91686556e-01, -4.25439298e-01, -5.22909224e-01, -3.05993110e-01, -3.57376486e-01, -3.10534716e-01, -2.63750911e-01, -5.45675814e-01, -4.23284829e-01, -1.21132560e-01, -3.99448633e-01, -1.06209874e-01, -3.28158796e-01, -5.71545482e-01, -1.91518441e-01, -5.93747199e-01, -3.57770473e-01, -2.59463459e-01, -3.54470700e-01, -5.28468788e-01, -3.18399936e-01, -4.58652407e-01, -2.15082556e-01, -4.01729345e-01, -5.17883539e-01, -3.26544613e-01, -2.71224737e-01, -3.98273647e-01, -3.58481169e-01, -4.33161110e-01, -4.17540729e-01, -4.41764146e-01, -4.13900316e-01, -4.20035869e-01, -4.61230427e-01, -5.54932415e-01, -2.82214522e-01, -4.63347554e-01, -1.69810832e-01, -4.96465772e-01, -3.57517421e-01, -3.08119923e-01, -5.05688906e-01, -4.42314088e-01, -2.96192914e-01, -3.34470212e-01, -1.58352673e-01, -1.96208924e-01, -1.71411768e-01, -2.78193712e-01, -4.14353907e-01, -5.30355275e-01, -6.04073517e-02, -3.42649519e-01, -3.25455546e-01, -2.80562103e-01, -3.16360086e-01, -1.88339218e-01, -1.45719707e-01, -2.42292494e-01, -4.38852787e-01, -3.38206112e-01, -3.96864712e-01, -3.94322276e-01, -5.04216313e-01, -4.47647184e-01, -3.14450145e-01, -4.03099447e-01, -3.31715465e-01, -5.11624277e-01, -2.97003418e-01, -1.71126753e-01, -5.23082614e-01, -3.97670537e-01, -3.33513230e-01, -2.73618609e-01, -2.16470793e-01, -3.16209495e-01, -3.54464471e-01, -4.11175519e-01, -3.47123802e-01, -4.30214375e-01, -2.96357393e-01, -3.50862324e-01, -4.91615891e-01, -2.99721777e-01, -3.46929997e-01, -6.17211461e-01, -3.17567289e-01, -2.59017050e-01, -5.43685555e-01, -3.07671517e-01, -3.49160552e-01, -5.83933473e-01, -2.99324632e-01, -3.56959522e-01, -4.75278974e-01, -4.69556153e-01, -4.00068551e-01, -3.57878387e-01, -2.61400610e-01, 5.89218475e-02, -4.75838035e-01, -3.18645447e-01, -2.91967630e-01, -3.57156008e-01, -5.22750139e-01, -1.99007168e-01, -4.69590366e-01, -3.96272272e-01, -2.64699906e-01, -3.60923529e-01, -1.91638634e-01, -3.21910053e-01, -2.48550743e-01, -3.98402393e-01, -4.65990812e-01, -3.43011886e-01, -3.76989871e-01, -3.99559021e-01, -3.55907410e-01, -4.88500834e-01, -3.14257860e-01, -5.45107484e-01, -1.72966227e-01, -3.69181097e-01, -2.42310494e-01, -3.32116067e-01, -3.86646032e-01, -2.56473958e-01, -3.48980218e-01, -3.98971230e-01, -2.64990151e-01, -4.63962048e-01, -1.67646125e-01, -3.03776145e-01, -2.27759063e-01, -4.02701110e-01, -3.71156693e-01, -4.49759185e-01, -4.83259469e-01, -4.44558293e-01, -3.49739552e-01, -4.34326917e-01, -2.64975369e-01, -2.89425910e-01, -4.49308783e-01, -2.49016702e-01, -4.31563139e-01, -4.08892900e-01, -3.69399726e-01, -6.46372512e-02, -3.25122327e-01, -1.55182466e-01, -1.11164600e-01, -4.79935288e-01, -5.89126289e-01, -5.05477190e-01, -3.91809165e-01, -4.87700224e-01, -5.42157114e-01, -5.01625121e-01, -3.88096064e-01, -4.09615874e-01, -3.30858119e-03, -3.16691220e-01, -4.25819814e-01, -2.07181290e-01, -2.52645940e-01, -4.78967577e-01, -4.66646552e-01, -5.08263230e-01, -2.38753840e-01, -3.27672154e-01, -5.24233878e-01, -3.54793280e-01, -3.08880091e-01, -2.62713552e-01, -3.00624251e-01, -4.21428025e-01, -4.42182183e-01, -5.11559665e-01, -2.42620170e-01, -3.54224473e-01, -3.06683362e-01, -4.52630949e-04, -4.44990546e-01, -3.27576101e-01, -2.59946615e-01, -5.11295557e-01, -6.64835870e-01, -1.85168549e-01, -5.48847735e-01, -2.54403502e-01, -4.44106311e-01, -2.09086508e-01, -5.46322465e-01, -4.56374437e-01, -2.26774529e-01, -3.03409159e-01, -2.95624912e-01, -4.80216295e-01, -1.95244759e-01, -2.66378999e-01, -3.54911506e-01, -3.88011873e-01, -2.92912245e-01, -4.73809302e-01, -2.48538107e-01, -2.53374189e-01, -5.34175217e-01, 1.01788407e-02, -4.37775195e-01, -3.74520659e-01, -4.16397095e-01, -2.86437750e-01, -1.17193200e-01, -4.65466827e-01, -2.45054185e-01, -1.55902386e-01, -2.60219306e-01, -3.81529987e-01, -4.36873585e-01, -3.58277500e-01, -4.33071882e-01, -4.30247694e-01, -3.75352204e-01, -3.81595612e-01, -4.86366630e-01, -4.01012272e-01, -2.53684759e-01, -3.21343958e-01, -5.21693230e-01, -2.32155509e-02, -3.01609904e-01, -2.36861959e-01, -3.18232119e-01, -1.21864028e-01, -3.91450346e-01, -5.88354170e-01, -3.05314749e-01, -3.78017843e-01, -9.56703201e-02, -3.85766625e-01, -2.07299024e-01, -8.47784728e-02, -3.33295763e-01, -4.15781707e-01, -4.48703825e-01, -1.40794486e-01, -1.18951991e-01, -2.74386436e-01, -4.96934414e-01, -3.64418119e-01, -3.70397896e-01, -1.35903224e-01, -1.35731950e-01, -3.99609268e-01, -2.93618023e-01, -3.04051191e-01, -5.70618927e-01, -5.02060175e-01, -2.81283766e-01, -6.31259501e-01, -3.06242526e-01, -2.12949291e-01, -4.86519247e-01, -1.59262896e-01, -4.03478205e-01, -1.81495771e-01, -4.62975919e-01, -3.86971891e-01, -5.12782514e-01, -2.76577592e-01, -2.94954062e-01, -2.17922330e-01, -4.69827503e-01, 1.06356489e-02, -3.15419227e-01, -3.53538573e-01, -5.48389792e-01, 5.30533865e-02, -3.23666394e-01, -3.68639082e-01, -3.19764733e-01, -4.70012695e-01, -9.86519307e-02, -3.48372728e-01, -2.50590980e-01, -3.00890654e-01, -4.49694991e-01, -3.42396975e-01, -2.40296051e-01, -1.83449000e-01, -4.94159639e-01, -3.83832961e-01, -2.35955358e-01, -4.39356416e-01, -2.83128679e-01, -5.06890595e-01, -4.84923810e-01, -3.48398417e-01, -8.36336315e-02, -3.36799473e-01, -4.19064090e-02, -5.25771677e-01, -4.51774269e-01, -4.36057091e-01, -4.80761558e-01, -3.56559277e-01, -4.32838291e-01, -4.25217807e-01, -2.08370864e-01, -6.56363890e-02, -5.52178204e-01, -4.09472764e-01, -1.91283375e-01, -1.69247791e-01, -3.70922804e-01, -4.36929941e-01, -1.94023415e-01, -2.98483998e-01, -1.78429320e-01, -2.56660700e-01, -3.49405706e-01, -2.60902107e-01, -1.27692908e-01, -2.62808502e-01, -3.31191778e-01, -5.31970501e-01, -2.73841202e-01, -3.46037507e-01, -2.07595572e-01, -3.45578700e-01, -3.25063765e-01, -3.59042734e-01, -5.38096130e-01, -3.52915227e-01, -2.89237618e-01, -4.53390926e-01, -3.57132286e-01, -2.32824370e-01, -1.73414394e-01, -3.02039832e-01, -4.06891137e-01, -3.11633587e-01, -3.86790335e-01, -4.44779336e-01, -4.03608739e-01, -3.27035785e-01, -5.77829838e-01, -3.05345178e-01, -5.01044929e-01, -4.90038365e-01, -3.42383653e-01, -2.16015473e-01, -2.34810472e-01, -3.94850314e-01, -3.71235818e-01, 1.26175291e-03, -4.32831019e-01, -1.67813487e-02, -4.22147512e-01, -3.73436451e-01, -4.40798849e-01, -4.10787642e-01, -4.72505748e-01, -2.18411952e-01, -3.72730881e-01, -3.14533114e-01, -2.04394564e-01, -2.78175622e-01, -2.42158204e-01, -3.91372234e-01, -5.00931203e-01, -2.58081257e-01, -4.08814311e-01, -4.02594358e-02, -3.35627794e-02, -2.13214949e-01, -3.36978287e-01, -2.62033820e-01, -6.90276861e-01, -3.95373791e-01, -4.39152598e-01, -2.85017580e-01, -4.47889686e-01, -4.36591983e-01, -3.10838968e-01, -4.41035658e-01, -2.79656291e-01, -1.40883625e-01, -3.65119666e-01, -1.61859736e-01, -1.76689908e-01, -3.66094023e-01, -4.24499989e-01, -4.19989884e-01, -2.08415791e-01, -2.32176647e-01, -3.55920881e-01, -2.76795477e-01, -4.64629948e-01, -2.86160856e-01, -5.59463799e-01, -3.76166940e-01, -3.82688463e-01, -4.46783721e-01, -3.94609421e-01, -3.84682864e-01, -2.80228592e-02, -3.13305557e-01, -3.56978267e-01, -4.11862016e-01, -6.72479212e-01, -2.63180643e-01, -2.53385037e-01, -4.69251305e-01, -4.94098127e-01, -3.67969900e-01, -5.01329899e-01, -4.63324428e-01, -4.17908072e-01, -2.42769625e-03, -3.66800427e-01, -4.42783326e-01, -5.09920537e-01, -3.79778743e-01, -2.11185992e-01, -4.76333320e-01, -2.97944933e-01, -5.35430849e-01, -5.54302871e-01, -1.78685069e-01, -5.05887330e-01, -3.60490650e-01, -2.97754109e-02, -4.16262656e-01, -2.02613860e-01, -4.54910725e-01, -5.27239859e-01, -3.19837511e-01, -3.31910402e-01, -5.04021525e-01, -5.96606016e-01, -2.31208146e-01, -5.22311389e-01, -3.61870646e-01, -2.52372891e-01, -5.43789089e-01, -3.68237197e-01, -2.00233698e-01, -3.31239879e-01, -3.89182925e-01, -1.77743852e-01], dtype=float32), kernel: Array([[-0.03629679, 0.12771748, 0.30917847, ..., -0.02645334, 0.2853967 , 0.25290728], [-0.05366539, -0.08199085, -0.77184826, ..., -0.28385365, -0.08720582, -0.2480316 ], [-0.30405623, -0.14440256, -0.16176778, ..., -0.6061087 , -0.53592414, -0.557683 ], ..., [-0.5002675 , 0.6923438 , -0.39955443, ..., 0.2793648 , -0.0890847 , 0.13734408], [ 0.42013338, -0.1837182 , 0.20519936, ..., 0.59244996, 0.15519385, -0.04555327], [ 0.2887559 , 0.03648978, 0.19605577, ..., -0.3638592 , -0.05200996, 0.14020972]], dtype=float32), }, output: { bias: Array([-2.03454956e-01, 2.89009154e-01, -5.26833301e-03, 1.99230332e-02, 7.02989474e-03, -8.10585693e-02, 1.01039574e-01, 1.28230199e-01, 1.65420137e-02, 1.76779047e-01, 2.31052637e-02, -2.84619331e-02, -1.72500789e-01, 8.57431665e-02, -2.58707665e-02, 8.68742466e-02, 3.48051847e-03, -8.32902640e-02, 8.47522356e-03, -6.26579765e-03, 6.64773211e-02, -5.39215617e-02, -1.33164413e-02, -2.61053070e-02, -1.25059590e-01, 9.69652552e-03, -6.74541369e-02, 5.93836829e-02, 6.43064156e-02, 6.59576431e-02, 1.18919395e-01, -1.21684916e-01, 1.34139672e-01, -7.79727027e-02, 7.02831373e-02, -7.30708465e-02, 2.56273188e-02, 8.22603703e-03, 5.56182116e-03, -1.27773434e-01, -1.63553078e-02, -2.40529538e-03, -7.88959324e-01, 2.98491977e-02, -1.15960285e-01, -1.11549925e-02, 1.14158385e-01, 4.73596305e-02, -4.17886041e-02, -5.26564419e-02, -9.78346448e-03, 4.26121093e-02, -1.87094569e-01, 8.54555890e-02, -4.69494089e-02, -3.20936963e-02, -1.27747595e-01, -3.98493744e-02, 1.56673312e-01, 5.93540259e-02, 1.29704997e-01, -2.26318836e-01, -4.20836620e-02, 1.89136285e-02, 2.25093830e-02, -1.10592782e-01, -8.14836994e-02, -1.74993854e-02, 6.80994093e-02, -1.73572097e-02, -1.06341034e-01, 2.98510771e-03, -6.83057960e-03, -4.16511074e-02, 8.99981439e-01, 1.04607493e-02, -1.10422350e-01, 4.36907373e-02, -1.15947202e-01, -4.37050452e-03, -4.16922748e-01, 3.20061632e-02, 4.93779741e-02, 2.31584487e-03, -1.06097618e-02, 8.79805684e-02, 1.83927685e-01, 1.49958774e-01, 1.10785790e-01, 1.96547806e-02, -6.48459792e-03, 2.37252992e-02, -1.06167868e-01, -1.58290878e-01, -3.44699137e-02, -5.64344190e-02, 2.31592474e-03, 1.24252474e-04, -6.33904487e-02, 1.05357006e-01, 2.25357682e-01, -6.24567755e-02, 3.09208464e-02, -1.09267943e-01, 1.43063843e-01, -6.33983240e-02, -1.62469558e-02, -1.03259414e-01, -8.94914567e-02, -3.48297022e-02, -1.44557551e-01, 1.30005326e-04, 1.44872010e-01, -1.74515136e-02, 4.52921912e-02, 1.08484700e-01, -9.68603417e-03, -4.61588614e-03, -3.65241021e-02, -3.43756750e-04, 1.57616019e-01, 8.44166130e-02, 2.49973107e-02, -9.32366401e-02, 2.59006135e-02, 4.68501188e-02, 5.38958088e-02, 1.02374971e-01], dtype=float32), kernel: Array([[-0.35964 , 0.58528113, 0.12339137, ..., -0.2728965 , -0.35934824, -0.06503713], [ 0.35246447, -0.02949085, 0.5403422 , ..., -0.09529549, 0.11235058, -0.22131431], [ 0.13894935, -0.3717892 , -0.66237545, ..., -0.41706035, 0.52529556, 0.16840486], ..., [-0.09083151, 0.03931169, 0.2619683 , ..., -0.20559356, 0.15253511, 0.3468645 ], [-0.02631908, -0.0662327 , 0.12175545, ..., 0.10028415, -0.3420218 , 0.01938576], [-0.15720125, 0.16154028, -0.15429054, ..., -0.42136976, 0.29129225, 0.06028228]], dtype=float32), }, }, }, }, 2: { attention: { attention: { key: { bias: Array([ 4.91330713e-01, 9.67662875e-03, 5.60255200e-02, 8.97967637e-01, -3.06834251e-01, 1.06595111e+00, 5.69995821e-01, -9.72940326e-01, -1.00492425e-01, 2.51781523e-01, -3.73477489e-01, 1.62789464e-01, -3.40229303e-01, 8.37876360e-05, 4.15999711e-01, 8.95448923e-02, -1.16417743e-01, 1.02886930e-01, -1.22432733e+00, -2.46076301e-01, -5.80214441e-01, -4.96384725e-02, -2.68343747e-01, -3.29451412e-01, 2.24202454e-01, -6.01632036e-02, -6.19756997e-01, -3.03309470e-01, -1.28122672e-01, -2.58336246e-01, 4.15848672e-01, 6.79572225e-02, 1.68727383e-01, -1.08576640e-01, 9.50729072e-01, 1.18212245e-01, 2.77229160e-01, 4.02590066e-01, 2.45219976e-01, -2.20667735e-01, -6.55377328e-01, -2.22052753e-01, -2.34352529e-01, 7.21003652e-01, -2.47502819e-01, -8.39031518e-01, 7.14816153e-01, 5.66149652e-01, 1.96338549e-01, -3.21684837e-01, 1.26060262e-01, -1.13100424e-01, 7.06214011e-02, -3.80029410e-01, -5.25692463e-01, 5.33015609e-01, -2.28610292e-01, 8.77432287e-01, 4.54767972e-01, 1.05058357e-01, -1.01594612e-01, -4.25737798e-01, 8.72221589e-01, -4.27799642e-01, -7.26574242e-01, 3.97354126e-01, 5.40428281e-01, -6.92076981e-01, -7.33269453e-01, -8.38262439e-01, -9.10226703e-01, -2.79593736e-01, 5.23640335e-01, 2.84005404e-01, 3.39674622e-01, -2.09556028e-01, -1.83250681e-02, 2.20996439e-01, 8.39932729e-03, -1.35142893e-01, -4.47591990e-01, -5.44816703e-02, 3.56293529e-01, 3.84467632e-01, -4.40692216e-01, 1.00856110e-01, 8.52475286e-01, 3.95925105e-01, -5.06571472e-01, 4.37813222e-01, -8.72387230e-01, -4.80205119e-01, -6.85112596e-01, -6.57971025e-01, 3.89101595e-01, -2.59548306e-01, -2.97444463e-01, 2.12126691e-02, -8.14053118e-02, 1.06517822e-01, 2.81973779e-01, 7.66788945e-02, -4.01952624e-01, -1.77236781e-01, 6.38018996e-02, -3.04998636e-01, -3.08628559e-01, 1.56093314e-01, 1.61672279e-01, 1.33270577e-01, 1.59121871e-01, -1.46263152e-01, 1.54846618e-02, 2.81419247e-01, -3.13295014e-02, 2.87094891e-01, -3.95765811e-01, 4.69570488e-01, -4.26991880e-01, 2.32855052e-01, -1.62792206e-01, -3.81025262e-02, 4.74744802e-03, -7.95326605e-02, 1.28198788e-02, 1.01442777e-01, -4.87101451e-02, -3.22961807e-02], dtype=float32), kernel: Array([[-0.20145601, -0.22677487, -0.05235473, ..., -0.06058222, -0.01446399, 0.16699584], [ 0.22185628, 0.15144752, 0.23851915, ..., -0.10337625, -0.07871008, 0.01390354], [ 0.01399281, 0.06500424, -0.15749013, ..., 0.14727156, -0.08356368, 0.03307934], ..., [ 0.01401845, -0.0230756 , -0.0378951 , ..., -0.40631384, -0.10683019, 0.05925401], [-0.00425362, -0.07440671, 0.03620811, ..., 0.21968801, -0.18367934, 0.02989441], [-0.1020968 , 0.5364823 , 0.00217419, ..., -0.05559423, 0.13003536, -0.06085972]], dtype=float32), }, query: { bias: Array([ 0.3528763 , 0.3471857 , 0.21516396, 0.44737092, -0.4285367 , 0.3081475 , -0.25559098, -0.5126486 , 0.21324182, -0.32549337, 0.02813848, -0.38898697, -0.34589097, -0.19787419, 0.28709203, -0.24419545, -0.26928642, 0.30525193, -0.32151905, 0.0448105 , 0.51535946, -0.11308531, 0.3212626 , 0.4922951 , 0.26803052, -0.38731664, -0.2744102 , 0.39860168, 0.4643956 , 0.20418273, 0.14548776, 0.0657926 , -0.21296258, 0.15750653, -0.3648503 , 0.61559546, -0.35865372, 0.0193048 , 0.31175947, 0.2440965 , 0.318281 , -0.7344339 , -0.88320374, -0.05781611, 0.18718946, 0.20290868, -0.23830453, 0.5799299 , 0.01096278, -0.4863952 , -0.03866521, 0.30802044, 0.44045374, -0.03443577, -0.02167755, -0.01554048, -0.511046 , 0.41503018, -0.58751154, 0.5616692 , -0.26840666, -0.20692529, 0.71333766, 0.2290065 , 0.15057702, 0.1327177 , 0.31366843, 0.3856127 , -0.18231715, -0.24914798, 0.321846 , -0.3574352 , -0.12077462, 0.36877826, 0.38871482, -0.3660267 , 0.7790568 , 0.06110919, -0.22040306, -0.23536305, -0.07361323, -0.47640693, 0.53327733, 0.49116883, -0.5596567 , -0.7786464 , -0.1766858 , -0.14165151, -0.4430259 , 0.38748688, 0.02364437, -0.46849003, -0.23776187, -0.45196384, 0.65864664, 0.39194986, 0.07788196, 0.10206337, 0.15233228, 0.01135532, 0.08919958, -0.0229385 , -0.00552564, -0.6363855 , 0.04719624, 0.07559558, 0.0246234 , 0.32256266, -0.02410899, -0.37407702, -0.063107 , -0.24087183, -0.0432428 , 0.01937803, -0.12409952, 0.38090006, -0.8962724 , 0.51235485, -0.21019219, -0.06553419, -0.0576391 , -0.00670896, 0.0239524 , 0.00629916, 0.46627957, 0.02785698, 0.03592508, 0.06080098], dtype=float32), kernel: Array([[ 0.09377078, 0.06110698, -0.01986536, ..., -0.09585967, -0.00882398, -0.09038808], [-0.1099542 , -0.0294884 , -0.3200316 , ..., 0.14632395, -0.085163 , -0.12561767], [-0.17357844, -0.08655021, -0.17681202, ..., 0.2461566 , -0.11222455, 0.1451854 ], ..., [ 0.01483 , 0.12659054, 0.3020677 , ..., -0.1301052 , -0.02523333, 0.0047435 ], [-0.21112014, -0.10351861, 0.12283702, ..., 0.04758199, -0.02335739, -0.20571615], [ 0.04751823, -0.08418679, 0.021736 , ..., 0.23970562, -0.10594929, -0.2381722 ]], dtype=float32), }, value: { bias: Array([-0.20210104, -0.22905709, -0.14039448, -0.03683947, 0.12594987, -0.02302479, 0.00906336, -0.00965133, -0.07539421, 0.19282112, -0.14035326, -0.02629162, -0.0613457 , 0.19251512, 0.22437336, 0.01865598, 0.09472503, 0.07302232, 0.23662573, 0.1634112 , -0.25386894, 0.03644704, -0.14476314, -0.15108086, 0.30487335, -0.01728077, -0.03989496, 0.10778196, 0.01612386, -0.02545331, 0.10517047, 0.00122259, 0.06249418, 0.1109454 , 0.07476445, -0.00254933, -0.08234914, 0.00984051, -0.18144692, 0.16224049, 0.0791403 , 0.03824679, 0.05325351, 0.0459534 , -0.01707704, -0.05914249, -0.07880631, 0.09950723, 0.01833864, -0.12028188, 0.0201016 , 0.17571941, 0.08609623, -0.07171538, 0.27483281, 0.19407238, 0.01660765, 0.03474303, 0.09089345, 0.19815543, 0.21818227, 0.1436114 , 0.00318472, 0.06330492, -0.06290373, 0.03819067, -0.06003238, -0.03846627, 0.07632727, -0.05140862, -0.07411028, 0.21224949, -0.07434353, -0.09569198, -0.01517388, 0.00378666, 0.0954181 , 0.02467517, -0.0480962 , -0.07227837, 0.08442853, -0.11686811, 0.04613666, -0.04021059, 0.03575824, 0.02964796, 0.09947117, 0.07083944, -0.06642204, -0.07676494, 0.02942543, -0.0550985 , -0.06826791, 0.06714635, -0.05162644, -0.03433613, -0.00067884, 0.05411651, -0.01232108, 0.10868998, 0.05962406, -0.00136539, 0.01625578, -0.00632739, -0.0961687 , -0.07040943, 0.02730714, 0.00544707, 0.01023827, 0.03284573, 0.02555647, 0.0085297 , 0.03900068, 0.02747485, -0.03433363, -0.01524182, -0.00976253, -0.06111204, 0.10030263, 0.04285727, -0.00326952, -0.04434157, 0.11089424, 0.04277305, 0.02115745, 0.02765508, 0.08399435, 0.08146379], dtype=float32), kernel: Array([[-1.31283207e-02, 5.85139990e-02, 6.22775187e-05, ..., 4.85775769e-02, -4.02368233e-02, -6.80026598e-03], [-2.73164004e-01, -3.41370434e-01, -2.39238873e-01, ..., 4.36691791e-01, 3.35174829e-01, 8.86783656e-03], [ 6.34363815e-02, 3.22204083e-01, 2.63729244e-01, ..., -9.32745263e-03, -1.88997626e-01, 2.92390492e-02], ..., [-1.07346885e-01, -1.71600416e-01, -1.54479533e-01, ..., -2.66532987e-01, -7.03294575e-02, -3.63519728e-01], [-1.77037895e-01, -1.31959140e-01, 3.59883085e-02, ..., 1.12166613e-01, -3.25531811e-01, 4.15854841e-01], [ 1.20714135e-01, 1.70483068e-01, 6.37787461e-01, ..., -3.24400365e-01, -7.82848775e-01, -1.08967312e-01]], dtype=float32), }, }, output: { dense: { bias: Array([-1.42582014e-01, 3.51325125e-01, -5.49227633e-02, -1.36979753e-02, -8.89001694e-03, -4.82381359e-02, 7.59436563e-02, 4.62903231e-02, 1.25182196e-02, 1.74479246e-01, 5.47183445e-03, -4.21341211e-02, -1.15092970e-01, 5.72893657e-02, -5.69992810e-02, 5.97054660e-02, 2.31458675e-02, -9.98827890e-02, -1.63127463e-02, -5.68029238e-03, 1.06309690e-01, -5.58492132e-02, -2.88810022e-02, -5.41146100e-02, -1.25681654e-01, 1.55297220e-02, -1.05181850e-01, 2.84042470e-02, 8.00593048e-02, 1.12994172e-01, 6.64846674e-02, -8.10639188e-02, 6.96494877e-02, -7.10230693e-02, 2.48000082e-02, -6.40962422e-02, 5.03854305e-02, 2.99889389e-02, 2.71755550e-02, -1.32961541e-01, -9.42786504e-03, 6.68036118e-02, -7.92975008e-01, 2.92515382e-02, -1.21656477e-01, 6.97143525e-02, 9.77309272e-02, 9.14023891e-02, -1.06936611e-01, -4.60654832e-02, -3.59379910e-02, 6.73950538e-02, -3.20891030e-02, 1.82818487e-01, -4.29846235e-02, 1.71033782e-03, -2.07776949e-01, -8.03176984e-02, 1.26594633e-01, 1.53843045e-01, 9.50956270e-02, -1.38820022e-01, -5.31725921e-02, 3.35701555e-02, 5.83558269e-02, -1.29202589e-01, -4.29277681e-02, 5.83481155e-02, 1.43310845e-01, 3.71531360e-02, -1.16031744e-01, 1.25720399e-03, -3.66386138e-02, -5.68410717e-02, 8.81127954e-01, -3.54486227e-04, -1.14806622e-01, 5.00139184e-02, -1.25719443e-01, -2.92788483e-02, -5.60584605e-01, 1.90249607e-02, 7.62136094e-03, 3.29709910e-02, -1.73091292e-02, 1.49499878e-01, 1.71786591e-01, 1.50196671e-01, -4.66331327e-03, 7.90596679e-02, -6.07569050e-03, 1.27625940e-02, -6.54113144e-02, -1.46687955e-01, -2.00468884e-03, -7.76169971e-02, -5.61596686e-03, -2.51500830e-02, -5.23540825e-02, 1.42120197e-01, 1.58276126e-01, -3.11181620e-02, 4.16391157e-02, -9.11307335e-02, 1.01979673e-01, -1.40924305e-01, -2.65695825e-02, 3.79197043e-03, -9.74692628e-02, -5.42117134e-02, -1.30008578e-01, 3.88463400e-02, 1.63258657e-01, -1.50325231e-03, 3.45914699e-02, 1.69843420e-01, 1.55994203e-02, -1.33964028e-02, -1.95804201e-02, 1.02751255e-02, 1.84371233e-01, 7.10138083e-02, 5.72973862e-03, -1.13018960e-01, -8.22554808e-03, 5.45389391e-02, 1.77288230e-03, 1.88416634e-02], dtype=float32), kernel: Array([[ 0.15479796, 0.53571814, 0.51493394, ..., -0.80009085, -0.05134518, 0.13081257], [-0.00908494, 0.42273548, 0.710739 , ..., -0.3385195 , -0.41763583, 0.50498915], [ 0.21792167, 0.02829238, -0.15942883, ..., 0.14649649, -0.3748477 , 0.5899068 ], ..., [ 0.69713473, -0.10411429, 0.20231156, ..., 0.13196032, -0.09708058, -0.21109731], [ 0.5224463 , -0.00511254, -0.9390049 , ..., 0.06903978, 0.24796766, 0.5909974 ], [-0.6673977 , 0.24624978, -0.7719052 , ..., -0.09019069, -0.29529244, -0.12763341]], dtype=float32), }, }, }, layernorm_after: { bias: Array([-2.69522518e-02, 7.07723275e-02, -1.92383816e-03, -1.69017464e-01, 6.28127530e-03, 1.35174885e-01, 1.43263070e-02, -1.49136512e-02, 1.43651441e-02, 2.47675721e-02, 2.31170952e-01, -1.36354938e-02, -5.00643514e-02, 4.64095622e-02, 7.30278119e-02, 2.58173402e-02, -7.96374157e-02, -9.20170024e-02, -1.74471997e-02, 1.98375121e-01, 2.97931284e-01, 1.71703979e-01, -1.74371853e-01, -8.47198628e-03, -2.85758138e-01, 2.43080053e-02, -2.33928546e-01, 1.65304869e-01, 1.08051397e-01, -3.64561914e-03, -4.06746566e-01, -2.43781097e-02, -1.94358621e-02, 2.50640996e-02, 1.21675162e-02, 3.38391075e-03, 4.25222097e-03, -1.16181098e-01, -6.87912405e-02, -1.43533750e-02, 6.30778596e-02, 5.16471975e-02, 2.78629586e-02, -6.81551844e-02, -2.16658652e-01, 1.19643852e-01, -1.77933127e-02, 4.19648476e-02, -6.94157332e-02, 8.80410895e-02, 3.07484251e-03, -2.26102751e-02, 2.71407124e-02, -1.37352645e-02, 1.24339305e-03, -1.10182655e-03, -4.25179452e-02, 3.90366204e-02, -6.09224401e-02, 7.32898619e-03, -5.39776146e-01, 8.10408741e-02, 1.43810920e-02, -1.46246385e-02, 1.65752345e-03, -2.01134533e-01, 1.59140024e-02, 4.93869092e-03, 4.72723007e-01, -1.37103930e-01, 1.31060192e-02, -4.23336178e-02, 4.09131646e-02, 3.04093435e-02, 5.28761335e-02, 3.03985029e-01, -2.57239580e-01, 1.94385331e-02, 2.98021808e-02, -3.17842066e-02, -3.57343435e-01, 6.69872388e-02, -2.05376260e-02, 6.03192970e-02, -2.77649582e-04, 4.66996022e-02, 2.09306199e-02, 1.26473635e-01, -1.99992478e-01, 6.02058368e-03, -9.60124508e-02, -6.86570778e-02, -2.69254763e-02, -1.16648778e-01, 6.80445060e-02, 8.54630917e-02, 6.16764985e-02, -6.65853033e-03, -6.19218759e-02, -3.73522252e-01, -1.32694215e-01, 4.03534174e-02, 5.64134538e-01, 5.32403849e-02, -7.38940388e-02, -9.58362669e-02, -1.27810821e-01, 5.41193448e-02, 1.32223934e-01, 4.17461758e-03, 7.13659152e-02, 3.29776555e-02, 3.19321036e-01, -3.77637222e-02, 2.86701359e-02, 9.89994928e-02, 5.24967327e-04, 6.47896230e-02, -9.30845514e-02, -8.97478983e-02, 3.75827640e-01, 7.06133619e-03, -1.70939237e-01, -2.22015311e-03, -8.06940794e-02, -1.80535465e-01, -1.17046207e-01, -5.05085103e-02], dtype=float32), scale: Array([-2.16153208e-02, 4.99568492e-01, 1.90679863e-01, 3.98646802e-01, 3.23043484e-03, 8.04649472e-01, 1.73618346e-01, 6.32202104e-02, 6.03341032e-03, 7.60933161e-01, 5.64841628e-01, 3.30200821e-01, -1.13521464e-01, 4.35642749e-01, 7.02156186e-01, 1.86426133e-01, 2.69546956e-01, 4.07335073e-01, 3.06657463e-01, 7.59951055e-01, 8.35915983e-01, 5.73645175e-01, 3.52393031e-01, 2.92183794e-02, 5.95538437e-01, -3.08835153e-02, 5.68952084e-01, 6.18548512e-01, 7.05874801e-01, -1.11141577e-01, 7.34429657e-01, 1.59030482e-01, 6.69286311e-01, 3.07584852e-01, 2.97907770e-01, 2.45770905e-02, 3.42297733e-01, 6.38300121e-01, 4.57696557e-01, 3.39488059e-01, 4.27823365e-01, 6.61583364e-01, 2.51028594e-02, 6.87740266e-01, 6.38814211e-01, 6.12671137e-01, 3.40550840e-01, 3.15654457e-01, 5.78372836e-01, 5.64573705e-01, 5.55405932e-05, 3.71645719e-01, 8.48703533e-02, 9.91551802e-02, 4.97666523e-02, 5.78866065e-01, -1.33863851e-01, 2.80352771e-01, 5.00911117e-01, 1.85352847e-01, 7.25611448e-01, 4.00964379e-01, 2.84439921e-01, 3.20623189e-01, -1.70927703e-01, 4.05740440e-01, 3.79946142e-01, 2.70140201e-01, 5.38314104e-01, 6.14893734e-01, 5.95924675e-01, 5.98044157e-01, 1.48603588e-01, 3.81722450e-01, 5.22187948e-01, 7.07299054e-01, 4.85696197e-01, 1.27313018e-01, 7.40394592e-02, 3.15272629e-01, 3.43401968e-01, 6.31168902e-01, 2.14417100e-01, -3.38677377e-01, 2.39464089e-01, 4.71792251e-01, 3.14170390e-01, 8.54388118e-01, 6.38605714e-01, 1.02670453e-01, 4.03134346e-01, 3.42762381e-01, 4.40316945e-01, 5.73590755e-01, 3.09652478e-01, 1.55366600e-01, 6.81986809e-01, -7.89772999e-03, 5.20319641e-01, 6.24004126e-01, 5.56326687e-01, 3.05760831e-01, 9.59146619e-01, 5.92343211e-01, 3.85424703e-01, 4.82657462e-01, 5.25808513e-01, 4.87956852e-01, 3.75198275e-01, -6.13166541e-02, 5.11129975e-01, 6.06624246e-01, 7.52410650e-01, 1.93235308e-01, -5.42348363e-02, 3.70484799e-01, 6.81148842e-03, 7.82989919e-01, 6.00687087e-01, 4.27850395e-01, 9.20885563e-01, 2.06756860e-01, 7.61731803e-01, 6.10257149e-01, 2.84399986e-01, 6.83825493e-01, 5.32033622e-01, 1.40386716e-01], dtype=float32), }, layernorm_before: { bias: Array([ 1.7358573e-02, -2.1321216e-01, -3.0296935e-02, -8.5394399e-04, -3.9229371e-02, 5.5817175e-03, -2.7252757e-03, -4.4029057e-02, -2.0724675e-03, -1.2441806e-01, -2.2284750e-02, -2.7261129e-02, -2.7341412e-03, 3.6942791e-03, 4.5538493e-03, 7.4563054e-03, -6.2132953e-03, 9.7167321e-02, -2.5345799e-02, -3.9951671e-02, -4.3273875e-03, -1.4203915e-03, 6.3152872e-02, 1.3384975e-03, -5.5942591e-02, 6.3980678e-03, -3.2894940e-03, 2.6343842e-03, 2.8435083e-02, 2.8260322e-02, 1.2178370e-02, -5.3195474e-03, -2.4072591e-03, 4.1922098e-03, -7.3268535e-03, 7.5856894e-03, 3.8502638e-03, -9.8595031e-02, 2.4951894e-02, -9.5122019e-03, -5.4523430e-04, 4.9118581e-03, 2.0277210e-01, -1.6341466e-02, 3.9525043e-02, -2.2266436e-02, -2.0717401e-03, -1.2440056e-01, 2.4503253e-03, 1.1633219e-02, 3.5871702e-04, 1.0992530e-03, 7.6659650e-02, -6.7069583e-02, 9.7138679e-04, -2.9395521e-02, -8.4895557e-03, -1.3592694e-02, 2.9208256e-02, 1.7238649e-02, 5.7587404e-02, 4.4692371e-02, 1.6647557e-02, -6.6133849e-03, -3.3748765e-03, 9.0554915e-03, 1.8321996e-03, 1.5999395e-02, -1.3479884e-02, 2.9679830e-03, -2.7784597e-02, 3.4300985e-03, 4.0611736e-02, -4.6936791e-02, -7.2169447e-01, 8.0935545e-03, -1.8306768e-02, -5.0263423e-02, -3.8130886e-03, 1.9901623e-03, 2.5550249e-01, 6.1443527e-03, -2.2893531e-02, -1.7245837e-02, 2.9439591e-02, -4.3969110e-02, -6.7670122e-03, -1.3685081e-02, -9.9776837e-04, 9.0234309e-02, -1.4077160e-02, 2.6768299e-03, -3.8174929e-03, -1.4334331e-02, 3.4874912e-02, 8.0644749e-02, 8.6499676e-03, -1.7143000e-02, 7.9911202e-03, 2.8301086e-02, -4.1625272e-02, -3.0587490e-03, -5.6571878e-02, 9.0596341e-03, -7.6397270e-02, -4.5915369e-02, -6.2590851e-03, 3.4895688e-02, 6.1382476e-02, -5.6433260e-02, 1.0131227e-02, 1.5290984e-03, -5.1695153e-02, 1.3336526e-01, 1.1029670e-02, -2.2941723e-02, 2.4411927e-03, -2.4650490e-02, -1.1010250e-02, -7.5843465e-03, -1.1662335e-02, 3.2463686e-03, -1.6854508e-03, 2.7790260e-02, 7.3739677e-03, -9.9853233e-02, 1.0033315e-02, -9.9916525e-02], dtype=float32), scale: Array([ 7.94857554e-03, 1.53508723e-01, 7.40691870e-02, 1.35285795e-01, 2.26536706e-01, 1.69742465e-01, 8.65431502e-04, 3.19571525e-01, 1.62680820e-03, 1.62808627e-01, 1.34147704e-01, 2.01822311e-01, 6.27691345e-03, 7.19762873e-03, 3.74700986e-02, 3.44181597e-01, 9.95386466e-02, 1.60152465e-01, 9.15851742e-02, 1.63604781e-01, 1.17830604e-01, -1.66135898e-03, 2.23148689e-01, 1.00941835e-02, 2.60033756e-01, 1.75621884e-03, 2.65612965e-03, 4.38557938e-04, 3.29755515e-01, 2.43666410e-01, 3.20717841e-01, 1.77067131e-01, 2.68771589e-01, 3.02107055e-02, 2.95764953e-02, 1.08823786e-02, -1.60506088e-02, 2.95251936e-01, -1.78114280e-01, 8.72005969e-02, -8.84137396e-03, 2.99656868e-01, 8.63162801e-02, 1.33525252e-01, 3.03551525e-01, 2.60979742e-01, -5.50631993e-03, 2.83051997e-01, 2.56603450e-01, 2.00551033e-01, 5.14968205e-03, 2.78209685e-04, 2.23627791e-01, 2.34901443e-01, -8.34921002e-03, 2.24900410e-01, -6.23680511e-03, -2.86224540e-02, 2.08953783e-01, 3.78563493e-01, 2.40491629e-01, 1.92855686e-01, 2.62306392e-01, 5.38155213e-02, 2.10460991e-01, 1.67684779e-02, 5.48215546e-02, 1.18601710e-01, 3.43158811e-01, 2.41343841e-01, 2.74241388e-01, -6.84597194e-02, 2.30040535e-01, 2.19160318e-01, 1.71067789e-01, 2.74585098e-01, 1.48560002e-01, 2.04053581e-01, -1.04316017e-02, 3.06529433e-01, 1.51589945e-01, -2.79646995e-03, 2.60486454e-01, 9.33720767e-02, 2.20337927e-01, 1.95606187e-01, -4.61113850e-05, 7.93705583e-02, 2.31670171e-01, 1.81787178e-01, 6.02688640e-02, 6.66163536e-03, 5.13362838e-03, 8.27364624e-02, 3.05262417e-01, 1.02641329e-01, 1.57344952e-01, 1.41675845e-01, 4.87802178e-03, 1.00217178e-01, 1.87011242e-01, -1.03036012e-03, 2.17317730e-01, 1.16062611e-01, 3.36587638e-01, 3.04190874e-01, 1.32016614e-01, 1.66113153e-01, 3.42909336e-01, 2.46614054e-01, 2.17617258e-01, 1.86562017e-01, 1.04256548e-01, 3.53437752e-01, 5.86070295e-04, 2.16139719e-01, 2.78897047e-01, 2.12332204e-01, 3.80921900e-01, 1.19425990e-01, 3.23329829e-02, -1.03264116e-02, 1.89511199e-02, 1.55454144e-01, -3.36376615e-02, 2.82643199e-01, 3.45727682e-01, 2.41249815e-01], dtype=float32), }, moe: { shared_experts: { intermediate: { bias: Array([-2.87919283e-01, -1.59886956e-01, -3.92205477e-01, -4.32191879e-01, -3.32138002e-01, -5.40607274e-01, -3.54391366e-01, -1.31854281e-01, -5.39756715e-01, -4.69244629e-01, -3.44658494e-01, -4.88029271e-01, -3.98942500e-01, -4.15098637e-01, -3.66716921e-01, -5.07466793e-01, -2.69494671e-02, -3.29423159e-01, -3.49476814e-01, -1.71700358e-01, -3.80648106e-01, -4.23726380e-01, -4.98886943e-01, -4.94191587e-01, -2.60305732e-01, -3.48132432e-01, -4.27568406e-01, -3.35825920e-01, -3.69937718e-01, -3.21786374e-01, -2.29871064e-01, -2.88744688e-01, -3.85481507e-01, -4.31343943e-01, -1.41554534e-01, -1.79118499e-01, -4.28618819e-01, -2.48103037e-01, -9.70930979e-02, -4.53777105e-01, -1.97259694e-01, -3.89665425e-01, -4.33900356e-01, -2.01605186e-01, -3.11053962e-01, -3.33365947e-01, -4.72573489e-01, -4.81180519e-01, -4.59191829e-01, -3.63978118e-01, -5.11594951e-01, -3.21153313e-01, -5.05568504e-01, -4.40596819e-01, -2.05569610e-01, -2.04613134e-01, -2.30359152e-01, -4.58120763e-01, -1.35851100e-01, -2.78527498e-01, -5.32928765e-01, -4.59086925e-01, -1.91062093e-01, -2.50361383e-01, -3.23409230e-01, -5.09010673e-01, -2.47201502e-01, -3.10001701e-01, -3.00568342e-01, -3.73322159e-01, -5.35207331e-01, -5.31656861e-01, -3.18275809e-01, -4.40275580e-01, -3.43852162e-01, -4.12837327e-01, -3.85347039e-01, -4.52745438e-01, -2.02980176e-01, -4.54181701e-01, -5.00234187e-01, -3.34358454e-01, -3.39169592e-01, -2.89554566e-01, -3.29827011e-01, -5.80846727e-01, -3.28373313e-01, -4.26805854e-01, -3.15485120e-01, -3.92600238e-01, -3.12504858e-01, -2.19271436e-01, -1.64767921e-01, -5.02310932e-01, -2.58719236e-01, -4.32749659e-01, -9.22016874e-02, -4.70086277e-01, -3.97373617e-01, -4.89154935e-01, -4.44214731e-01, -2.03317106e-01, -4.75122511e-01, -2.69819319e-01, -2.67087281e-01, -4.90763247e-01, -4.22214806e-01, -2.64238060e-01, -2.23214269e-01, -3.43690515e-01, -2.78915852e-01, -4.24799353e-01, -5.69192111e-01, -2.07584277e-01, -1.64182380e-01, -1.91950992e-01, -5.61932206e-01, -5.16008019e-01, -3.33474815e-01, -4.25828815e-01, -1.61096185e-01, -3.98274213e-01, -5.68151116e-01, -2.51645744e-01, -2.21852303e-01, -4.22558784e-01, -3.82831007e-01, -3.19712460e-01, -1.66578472e-01, -2.49394879e-01, -4.19118315e-01, -2.93457031e-01, -2.79008389e-01, -5.07268071e-01, -1.48156717e-01, -4.60653484e-01, -2.05382407e-01, -4.70562130e-01, -2.83998400e-01, -4.29632753e-01, -2.36523062e-01, -3.75578731e-01, -3.47366124e-01, -2.43162408e-01, -4.38231677e-01, -5.36709070e-01, -1.51090562e-01, -4.71059680e-01, -4.89868373e-01, -3.91524017e-01, -3.24053317e-01, -3.04224849e-01, -4.36558515e-01, -2.35701188e-01, -4.07339513e-01, -2.95855463e-01, -2.55902320e-01, -4.83437032e-01, -3.88129205e-01, -1.03971243e-01, -2.60267794e-01, -3.92015398e-01, -4.49646682e-01, -2.48017311e-01, -2.56850272e-01, -1.90488473e-01, -4.68013018e-01, -4.13734376e-01, -5.12630165e-01, -4.26820993e-01, -3.74209106e-01, -3.59761298e-01, -4.02847528e-01, -2.97854811e-01, -3.58279765e-01, -3.24418694e-01, -3.66349548e-01, -4.33402061e-01, -4.91001993e-01, -5.03202379e-01, -5.00787556e-01, -3.92910451e-01, -2.84466565e-01, -3.34319711e-01, -4.25348043e-01, -2.26335734e-01, -2.37801537e-01, -4.87084776e-01, -3.79190952e-01, -3.08594137e-01, -3.53578776e-01, -2.29536459e-01, -4.85832751e-01, -3.31020027e-01, -3.49860758e-01, -4.98145282e-01, -2.17634872e-01, -3.08456481e-01, -4.34388995e-01, -2.14153975e-01, -2.20951349e-01, -4.75258082e-01, -2.87501752e-01, -9.52811018e-02, -5.12147367e-01, -2.70634055e-01, -2.17771903e-01, -3.38476300e-01, -3.06095690e-01, -3.33468020e-01, -1.24255188e-01, -5.16220808e-01, -2.23946184e-01, -5.59429586e-01, -2.83430904e-01, -4.03296888e-01, -3.68934721e-01, -3.25362712e-01, -7.39004537e-02, -3.09295535e-01, -4.49250668e-01, -5.65693378e-01, -2.64678180e-01, -3.42923045e-01, -3.37939441e-01, -3.71786445e-01, -4.06780869e-01, -5.13805926e-01, -1.10545248e-01, -4.07462388e-01, -4.62072462e-01, -3.73685181e-01, -4.82776046e-01, -1.83113769e-01, -1.37951612e-01, -4.32110876e-01, -4.39051688e-01, -4.20039445e-01, -4.49939400e-01, -5.46136618e-01, -4.43518162e-01, -3.36255163e-01, -2.60323614e-01, -1.48123205e-01, -3.08105409e-01, -5.92286289e-01, -3.61411870e-01, -5.49996793e-01, -4.72203493e-01, -1.12566963e-01, -3.46157908e-01, -2.21687108e-01, -1.49518579e-01, -4.22669083e-01, -5.09057462e-01, -3.78608346e-01, -2.69701719e-01, -2.57610738e-01, -3.43267351e-01, -3.99610460e-01, -5.19208014e-01, -3.55502129e-01, -3.64218682e-01, -4.58064228e-01, -1.66265354e-01, -4.56624329e-01, -4.21405643e-01, -3.38668913e-01, -4.08919096e-01, -2.49855325e-01, -2.68893301e-01, 5.37731161e-04, -4.11918223e-01, -2.53745079e-01, -2.37736523e-01, -3.77250522e-01, -3.40394109e-01, -4.99163866e-01, -3.26041222e-01, -3.24631035e-02, -4.25087065e-01, -3.99343103e-01, -4.45631206e-01, -2.19975799e-01, -4.91138816e-01, -3.16317588e-01, -1.97857752e-01, -4.38183516e-01, -2.38349304e-01, -3.12527388e-01, -4.46206242e-01, -1.97425947e-01, -3.54068547e-01, -5.62409818e-01, -4.79922563e-01, -4.07248676e-01, -5.08961380e-01, -3.35608751e-01, -4.69772518e-01, -4.64478105e-01, -2.38058373e-01, -3.87834966e-01, -4.91571218e-01, -1.16468966e-01, -1.88293502e-01, -2.19764918e-01, -2.41493329e-01, -3.08782160e-01, -3.80097032e-01, -2.52258271e-01, -3.71472836e-01, -3.24205458e-01, -4.05035138e-01, -5.40303230e-01, -1.73389211e-01, -3.20962131e-01, -2.16032237e-01, -1.91938236e-01, -1.65382430e-01, -4.07554179e-01, -3.40726435e-01, -2.95837790e-01, -9.07381624e-02, -1.67001665e-01, -3.45026761e-01, -4.43528742e-01, -4.61021096e-01, -2.25696385e-01, -2.09777251e-01, -5.53110957e-01, -3.23776424e-01, -3.86852056e-01, -5.13776362e-01, -1.70138955e-01, -4.78795141e-01, -2.65858293e-01, -1.22696631e-01, -5.30045271e-01, -3.90655428e-01, -2.64281482e-01, -4.28456664e-01, -1.40250340e-01, -2.27036089e-01, -3.66672546e-01, -4.42133546e-01, -4.25859839e-01, -2.67484695e-01, -4.62734908e-01, -3.37051451e-01, -4.50581521e-01, -7.01922104e-02, -3.47592622e-01, -3.91412824e-01, -4.24011290e-01, -3.32346976e-01, -3.72510850e-01, -3.09896857e-01, -1.88529536e-01, -3.71884912e-01, -2.54814714e-01, -3.63058448e-01, -4.75363225e-01, -3.06162030e-01, -4.65719730e-01, -1.81163147e-01, -4.85125124e-01, -3.95848036e-01, -4.33863193e-01, -3.04795027e-01, -4.90597248e-01, -2.44484976e-01, -3.44704390e-01, -3.45088750e-01, -3.29458416e-01, -2.94500917e-01, -2.62573928e-01, -1.69639900e-01, -4.71560776e-01, -2.42599428e-01, -3.69580209e-01, -3.42656642e-01, -3.11950177e-01, -1.34305000e-01, -1.57368377e-01, -3.91964465e-01, -3.12808394e-01, -2.37698436e-01, -3.69058728e-01, -1.38768077e-01, -2.69422363e-02, -4.55560267e-01, -3.79797220e-01, -4.36610907e-01, -2.92106032e-01, -3.98713797e-01, -4.69899833e-01, -1.97569534e-01, -3.96044046e-01, -1.28238350e-01, -3.10444951e-01, -4.51549977e-01, -2.95563221e-01, -4.12408620e-01, -4.34255183e-01, -2.34395146e-01, -4.32429850e-01, -2.51709074e-01, -3.16113442e-01, -2.24395722e-01, -4.12795454e-01, -3.96394074e-01, -3.76580715e-01, -2.59395570e-01, -5.12954056e-01, -3.32104653e-01, -4.36192185e-01, -5.13575554e-01, -1.39632359e-01, -4.51251060e-01, -4.95827556e-01, -2.47558579e-01, -3.28758836e-01, -4.00682449e-01, -1.53779313e-01, -2.39558816e-01, -3.46421301e-01, -2.07403153e-01, -4.59828466e-01, -4.13055509e-01, -5.19209445e-01, -3.49062324e-01, -3.61741424e-01, -5.46588659e-01, -3.25105667e-01, -2.64346927e-01, -3.62703383e-01, -3.54973465e-01, -3.57288867e-01, -3.40226203e-01, -1.87480614e-01, -3.84213597e-01, -3.67905647e-01, -2.21579894e-01, -4.32286561e-01, -3.49667877e-01, -2.67124563e-01, -2.31143355e-01, -2.09680378e-01, -4.44781452e-01, -1.10374503e-01, -1.90934509e-01, -4.75128025e-01, -4.90421832e-01, -3.84051919e-01, -3.26278806e-01, -3.82725775e-01, -2.96725929e-01, -4.35144246e-01, -3.04508001e-01, -4.23557281e-01, -3.32650095e-01, -2.34854251e-01, -2.51995057e-01, -3.14376324e-01, -4.71209139e-01, -5.88116467e-01, -3.59949678e-01, -3.27794611e-01, -5.37792504e-01, -2.25097418e-01, -5.30355632e-01, -5.33735216e-01, -2.61757433e-01, -2.64203936e-01, -3.20991725e-01, -3.19024026e-01, -4.41864610e-01, -1.51468694e-01, -6.96782321e-02, -4.49556112e-01, -3.41252834e-01, -6.03692472e-01, -4.20485705e-01, -4.03241307e-01, -4.20258135e-01, -4.89987731e-01, -2.22824812e-01, -1.52097628e-01, -3.01070571e-01, -3.31862330e-01, -2.91463912e-01, -2.47648209e-01, -4.18553919e-01, 8.17964971e-02, -4.42326665e-01, -4.14036870e-01, -4.23043162e-01, -4.24916655e-01, -3.22944313e-01, -3.46353531e-01, -3.03149909e-01, -4.78416532e-01, -3.12508672e-01, -3.28645647e-01, -3.10102880e-01, 6.61208481e-02, -3.10083032e-01, -4.34432387e-01, -2.68977493e-01, -3.87750000e-01, -3.06754440e-01, -2.68470585e-01], dtype=float32), kernel: Array([[-8.63173604e-02, 5.54475226e-02, -7.44883567e-02, ..., -1.60854563e-01, -1.92300782e-01, 1.49715483e-01], [-2.55667627e-01, -5.76032937e-01, -2.72577375e-01, ..., -1.66639566e-01, -9.29499939e-02, -8.04038703e-01], [ 1.05862938e-01, -7.12434500e-02, -3.51943791e-01, ..., -4.56090540e-01, 6.42790049e-02, 2.83889264e-01], ..., [ 3.32350917e-02, -2.16388881e-01, -4.15698439e-01, ..., -2.22950518e-01, 1.03157982e-01, 4.05906420e-03], [ 1.34180099e-01, 2.93415725e-01, -3.85918528e-01, ..., 5.76932847e-01, -1.18126400e-01, 1.22173645e-01], [-3.73824616e-04, 1.94809213e-02, 2.37895355e-01, ..., -2.55678475e-01, -4.13769521e-02, 8.79469365e-02]], dtype=float32), }, output: { bias: Array([-0.14310208, 0.30402932, -0.04476445, 0.05614336, -0.00636329, -0.10514647, 0.06905492, 0.06197605, 0.02608861, 0.15183868, -0.0424929 , -0.01462859, -0.1350344 , 0.02212298, -0.05124652, 0.06123491, 0.03247906, -0.09240599, -0.01013029, -0.03223318, -0.0887438 , -0.06346248, 0.02817548, -0.05015382, -0.05139951, 0.01631817, -0.04831094, -0.02007524, 0.04097073, 0.1089523 , 0.14585266, -0.05194474, 0.04915197, -0.09083902, 0.01939188, -0.06052509, 0.05472679, 0.04450262, 0.03008708, -0.1406127 , -0.0286117 , 0.04946226, -0.8201933 , 0.05301183, -0.06421784, 0.03291068, 0.08303672, 0.10530804, -0.05531342, -0.09611814, -0.02686962, 0.10026816, -0.01932072, 0.17867358, -0.03787487, 0.01405966, -0.20247376, -0.07591344, 0.15851738, 0.16269532, 0.17163317, -0.16734251, -0.05690756, 0.02852942, 0.06089685, -0.04197973, -0.06455492, 0.07540473, -0.06004319, 0.08624789, -0.1040978 , 0.00247931, -0.02840352, -0.04520692, 0.8720378 , -0.06833719, -0.07094937, 0.05479927, -0.12080298, -0.0121475 , -0.5797822 , -0.02564085, 0.03635592, 0.03610236, -0.00385189, 0.1565094 , 0.14343542, 0.05017952, 0.00714095, 0.09029737, 0.0294838 , 0.01819415, -0.04793838, -0.1418107 , -0.00120118, -0.06547695, -0.03586074, -0.01915462, -0.03162472, 0.25057518, 0.21378383, -0.03219284, -0.03374225, -0.07630365, 0.13850403, -0.09364668, 0.02092841, -0.02086668, -0.11372983, -0.06087056, -0.14589557, 0.03926003, 0.0446431 , 0.00222743, 0.03577771, 0.14117669, 0.01693195, -0.06432896, 0.02443499, 0.04788267, -0.13781525, 0.06813591, 0.10359713, -0.10075218, 0.02409258, 0.05323493, 0.00397177, 0.03774352], dtype=float32), kernel: Array([[ 0.16250458, -0.15818997, 0.3543968 , ..., -0.06277009, -0.458476 , 0.21236774], [ 0.17157121, -0.33716652, 0.17193943, ..., -0.07994235, 0.03173731, -0.09947567], [ 0.20364675, 0.04469377, -0.3887421 , ..., -0.24748199, 0.29170632, -0.02271716], ..., [ 0.29266545, -0.02244417, 0.41813877, ..., -0.37108174, -0.26156428, -0.10771853], [-0.07057951, -0.23378706, 0.05101302, ..., 0.08562624, 0.45771068, 0.08117881], [-0.21127233, -0.6031122 , -0.4248507 , ..., 0.16734137, 0.09727449, -0.12857328]], dtype=float32), }, }, }, }, 3: { attention: { attention: { key: { bias: Array([-0.22031489, 0.02730584, 0.23429762, -0.3125938 , 0.22592317, -0.6771136 , -0.21247995, -0.341136 , 0.07302368, 0.44729188, 0.22906882, -0.524169 , -0.04314182, -0.22182061, -0.67601067, -0.01997758, -0.00890862, 0.31220517, 0.3243842 , -0.08772013, 0.18925034, -0.43556792, -0.30325386, 0.13643748, 0.27908477, 0.26076847, 0.19125256, 0.6231867 , 0.34878573, 0.21458477, 0.12557186, 0.25881395, 0.11842591, 0.14228185, -0.14626004, 0.02273282, 0.04946363, 0.3702873 , 0.4587399 , 0.46696767, 0.2649066 , -0.02744056, 0.21641424, 0.4786166 , -0.39902607, -0.25150967, 0.20756108, -0.9501807 , 0.2439707 , 0.07372095, 0.1429905 , 0.5078466 , -0.47601166, 0.01509763, -0.63272023, 0.5627352 , 0.5814683 , 0.3356227 , -0.07830408, -0.29685998, 0.09222841, -0.49363494, -0.09527437, -0.01283222, 0.10691621, 0.44450942, 0.2378995 , 0.14754085, 0.07311877, -0.4130656 , 0.12185058, 0.1367209 , 0.18846467, -0.11077244, -0.10472129, 0.13243866, -0.6801274 , 0.18685853, -0.40455377, -0.17242797, -0.3273368 , 0.21752419, 0.5587714 , 0.33085388, -0.13658075, 0.11016981, -0.14520068, -0.0551539 , -0.0988646 , 0.19754803, 0.28224358, 0.27123928, 0.01996558, 0.25261068, 0.3291632 , 0.73044914, -0.7135282 , -0.2712479 , 0.17239682, 0.4813283 , 0.350012 , 0.76145405, 0.01346729, 0.28428316, 0.17260526, -0.40829295, 0.66129416, -0.35163417, -0.15398747, -0.87850165, 0.16401769, -0.36521718, 0.08581091, 0.12348412, -0.731215 , 0.24665482, 0.77237046, 0.39620268, -0.4511514 , 0.34254473, -0.669006 , -0.50109476, -0.18661116, 0.2477596 , 0.65949994, 0.01988118, -0.04483053, -0.7621275 ], dtype=float32), kernel: Array([[ 0.14074115, 0.11149215, 0.02833093, ..., 0.24423462, -0.3480132 , -0.06901791], [ 0.13614887, -0.2725882 , -0.31767422, ..., -0.30451745, -0.0344476 , 0.2648898 ], [ 0.19196813, 0.24347357, 0.09369045, ..., 0.3346163 , -0.10084 , -0.19047955], ..., [ 0.02364591, 0.3210728 , 0.15941176, ..., -0.5476011 , 0.30842453, 0.16627334], [-0.3327589 , 0.2611727 , 0.04791668, ..., 0.5718048 , -0.00553066, -0.12860055], [ 0.13814813, -0.2723826 , -0.10484312, ..., 0.10856424, -0.46137026, 0.3135762 ]], dtype=float32), }, query: { bias: Array([ 0.01636963, 0.00303602, -0.2237812 , 0.17135 , -0.5504063 , 0.71538967, 0.34468257, 0.6879721 , -0.32814807, 0.12084308, -0.23467594, 0.24040897, -0.03144267, 0.41499555, 0.18407044, -0.19885245, 0.29428306, -0.17422631, -0.72670996, 0.19728792, -0.12558015, -0.04110402, 0.7530146 , -0.32672936, -0.34328216, 0.39724907, -0.19968817, -0.0735909 , -0.11611477, 0.2887974 , 0.27589712, -0.19556114, 0.00310108, 0.05990869, -0.06629719, 0.11465285, 0.30233756, -0.4840145 , 0.36792257, 0.01198539, 0.17746153, 0.00102492, -0.73527193, -0.04222427, 0.1278375 , -0.04002868, -0.04392892, 0.07372226, 0.00746391, 0.01089494, -0.06465401, -0.01894091, -0.04520243, -0.05135101, 0.10821903, -0.36141765, 0.01786461, -0.00758973, 0.20370121, 0.07571532, -0.04988701, -0.50066465, -0.29137757, 0.0468887 , -0.0369802 , -0.28495568, -0.15369293, -0.04906723, -0.05060489, 0.23548573, 0.02296117, 0.29195184, -0.4791245 , -0.32274112, 0.00163329, 0.6024667 , -0.02721317, 0.4371868 , -0.45318347, 0.15298416, 0.53022355, 0.23577152, -0.11320106, -0.05042947, 0.00792542, 0.34630367, 0.45112005, -0.17176089, -0.30412972, -0.00977058, 0.3797044 , -0.55427474, -0.14912651, -0.31753495, -0.14334434, -0.12312431, 0.29336822, 0.08129798, -0.09826545, -0.37803188, -0.03996738, -0.565557 , -0.15023306, -0.17836387, -0.00129342, 0.31617662, -0.40511948, -0.0246349 , 0.8049514 , 0.50346047, -0.08232877, 0.7958381 , 0.11525708, -0.87054974, 0.02044921, 0.10254359, -0.41025582, -0.11836894, 0.3934775 , -0.2755682 , 0.28480217, 0.28756127, -0.06242 , -0.36147273, -0.31789255, -0.1169754 , 0.13862565, 0.5221684 ], dtype=float32), kernel: Array([[-0.03366861, 0.1951855 , 0.13829306, ..., 0.20226802, 0.14245163, 0.0851718 ], [-0.12275442, 0.09568106, 0.1026165 , ..., -0.11756799, -0.03399425, 0.1083973 ], [-0.11897923, 0.44871688, 0.22821409, ..., -0.19063285, 0.14112021, 0.05127604], ..., [ 0.42162004, -0.30832654, -0.31275368, ..., -0.48211238, 0.221526 , 0.17857978], [ 0.23314169, 0.38464716, 0.37320703, ..., 0.7464507 , 0.65437895, 0.09728524], [-0.02876819, -0.11770961, -0.18619719, ..., -0.4305184 , -0.24435337, -0.0622173 ]], dtype=float32), }, value: { bias: Array([ 3.80855761e-02, 1.19626271e-02, 9.54822153e-02, 3.39327864e-02, 6.47817999e-02, 1.12312162e-04, 2.36410853e-02, 1.12216197e-01, 1.71483278e-01, -4.48498651e-02, -6.95201615e-03, -5.79331703e-02, 1.67439714e-01, -1.31781459e-01, 1.07828841e-01, -1.07748993e-01, -2.16874084e-03, -2.65070498e-02, -1.59811676e-02, 7.09029958e-02, 5.01474105e-02, -4.89775911e-02, 2.54220992e-01, -4.80323769e-02, -1.79449767e-02, -5.27789593e-02, -4.24154028e-02, -1.70985550e-01, -1.36494562e-01, 1.54837176e-01, -1.15875848e-01, -1.71242371e-01, 5.86349219e-02, 1.91222817e-01, -2.35057212e-02, -8.43578875e-02, -4.65269759e-02, -8.38292018e-03, -4.65860590e-02, -5.02779474e-03, 4.50352840e-02, -2.31239274e-02, 5.93480729e-02, 5.71514890e-02, 5.99062406e-02, 2.83261593e-02, -4.82074209e-02, -4.80451956e-02, -6.26677051e-02, 5.32909296e-02, 6.67422032e-03, 4.72835749e-02, 4.04058099e-02, -4.40718755e-02, -4.50864471e-02, 1.62461493e-02, -5.04599810e-02, -3.60567644e-02, 4.91038598e-02, -7.88058713e-03, 1.09715499e-01, 5.14943898e-02, 3.80876176e-02, 5.21656685e-02, 1.56386979e-02, 6.83846846e-02, 7.60920197e-02, -8.33767503e-02, -1.77296321e-03, -9.03388578e-03, -5.15193716e-02, 2.69258358e-02, 4.77294379e-04, -2.96406243e-02, -8.59358441e-03, -2.96034720e-02, -2.67447811e-02, -6.73482614e-03, 1.27622047e-02, -2.17594709e-02, 8.59617069e-02, 1.03174895e-01, 2.63699442e-02, -1.10600488e-02, -7.31578097e-02, -3.59972566e-02, 4.40209918e-02, -4.44998406e-02, -1.85169447e-02, -4.11919132e-02, -3.58521268e-02, 1.47900637e-03, 4.30759192e-02, 2.79228836e-02, 7.82085881e-02, 3.90527584e-02, 3.11766323e-02, 2.85879765e-02, -8.20098892e-02, -7.59413168e-02, -6.41580895e-02, 2.49105208e-02, -1.47475805e-02, -9.27800685e-02, -1.73971094e-02, 8.51288959e-02, -2.54925918e-02, 4.36546579e-02, -9.24658105e-02, -1.01948939e-02, -4.81284559e-02, -3.62936854e-02, 2.90373899e-02, -4.27892022e-02, 3.08775119e-02, 5.98094938e-03, 2.10134909e-02, 4.55508381e-02, 3.65342051e-02, 1.74732562e-02, 1.06893666e-02, -1.42213493e-03, -2.61030179e-02, 2.22432707e-02, 1.47231504e-01, 6.33382201e-02, 2.16640271e-02, 4.13991958e-02], dtype=float32), kernel: Array([[ 0.00939619, 0.02482712, 0.01164271, ..., 0.0164959 , 0.00769011, -0.03742562], [ 0.18655218, -0.36816835, -0.08651444, ..., 0.2512268 , 0.02853749, 0.00129432], [ 0.09724977, -0.17311321, -0.15452595, ..., -0.4560767 , -0.48346895, -0.4558498 ], ..., [-0.06453069, 0.4239059 , 0.45162615, ..., -0.27700382, 0.13836603, -0.33547392], [-0.02759907, -0.9605103 , -0.82787424, ..., -0.51210946, 0.33198425, 1.0528204 ], [-0.12098552, -0.10120461, -0.10430597, ..., 0.78033483, -0.08541391, 0.05855659]], dtype=float32), }, }, output: { dense: { bias: Array([-1.72631651e-01, 2.81819075e-01, -1.31835248e-02, -6.50164206e-03, -9.19895153e-03, -6.00125082e-02, 4.56261076e-02, -1.17319543e-03, 3.18587162e-02, 2.30925724e-01, -2.78387107e-02, -7.64354365e-03, -7.98858032e-02, 2.95766089e-02, -4.09963764e-02, 1.03628486e-01, 5.56401908e-02, -8.70198682e-02, -1.29737146e-02, 1.11193627e-01, -5.96015491e-02, -5.83102107e-02, -2.50512566e-02, -3.02468054e-02, -3.29878554e-02, 4.26750369e-02, -3.15967947e-02, 1.79770100e-03, 6.93126023e-02, 8.52573961e-02, 4.88307029e-02, -6.38186038e-02, 1.12157442e-01, -9.84508917e-02, 5.63154779e-02, -7.03682825e-02, 2.64801327e-02, -1.89797301e-02, 6.20345166e-03, -8.26654360e-02, 2.39220913e-02, -9.58621129e-03, -8.39129508e-01, 4.56922092e-02, -1.53480023e-01, 6.83549270e-02, 7.20679983e-02, 2.08752692e-01, 4.47404385e-02, -5.50162159e-02, -5.08189276e-02, 5.22453226e-02, -2.34786086e-02, 1.55734494e-01, -4.19281684e-02, -1.19545199e-02, -8.40202495e-02, -8.96746367e-02, 6.41859993e-02, 7.71264508e-02, 8.69383588e-02, -1.31521776e-01, -2.94522438e-02, 4.01933566e-02, -1.60081685e-03, -3.63952629e-02, -5.23505732e-02, 8.07748511e-02, 3.52987312e-02, 8.45032781e-02, -1.18315943e-01, 3.48625183e-02, 5.90237789e-02, -7.91934412e-03, 8.07135582e-01, -6.57365983e-03, -6.89911172e-02, 1.08262442e-01, -1.27490476e-01, 1.97968110e-02, -6.29805326e-01, 5.13264053e-02, 3.17039192e-02, 1.63930492e-03, 2.53316425e-02, 1.45042747e-01, 1.39836982e-01, 1.34169623e-01, 4.38121334e-02, -1.00197628e-01, 3.51289548e-02, 3.51637453e-02, -1.21273242e-01, -1.04602434e-01, -9.42274835e-03, -2.06037387e-02, -4.44292687e-02, -2.39612963e-02, -1.44756138e-02, 9.55058485e-02, 3.27826925e-02, 5.12430328e-04, 1.59999877e-02, -6.98894709e-02, 1.01640105e-01, -5.48832715e-02, 6.11420302e-03, -7.91232064e-02, -1.24512471e-01, -3.10322326e-02, -1.39527261e-01, 3.97435911e-02, 1.18619666e-01, -1.12082511e-01, 1.25646042e-02, 1.72376752e-01, 4.17521298e-02, -2.35856101e-02, -3.09479032e-02, 6.41909391e-02, -2.64365948e-03, 2.64008511e-02, 1.21359736e-01, -4.78715599e-02, 4.74941581e-02, 5.94909228e-02, -6.63625225e-02, -2.96208411e-02], dtype=float32), kernel: Array([[ 0.30733457, 0.11394037, -0.11875194, ..., -0.02600583, 0.26162902, -0.24450323], [ 0.32579228, -0.01019196, 0.08372336, ..., -0.23464818, 0.77629703, -0.14200607], [ 0.40005246, -0.09073471, 0.01217228, ..., 0.12469898, 0.4625953 , -0.2634586 ], ..., [-0.14610979, 0.26803997, -0.26479524, ..., -0.581134 , 0.39575958, -1.1754091 ], [-0.10739683, -0.4411798 , 0.68852794, ..., -0.01546267, 0.06261325, -0.04542963], [ 0.20287202, 0.13014646, 1.1704878 , ..., -0.26887265, -0.26802245, 0.629474 ]], dtype=float32), }, }, }, layernorm_after: { bias: Array([ 1.71948429e-02, -1.09049575e-02, 6.19933335e-03, -3.97290401e-02, -1.75763867e-04, 5.07760718e-02, 1.12305544e-02, -2.87154932e-02, -2.46254611e-03, -1.75729260e-01, 9.95321851e-03, -1.72805071e-01, -1.28897009e-02, 2.74409372e-02, -1.33280344e-02, 1.57673303e-02, -2.70904624e-03, 1.06892176e-03, 2.89133284e-03, 1.36673786e-02, 4.77137081e-02, -9.59468889e-06, 6.27165893e-03, -6.68704743e-03, 1.20838732e-02, 1.39770750e-02, -3.87260579e-02, -5.88312745e-03, 4.62619169e-03, 6.23012260e-02, 8.07429329e-02, 7.94423558e-03, -2.50669499e-03, 1.25099311e-03, -6.49371138e-03, -9.59886145e-03, 3.83810289e-02, 7.55729433e-03, 4.41730954e-03, 1.75167769e-02, 3.34830210e-02, 9.28634033e-03, 1.21751567e-02, -1.52964927e-02, -2.70186807e-04, 7.17550702e-03, 8.44905898e-02, -9.36958939e-02, -1.57131581e-06, 4.74082539e-03, -8.86013955e-02, 7.37810042e-03, -4.76047629e-03, 6.89025401e-05, -9.31238383e-03, 6.64415536e-03, 7.76583795e-03, 2.29338557e-02, -1.50072891e-02, -3.97083163e-03, 5.64871766e-02, 1.86805464e-02, 5.85068855e-03, 3.33538577e-02, 3.15117603e-03, 3.10075725e-03, -3.19483913e-02, 3.20728831e-02, 2.38816179e-02, 1.34205222e-02, -1.76607758e-01, -5.26308222e-03, 2.82549043e-03, -1.20671205e-01, -1.07251152e-01, 3.50134517e-03, 2.33329542e-04, -1.22928575e-01, -5.50287850e-02, -2.40997149e-04, 4.36784141e-03, -3.80961671e-02, -1.24475360e-01, -1.74865853e-02, 9.07492265e-02, -1.61794608e-03, 4.57073562e-03, -6.20833039e-03, -5.41980378e-03, 2.80724547e-04, -4.64204960e-02, -4.63194735e-02, 1.02222068e-02, -5.72193079e-02, -3.21766338e-03, 6.88658096e-03, -4.31186426e-03, -3.94533984e-02, -1.81648252e-03, 5.71832731e-02, -6.70271181e-03, 5.93083352e-03, -8.59189108e-02, -5.01573570e-02, -3.59455287e-03, -8.11670348e-02, -3.07016950e-02, 2.01600092e-03, 4.20039892e-02, -4.99186059e-03, -1.39956048e-03, -5.72945038e-03, 1.23080192e-03, 1.21427193e-01, 1.27224373e-02, 5.32536209e-02, -1.02717979e-02, -3.96808702e-03, 3.51909577e-04, 1.07945967e-02, -4.73669693e-02, -5.23439702e-03, -3.12553300e-03, 6.25749454e-02, -9.56553966e-03, -9.94754490e-03, 2.23868527e-02, -2.48279348e-02], dtype=float32), scale: Array([ 4.24516723e-02, 1.21176187e-02, -4.84997500e-03, 5.57146311e-01, 2.28432156e-02, 6.31359816e-01, -4.46775509e-03, 6.18052892e-02, -4.33097221e-03, 2.61805564e-01, -1.88892316e-02, 6.95380867e-01, -6.15348248e-03, 5.73870949e-02, 1.55512363e-01, -5.18870428e-02, 1.60957292e-01, -1.62363220e-02, 4.54635024e-01, -1.21937275e-01, 4.06006753e-01, 4.14683282e-01, 3.15475985e-02, 4.67635952e-02, 1.88465670e-01, -1.32318744e-02, 4.83835369e-01, 8.42766091e-02, -7.77654797e-02, 5.02031922e-01, 4.80292380e-01, 3.83823998e-02, -4.09559421e-02, 1.13746740e-01, -1.26264850e-02, 7.38768047e-03, 3.53148162e-01, -1.56045742e-02, 2.96747178e-01, 1.84800133e-01, 2.40674987e-01, -5.23572266e-02, 1.17433036e-03, 6.38127849e-02, 2.33978732e-03, -3.27690654e-02, 4.07037109e-01, 1.53259337e-01, 3.08140907e-02, -5.92192961e-03, 3.31769854e-01, 3.03894043e-01, -1.66736450e-02, -4.60295379e-02, 3.50147933e-01, 4.64291312e-02, 7.68471509e-03, 9.69270691e-02, 1.85923763e-02, -9.66442749e-03, 4.18877095e-01, 1.00185797e-01, 1.32984370e-01, 4.85280842e-01, -7.05129327e-03, 4.97093098e-03, 3.28959554e-01, 9.48822647e-02, 2.17689544e-01, 2.05316558e-01, 5.99317133e-01, 3.90748940e-02, 5.26148966e-03, 3.09464663e-01, 3.66784409e-02, 4.52369526e-02, -4.48510870e-02, 2.84545600e-01, 3.67504448e-01, 4.94225929e-03, -5.27279684e-03, 8.71935904e-01, 4.51588362e-01, 2.58467104e-02, 3.65057826e-01, 6.95686322e-03, -1.69470385e-02, 1.21857105e-02, -1.65910535e-02, -8.54126003e-04, 5.98546147e-01, 3.60920489e-01, 2.29799710e-02, 3.24622720e-01, -8.44317749e-02, -2.27606995e-03, 3.23808223e-01, 2.70338178e-01, 5.69886751e-02, 1.97269484e-01, 3.14833894e-02, -3.49436514e-02, 3.14999282e-01, 5.10905981e-01, 2.21107565e-02, 3.05217803e-01, 2.38849521e-01, -1.36028053e-02, 4.52475280e-01, 2.09674463e-02, -2.62735710e-02, -6.97454140e-02, -8.80253036e-03, 4.10522908e-01, -2.87996326e-03, 2.52809376e-01, -1.01799294e-02, 1.55074904e-02, 4.01195139e-02, 1.15418859e-01, 1.29438698e-01, -9.92912054e-03, 3.01613122e-01, 9.44039226e-02, -3.23632099e-02, 1.09247090e-02, 5.09600282e-01, 6.76498190e-02], dtype=float32), }, layernorm_before: { bias: Array([ 2.0053135e-02, -2.0385702e-01, -5.2851941e-02, -1.5826782e-02, -3.9745602e-03, -5.1497813e-02, 4.4322563e-03, -2.0497143e-02, 3.5912704e-02, -1.5449695e-01, -6.0098185e-03, 4.4156699e-03, -5.7948790e-03, 1.7218473e-03, 8.6186066e-02, -8.2310978e-03, 8.4572081e-03, 3.2470144e-02, -7.2876806e-04, -7.4659437e-02, -1.4426826e-02, -6.7664213e-03, 1.5026542e-02, 4.0727290e-03, 2.3569600e-04, -9.6417527e-04, -1.1503830e-03, -1.6481975e-02, -7.0209317e-03, 5.3436416e-03, 6.8159692e-02, -2.5559873e-03, -8.0474456e-05, -3.0456469e-03, -3.3534174e-03, 6.1106024e-04, -6.6513027e-04, -1.2051529e-01, -4.2777561e-02, 6.5030590e-02, 2.1362038e-02, 3.2946058e-02, 2.9181463e-01, 1.6087948e-03, 8.0183737e-02, 6.0054217e-04, -1.5031694e-03, -1.3727798e-01, 2.0673402e-02, 1.6634881e-02, -6.1021522e-02, -9.7963065e-03, -4.4847801e-03, -1.6810349e-03, -6.0316608e-03, -1.3532469e-04, 8.6372629e-02, -1.0683674e-02, -1.5961612e-02, 4.4840686e-02, 5.4682657e-02, 1.3794379e-02, 2.7690900e-03, 3.4621611e-02, 3.8512382e-03, 1.3362155e-03, 2.2860983e-02, 1.7252501e-02, 7.1021970e-03, -1.8353600e-02, -9.1281412e-03, -2.5052501e-02, 6.4535923e-02, -1.1188607e-01, -7.4342960e-01, 9.8237805e-03, -4.8992522e-02, -3.0361801e-02, 4.7444657e-05, -1.2857079e-02, -3.5183441e-03, -2.8601723e-02, -4.8774560e-03, -4.9416926e-02, 2.1382656e-03, -1.7278522e-02, 4.1791960e-03, -6.8051869e-04, 5.9474696e-04, 1.3638437e-01, -9.9423109e-03, 1.4076917e-03, -2.9004067e-03, 1.2549493e-02, 1.8330464e-02, 9.2371933e-02, -8.4709952e-04, -1.5642725e-02, 1.8227950e-02, 4.0236771e-02, -4.6799608e-02, -6.4027351e-03, -1.0166033e-01, 2.3036268e-02, -2.3710703e-02, -2.6576657e-02, 4.9998323e-03, 2.3283776e-02, 6.9294289e-02, -2.0691980e-02, 4.1049831e-02, 1.8761930e-04, -1.0511643e-01, 8.2109414e-02, -1.6860457e-02, -3.8362190e-02, 4.8856564e-02, -2.3737226e-02, -1.6952256e-02, 1.5742332e-02, -1.9643296e-02, 7.6947906e-03, 1.0548260e-03, 5.7553727e-02, 1.7045023e-02, -3.2199997e-02, 7.6176226e-02, -7.1644776e-02], dtype=float32), scale: Array([ 1.36227664e-02, 1.40898958e-01, 1.92645520e-01, 4.70757820e-02, -2.37463228e-03, 3.57880324e-01, 1.77441984e-01, 2.04336628e-01, 5.04735708e-02, 2.16793969e-01, 1.84319422e-01, 1.62005424e-01, -2.82178313e-04, 3.39963939e-03, 3.45121980e-01, 3.06478173e-01, 4.18081768e-02, 7.82519281e-02, -1.85935143e-02, 2.79815465e-01, 2.39701569e-01, 2.40149871e-02, 9.43208337e-02, 1.05237709e-02, 9.96645987e-02, 5.39937872e-04, -1.31208654e-02, 1.39623404e-01, -3.63838188e-02, 3.99398878e-02, 3.38648826e-01, 9.40667465e-02, 1.93201587e-01, -4.52386551e-02, 3.65994535e-02, 3.85606177e-02, -3.78566347e-02, 2.66506523e-01, 2.40695551e-01, 3.16709459e-01, 8.35959762e-02, 2.61536777e-01, 1.21971466e-01, 1.21512767e-02, 3.52942318e-01, 2.85213944e-02, -1.15962522e-02, 2.87642092e-01, 2.67458588e-01, 3.02993476e-01, 1.73773259e-01, 1.12724066e-01, -8.12414661e-03, 2.45114770e-02, 1.12504624e-01, 2.88635880e-01, 1.51033133e-01, -3.74849811e-02, 3.15434486e-01, 3.69797438e-01, 3.93722087e-01, 1.04381077e-01, 1.80823863e-01, 2.28892982e-01, 2.66926765e-01, -5.73799293e-03, 1.90365806e-01, 2.51159281e-01, 2.34797284e-01, 2.54733443e-01, 1.83505401e-01, 1.44614145e-01, 2.63359755e-01, 4.59246188e-01, 2.01807499e-01, 2.74832934e-01, 2.06239209e-01, 1.09179243e-01, 6.59588631e-03, 2.21557349e-01, -5.72106568e-03, 2.62187541e-01, 5.55602312e-02, 2.09347799e-01, 6.81416085e-03, 6.24816194e-02, -1.57044847e-02, 5.38076796e-02, -1.21786725e-03, 3.13683510e-01, 7.88503736e-02, 1.91961545e-02, -7.89848214e-04, 2.04859436e-01, 1.77144408e-01, 1.27074376e-01, -2.01367047e-02, 1.25420019e-01, 6.45905286e-02, 2.66942471e-01, 3.49235058e-01, 2.52543539e-01, 3.24409813e-01, 2.59198427e-01, 3.92820060e-01, 1.41132906e-01, 9.48882196e-03, 1.74179897e-01, 5.39034963e-01, 1.58920869e-01, 2.73100972e-01, 1.81179568e-02, 1.68145299e-01, 3.43630970e-01, 1.51725076e-02, 2.98200041e-01, 2.57570058e-01, 2.81464636e-01, 2.47943982e-01, 1.76195994e-01, 5.23593724e-02, 1.48531556e-01, 2.94840173e-03, 2.51852840e-01, -7.62507394e-02, 1.64415836e-01, 4.18539464e-01, 1.66102067e-01], dtype=float32), }, moe: { shared_experts: { intermediate: { bias: Array([-8.55890103e-03, 3.67313088e-03, -4.36119083e-03, -8.68369080e-03, -1.34754460e-03, -6.52316585e-03, -5.01648569e-03, -8.97654798e-03, 3.13179940e-03, -7.83655513e-03, 2.37924187e-03, -5.66111412e-03, -1.27145583e-02, -8.22409242e-03, 2.74701975e-03, -4.37194854e-03, -4.04660078e-03, -3.13023920e-03, -1.81288610e-03, -9.03141592e-03, -1.66141614e-03, -5.17393708e-01, -1.02338977e-02, 3.56379431e-04, -7.41085410e-03, 9.37872624e-04, -2.64340243e-03, 5.81945898e-03, -1.19625553e-02, -4.65264637e-03, -6.52810233e-03, -7.84709677e-03, 1.05619885e-03, -9.87111218e-03, 4.54840425e-04, -5.49622951e-03, 7.56876124e-03, -5.51949022e-03, -3.83866858e-03, -1.84899215e-02, 3.52073257e-04, -2.78544938e-03, -5.13466494e-03, -4.08213615e-01, -5.55376470e-01, -1.75705249e-03, 4.89682052e-03, 1.53476396e-03, 1.07086054e-03, -3.85978585e-03, 9.40655766e-04, 6.59122784e-03, -3.99606815e-03, -6.94369990e-03, -2.59699044e-03, 7.18095253e-05, -8.83713830e-03, -1.96340843e-03, -2.48700310e-03, -9.87609662e-03, -6.28701295e-04, -7.88388960e-03, 7.09942728e-03, 9.48800612e-03, -1.95277121e-03, -1.22706648e-02, 2.85065360e-03, 9.00720805e-03, -7.02000320e-01, 4.25555557e-03, -1.28851901e-03, 2.98675569e-03, -3.15172016e-03, -1.08107440e-02, -8.73899739e-03, -1.31636495e-02, -9.91051551e-04, -1.34902084e-02, 3.62220523e-03, 3.06941615e-03, -1.32400328e-02, -2.95568615e-01, 7.01408414e-03, -5.55180886e-04, 3.73574020e-03, -7.83216953e-03, 7.60261260e-04, -9.49840934e-04, -1.34194596e-02, -1.67394914e-02, -8.75537097e-03, -1.25364307e-03, -6.53209072e-03, -1.00438660e-02, -4.49657347e-03, -1.30805969e-02, 7.16269715e-03, -1.18865566e-02, 1.15038871e-04, -7.90204573e-03, -1.13492664e-02, -1.28111634e-02, -2.23936560e-03, -5.98883303e-03, -1.25457710e-02, -1.07937502e-02, -3.34314862e-03, -7.55358534e-03, -6.86253095e-03, -2.35536560e-01, -9.40762553e-03, 1.64968718e-03, -2.74949777e-03, -3.80960264e-05, 1.37860198e-02, 1.36729311e-02, -1.17542176e-03, -4.75597620e-01, 1.06082670e-02, -4.49829688e-03, 1.80422806e-03, -3.01245158e-03, -5.77643573e-01, -8.06397665e-03, 1.37929013e-03, -9.44454595e-03, -2.07981328e-03, 9.04347980e-05, -5.42783411e-03, 1.13049364e-02, -6.55044988e-03, -9.25507303e-03, -1.70839131e-02, -3.13228741e-03, -1.04950890e-02, -9.20292269e-03, -4.24578832e-03, -8.48777406e-03, -1.27782347e-02, -7.17586139e-03, -5.02940093e-04, -6.85630506e-03, -5.49637014e-03, 5.60814887e-03, -7.42090074e-03, -1.33384299e-02, -4.22688900e-03, -3.86846159e-03, -1.31040469e-01, -5.72234035e-01, 1.44373742e-03, -2.45060981e-03, -5.27091743e-03, -1.61155630e-02, -8.04609526e-03, -3.71574331e-03, 4.77314461e-03, -2.98696361e-03, -1.80231046e-03, -3.76312784e-03, 2.70129368e-03, -5.70825338e-01, -1.65502690e-02, -6.61239505e-01, -9.16241575e-03, -1.41646145e-02, -5.00553148e-03, -4.18055616e-03, -4.17894637e-03, -1.43527528e-02, 1.20390498e-03, 1.38255325e-03, -2.05959729e-03, -1.65631052e-03, 8.46141134e-04, -4.89455275e-03, -3.43874632e-03, -4.29662468e-04, -9.45413113e-03, -4.12517926e-04, -2.43844930e-03, -8.26839264e-03, -6.06988370e-03, -1.55913727e-02, -6.00419287e-03, 7.41206226e-04, -4.03943751e-03, -8.32396373e-03, -1.77192141e-03, -8.27328023e-03, -4.80760727e-03, -1.19567281e-02, -3.16868187e-04, -1.15141738e-02, 7.74921337e-03, -2.20119534e-03, 1.03418762e-03, -5.75479446e-03, -4.55250964e-03, -1.04654999e-02, -6.14567241e-03, -1.54978642e-02, -7.39947148e-03, -1.07835466e-02, -1.21487593e-02, -6.76209899e-03, 1.25275925e-03, -3.19723099e-01, -3.41119803e-03, 2.97757820e-03, -5.52376360e-03, -9.89249256e-03, -4.31803286e-01, -1.36245508e-02, -7.62469275e-03, -1.74430560e-03, -1.96244684e-03, 5.65344701e-04, -6.50570095e-01, -1.45257404e-03, -2.76738009e-03, -8.20099376e-03, 2.03965130e-04, -1.13646872e-02, -2.37802067e-03, -5.39474888e-03, 2.26747943e-04, -2.44784285e-03, -1.09350355e-02, 8.08892888e-04, -8.32007360e-03, -1.91815558e-03, -9.69145447e-03, 1.22545648e-03, 4.08804649e-03, -5.84384939e-03, 3.66018945e-03, 4.44652513e-03, -9.09210462e-03, -4.17519035e-03, -3.98625523e-01, -3.37073533e-03, -8.35657492e-03, -4.79207002e-03, -1.39846019e-02, -8.26936215e-03, -8.90700985e-03, -2.52558112e-01, 3.46473581e-03, -1.71433843e-03, -1.91845442e-03, -3.26462858e-03, -8.05919431e-03, -5.96231222e-03, -2.60497467e-03, -5.12533821e-03, -9.94778052e-03, -1.35790939e-02, -7.90751539e-03, -5.66528225e-03, 2.63936934e-03, -4.14405525e-01, 2.22919116e-04, 8.12191633e-04, -1.82921547e-04, -7.25358841e-04, -1.08812936e-02, 3.49700474e-03, -1.17458524e-02, -1.99729693e-03, -5.74411824e-03, -1.04112234e-02, -5.87427756e-03, 5.42524364e-03, -4.66417568e-03, -3.61002120e-03, -4.92561430e-01, -5.91339357e-03, -8.21670890e-03, 6.48486370e-04, -1.16507420e-02, -7.28300912e-03, 9.69227776e-03, -1.40586449e-03, 1.34252605e-03, -4.55454225e-03, -1.05377687e-02, 8.61465523e-04, -1.29449442e-02, 3.23367422e-03, -1.80069392e-03, 2.35577201e-04, -8.91069509e-03, -1.41903665e-03, -8.80321302e-03, -4.76717623e-03, -2.86917109e-03, -3.21599003e-03, 3.61907203e-03, -7.99222197e-03, -6.45599188e-03, -5.55047020e-03, -9.93613340e-03, -8.86950456e-03, 1.25965700e-02, 3.33399116e-03, -1.01383878e-02, 2.34254939e-03, 5.69945900e-03, -5.75918052e-03, 7.65805482e-04, -1.72046013e-02, -8.67277035e-04, -1.03675043e-02, -7.15132989e-03, -1.15488358e-02, -2.48059118e-03, -1.51636070e-02, -1.34787019e-02, -8.78769148e-04, -6.11743808e-01, -6.02744520e-04, 7.56409660e-04, -3.29326536e-03, 5.58212679e-03, -9.33979824e-03, -2.45822361e-03, -5.48888650e-03, -4.64835018e-03, -4.38324921e-03, -3.24640493e-03, -6.00662351e-01, 1.93342322e-03, -1.04327723e-02, 4.34050598e-04, 3.05471686e-03, -7.74156442e-03, -1.68287978e-02, -1.33079069e-04, -3.91604519e-03, -1.49135012e-02, -8.73885211e-03, -5.66676259e-03, 3.75113130e-04, -5.36525948e-03, -1.39129013e-02, 1.88402925e-03, -6.80929542e-01, 2.44663190e-03, -1.16351806e-02, -5.38230175e-03, -3.69915483e-03, -4.62271646e-03, 4.26167302e-04, 1.99855946e-04, -1.86087582e-02, 7.48794526e-03, -1.61632244e-02, -4.98278052e-01, -8.40996485e-03, -6.01133797e-03, -8.28853925e-04, -5.63414209e-03, 5.07016899e-03, -9.90651455e-03, -4.54475218e-03, 1.22021348e-03, 5.08222263e-03, -5.74093172e-03, 2.58398801e-03, -5.80827938e-03, 2.76929396e-03, -8.52370542e-03, -8.13993160e-03, -3.10428627e-03, 9.73783247e-03, -2.26997398e-03, -7.61909829e-03, 4.68078069e-03, -3.60694993e-03, -8.70567188e-03, -1.44884512e-02, -2.72786571e-03, 4.09194687e-03, -9.40709934e-03, 4.71121492e-03, -5.04164118e-03, -1.17910588e-02, 1.36497710e-03, -1.07826991e-02, -1.53774535e-03, -1.12592215e-02, -2.51332531e-03, -1.03800860e-03, 9.40567255e-03, -1.59978624e-02, -9.16326977e-03, 5.42400870e-04, -2.22451333e-03, -7.41019659e-03, -3.90202254e-01, -1.51460301e-02, -6.29314408e-03, -4.74618049e-03, 3.59613076e-03, -1.17005240e-02, -1.27715422e-02, -1.15625896e-02, -1.68063808e-02, -1.12238340e-02, -5.49799949e-03, 7.02496106e-03, -2.54716747e-03, -9.75850690e-03, -3.14950384e-03, -7.65251042e-03, -1.46619789e-03, 1.40559685e-03, -2.16309223e-02, 4.46918281e-03, -8.65491666e-03, 5.14270552e-03, 3.51362582e-03, -9.26864985e-03, -1.17338272e-02, -2.89736595e-03, -1.97741878e-03, -5.52454265e-03, -8.56413040e-03, -9.34949901e-04, 2.97652930e-03, -2.00355076e-03, -3.76361725e-03, 1.10566849e-03, -5.63339936e-03, -2.81627057e-03, -1.68547058e-03, -4.57938435e-03, 6.58686552e-03, -4.55372721e-01, -5.61221363e-03, -6.51222235e-03, 1.07192027e-05, -8.45061056e-03, 5.90987783e-03, -5.17501831e-01, -4.53207910e-01, 1.17717020e-03, -7.01933773e-03, 4.97156382e-03, 1.22358715e-02, -3.75395943e-03, -1.19871208e-02, 7.76410243e-03, -1.40578160e-02, 3.29062645e-03, 6.30691764e-04, -1.67252067e-02, 1.95841817e-03, -9.67236888e-03, -1.17397988e-02, -6.99722324e-04, -6.32376294e-04, -6.49616960e-03, -2.94766188e-01, 1.18942198e-03, -7.66102644e-03, 3.43931909e-03, 2.72910809e-03, -5.50179362e-01, -1.15189431e-02, -5.44976583e-03, -3.17149970e-05, 1.41578354e-03, -6.40626810e-03, -3.47646652e-03, 6.37558231e-04, -2.99780950e-04, 5.98926889e-03, -4.70549008e-03, -7.91283231e-03, -9.49787791e-04, -3.55603476e-03, -1.14685595e-02, -1.00854738e-03, 2.23665382e-03, -2.58625904e-03, -1.55551946e-02, -9.62937716e-03, -3.64469341e-03, -6.78142011e-01, -3.52118816e-03, -1.36016486e-02, -1.84007958e-01, -5.86909475e-04, -8.81857052e-03, -1.06971972e-02, -7.59707938e-04, -5.68926990e-01, -5.22532880e-01, -2.09414326e-02, -1.72124021e-02, -6.04376057e-03, -3.09983082e-03, -4.42491425e-03, -8.42403620e-03, 7.68742245e-03, -3.21401260e-03, 6.66661141e-03, 2.13651289e-03, -4.34801867e-03, -7.12090218e-03], dtype=float32), kernel: Array([[-0.06747823, 0.12502806, 0.01688226, ..., 0.04402845, 0.0489017 , 0.02632801], [-0.01445617, 0.03798477, -0.00418379, ..., 0.01224544, 0.02048267, 0.01027028], [-0.01829829, 0.01891246, 0.01610937, ..., 0.03225997, -0.01639963, -0.03890606], ..., [ 0.00887221, -0.00487363, 0.02399558, ..., 0.01346108, 0.01049829, -0.03240183], [-0.01674049, -0.00387911, -0.03074075, ..., 0.00146568, -0.00876305, -0.02836148], [ 0.01625663, 0.03327096, -0.04843638, ..., 0.08134065, 0.03410971, 0.01957767]], dtype=float32), }, output: { bias: Array([-0.16341606, 0.2722003 , -0.01811248, 0.01928756, -0.02578784, -0.10320964, 0.0457683 , 0.00972074, 0.05574008, 0.24268645, -0.02999464, 0.043408 , -0.0737564 , 0.03339861, -0.04492559, 0.09947823, 0.05016089, -0.08309133, -0.05127812, 0.11153366, -0.08190622, -0.06840179, -0.02421227, -0.02895692, -0.04004822, 0.03941108, -0.0170032 , -0.00097349, 0.07023378, 0.02564083, 0.03406862, -0.06778747, 0.11675587, -0.08771975, 0.04982556, -0.07175723, -0.04058652, -0.01989682, -0.02999596, -0.07330333, 0.03119163, -0.0147863 , -0.83884764, 0.04124971, -0.147959 , 0.06891607, 0.05300924, 0.2076162 , 0.04646805, -0.05764642, -0.03448724, 0.05170846, -0.02983713, 0.14581041, -0.04477751, -0.0028539 , -0.08908532, -0.09743858, 0.05685575, 0.08213867, 0.0757231 , -0.13601153, -0.02382083, -0.01865633, -0.00175037, -0.03086867, -0.03948168, 0.06656119, 0.03003574, 0.07914743, -0.02571616, 0.03044318, 0.05824798, 0.02247159, 0.7948062 , -0.00407046, -0.07054033, 0.12808433, -0.08461454, 0.02311847, -0.65620077, 0.10964675, 0.10405939, -0.00352513, -0.02670519, 0.15050422, 0.13908558, 0.14308207, 0.03576402, -0.11146745, 0.07615335, 0.04982926, -0.11778215, -0.10201599, -0.01381564, -0.01693749, -0.03509882, 0.01073222, -0.01447865, 0.08480705, 0.02910439, 0.00359502, 0.01869445, -0.05797101, 0.09884419, 0.0161234 , 0.00668302, -0.08439763, -0.13141344, -0.03596077, -0.14647698, 0.03868678, 0.11478355, -0.14481665, 0.00641343, 0.16066928, 0.04811645, -0.02036389, -0.03384693, 0.06242058, -0.00735435, 0.02676932, 0.12175596, -0.06479023, 0.04808705, 0.05643554, -0.07896087, -0.02733074], dtype=float32), kernel: Array([[ 0.04431782, -0.07884661, -0.02030264, ..., 0.03209 , 0.02117256, 0.00271614], [-0.05527279, 0.03204606, -0.00293706, ..., -0.03798404, -0.02230972, 0.00913886], [-0.01904109, -0.02136857, 0.00946268, ..., 0.00186177, -0.02516458, 0.05516352], ..., [-0.03168841, 0.07299381, -0.00359565, ..., -0.01263119, -0.04168412, 0.01372963], [-0.02394411, 0.08588913, 0.00119561, ..., 0.0126469 , -0.04365737, 0.0029881 ], [-0.01780676, 0.05708533, 0.01049639, ..., 0.03207007, -0.03454534, 0.0507676 ]], dtype=float32), }, }, }, }, 4: { attention: { attention: { key: { bias: Array([ 0.25172058, 0.68627775, 0.5949247 , 0.1658349 , -0.3173467 , 0.1774752 , 0.25412992, 0.08010545, 0.56002 , 0.17958656, -0.17482378, 0.07066374, 0.53656256, -0.5385957 , 0.03753721, 0.3764516 , -0.05485516, 0.7278333 , -0.7301842 , 0.7662722 , 0.8557523 , 0.19676152, -0.7459649 , 0.11304809, -0.27465415, 0.41445032, 0.31016302, -0.9027352 , 0.15300739, -0.12067419, 0.11333649, 0.36544785, 0.10615282, -0.19266807, 0.00742444, -0.3133698 , 0.13811715, 0.17462967, -0.04043375, 0.01431966, 0.02120961, -0.31457424, 0.24998754, -0.30905357, -0.33957723, 0.44098425, 0.31071022, 0.5162943 , 0.1519571 , 0.20767058, 0.11702721, 0.30976802, -0.26687843, 0.02610935, -0.38046166, 0.15602328, 0.1049099 , -0.3816162 , 0.34945863, -0.13488789, 0.25807512, -0.14170453, -0.2737117 , 0.6794868 , -0.5637404 , -0.7506761 , -0.59640557, -0.35966128, -0.36779413, 0.6445572 , -0.15863825, 0.28188404, 0.6817374 , -0.30411932, 0.15068318, 0.21245846, 0.47254717, -0.3180801 , -0.72213024, -0.00796549, 0.52852696, -0.74865776, 0.72052324, -0.6231846 , -0.35524404, -0.6322673 , 0.5927172 , 0.7579426 , 0.22011517, 0.37161058, -0.58662343, -0.5653432 , 0.4952815 , 0.34009102, 0.4199334 , -0.04050065, 0.967862 , 0.62645787, -0.24915175, 0.6059676 , -0.12386217, -0.97235304, 0.98436683, -0.71207863, -0.2554404 , 0.29857978, -0.6535443 , -0.00581208, 0.54410416, -0.64588374, -1.2313039 , -0.7612568 , -0.5072008 , -0.15740068, 0.5934992 , 0.29966694, -0.78861797, 0.95648044, 0.90275365, 0.3240316 , -0.7827806 , -0.90508664, 0.24878727, -0.92607623, 0.45041922, -0.7728655 , 0.2784089 , 0.75287944], dtype=float32), kernel: Array([[ 1.6211687e-01, -2.6181787e-02, -5.1868878e-02, ..., -2.0115136e-01, -1.6275570e-01, 4.0626042e-02], [ 2.0827402e-01, -1.1491413e-01, -1.4099246e-01, ..., 1.2704584e-01, 2.0065814e-01, 3.5636088e-01], [ 1.4854251e-01, -1.1401314e-01, -4.7746781e-02, ..., -8.4292240e-02, 1.9939394e-01, 9.9089958e-02], ..., [-5.0814073e-03, 7.1649961e-02, 8.1174605e-02, ..., 1.8439499e-01, -1.8450679e-01, 3.2407336e-04], [ 5.2984446e-02, -4.0257105e-01, -3.7533784e-01, ..., 2.7334958e-01, -7.9834215e-02, -2.6604572e-01], [-1.3808161e-01, -2.5356847e-01, -2.2354984e-01, ..., -3.9307380e-01, 3.9180964e-01, 4.5249498e-01]], dtype=float32), }, query: { bias: Array([ 3.67379338e-01, -3.86799455e-01, -3.21791142e-01, -8.42057094e-02, 4.18459654e-01, 5.85961580e-01, -4.60834682e-01, -6.04065180e-01, 1.94889054e-01, 5.26792884e-01, -4.81182262e-02, -8.55236873e-02, -5.21454439e-02, 4.12250131e-01, -1.91803545e-01, -5.08220792e-01, -6.88099623e-01, 1.73875868e-01, -6.42500934e-04, 3.13504860e-02, 7.57871941e-02, -1.18366331e-01, 7.69717107e-03, 6.50874972e-01, -1.46498501e-01, -3.85033339e-02, 1.68164968e-01, 2.94075936e-01, 5.43904722e-01, 1.82771876e-01, 4.28779982e-02, 3.94661844e-01, 2.40447834e-01, 7.63046503e-01, 1.41106710e-01, 1.50562599e-01, 7.20623806e-02, -3.31430919e-02, 9.66051295e-02, -4.25018109e-02, 2.59480290e-02, 5.50236627e-02, 6.59183860e-02, 7.35617638e-01, 9.20522138e-02, -1.12395249e-01, -2.63706028e-01, -4.15704325e-02, 9.31539387e-02, -6.19517326e-01, 3.92871583e-03, -2.88900491e-02, 8.03363383e-01, -1.52911708e-01, 7.79530257e-02, -1.06074944e-01, -5.33656120e-01, 1.22914307e-01, 1.09775752e-01, 3.54613900e-01, -4.29570854e-01, 1.18217006e-01, 5.40162444e-01, -1.17120035e-01, -6.53074026e-01, -4.32521045e-01, -7.02763855e-01, -4.33477342e-01, -4.24702972e-01, 4.39991415e-01, -2.19614744e-01, 4.09397632e-01, 5.78722954e-01, 1.95280928e-02, 2.52549767e-01, 5.14830410e-01, 5.46005607e-01, -6.23379529e-01, -4.24228251e-01, -2.12890625e-01, 2.95019180e-01, -6.27236724e-01, 5.73115885e-01, -3.63081634e-01, -3.95359695e-01, -4.50571030e-01, 5.81661999e-01, 5.84261417e-01, 2.53536344e-01, 5.42065144e-01, -3.84602755e-01, -4.97589141e-01, 4.21692282e-01, 4.60142285e-01, 3.82059246e-01, -3.72222006e-01, 5.13080180e-01, 2.00476050e-01, -6.01216495e-01, 8.54436874e-01, 2.28271738e-01, -3.90953124e-01, -9.38059986e-02, -8.37827742e-01, 5.72855584e-02, -1.37482524e-01, 1.96730390e-01, 1.36508822e-01, 1.39595941e-01, -4.52210784e-01, 1.51813626e-01, -7.74807990e-01, 2.37015381e-01, -3.03821355e-01, -3.49135190e-01, -1.64326906e-01, -8.14965189e-01, -1.61662132e-01, 8.50251555e-01, 2.36134544e-01, 1.86818361e-01, -8.37599099e-01, 9.94282067e-02, 8.31496865e-02, 1.03579015e-01, -4.60613877e-01, 8.01612064e-02, 8.32802534e-01], dtype=float32), kernel: Array([[ 0.00486563, 0.01157592, 0.0173437 , ..., 0.06944456, -0.07013498, 0.01022998], [-0.07428966, -0.05709548, -0.03850973, ..., -0.09689807, 0.22320649, 0.2770357 ], [-0.10973258, 0.19551165, 0.12421291, ..., -0.02782791, -0.36778155, -0.19087008], ..., [ 0.30281523, 0.3621789 , 0.32462892, ..., 0.0546729 , -0.04859495, -0.10934605], [ 0.5838381 , 0.22978936, 0.17813566, ..., -0.16678724, 0.70342004, 0.18783332], [-0.48419687, 0.04412645, -0.01096392, ..., -0.05038855, 0.37432894, 0.02556431]], dtype=float32), }, value: { bias: Array([ 0.00544928, 0.17689343, 0.04735808, -0.18264541, -0.04237033, -0.17941093, 0.00340467, -0.00149406, -0.23728922, -0.07449124, -0.2651422 , -0.0984367 , -0.12018497, -0.08068941, -0.01734875, -0.06606279, 0.1477764 , -0.02022878, -0.1402983 , -0.02072977, 0.12049822, 0.06401265, 0.00406305, 0.15095161, -0.20597391, 0.21405579, -0.08197237, -0.00951357, -0.13306661, -0.08558191, 0.08653639, -0.10818963, -0.1644184 , -0.15369424, 0.3490528 , 0.01967676, -0.16458155, 0.10356332, -0.2320539 , -0.0433312 , 0.294788 , 0.08036903, -0.12358049, 0.16056572, 0.25535643, 0.16914676, -0.06201077, 0.28501964, -0.13111252, 0.11861207, 0.26627177, 0.16901444, -0.19237208, -0.08137509, 0.2016543 , 0.15213989, -0.14299503, -0.24307302, -0.29300013, -0.0722383 , 0.05268372, -0.14988767, -0.19426815, -0.24243473, -0.06109034, -0.13096058, 0.155086 , 0.15769799, 0.10678913, 0.00772086, 0.12339002, -0.02504941, -0.13470462, 0.02844545, 0.09698638, 0.11937644, -0.01387584, -0.13328613, -0.05046262, -0.02981377, -0.11246655, -0.06740061, -0.00562667, 0.15140204, 0.08057123, -0.03175431, 0.05622792, 0.11395832, -0.09259764, -0.02921034, -0.10965273, 0.09367548, 0.07572406, 0.17622064, -0.12645517, -0.06927612, -0.13641733, 0.11861874, -0.01573502, -0.10696157, -0.14588912, -0.16159363, 0.1815869 , 0.08687254, 0.11738776, -0.0497781 , -0.12963611, 0.01230228, -0.10121961, -0.06638794, 0.11086646, -0.05197389, 0.24918154, -0.16594617, 0.20172946, 0.06618202, 0.1124498 , 0.11753738, -0.19368784, 0.04725859, -0.13396324, -0.10240424, -0.11015446, 0.0015793 , 0.05593257, -0.12091528, 0.08938907, -0.03860855], dtype=float32), kernel: Array([[ 0.06369832, 0.04487664, -0.0572863 , ..., -0.00566542, 0.00958204, -0.0823305 ], [-0.48305708, -0.05723378, 0.09843583, ..., 0.01919795, 0.04487755, 0.36410812], [-0.39578405, 0.17672975, -0.04119537, ..., 0.06564505, -0.01046039, -0.30802208], ..., [-0.04638171, -0.03762753, -0.15475406, ..., 0.11968867, -0.1331625 , 0.01799979], [ 0.34840006, 0.0131262 , -0.17438433, ..., 0.21620421, -0.14971787, -0.73045504], [ 0.14833876, 0.18964465, 0.05431836, ..., 0.10322906, 0.10638059, -0.31155896]], dtype=float32), }, }, output: { dense: { bias: Array([-1.46490365e-01, 2.10801542e-01, -1.72206163e-02, -3.77512947e-02, -7.96984136e-03, -7.52876028e-02, 1.01055026e-01, 2.95142401e-02, 4.51585725e-02, 2.89253950e-01, -3.78346108e-02, 3.64244655e-02, -6.14614636e-02, 4.95899320e-02, -3.73293981e-02, 5.79853542e-02, 8.40439461e-03, -1.02269538e-01, -7.46445209e-02, 1.14303716e-01, -2.32893787e-02, -3.25526744e-02, -1.71621908e-02, -2.69102436e-02, -4.24926169e-02, 6.34983480e-02, -9.30675417e-02, -2.39975707e-04, 8.57859775e-02, 5.52252270e-02, 8.55132714e-02, -9.58903059e-02, 1.34693608e-01, -1.06799692e-01, 8.42991620e-02, -7.29722008e-02, -3.16112302e-02, -1.33186892e-01, -3.01891286e-02, -1.23371415e-01, 2.47585457e-02, 9.32596158e-03, -8.83968234e-01, 3.25804874e-02, -1.36455193e-01, 5.22013418e-02, 7.06611201e-02, 2.37159967e-01, 8.91016126e-02, -2.99384054e-02, -1.87199526e-02, 2.26440327e-03, -2.86240019e-02, 1.45185545e-01, 1.16965972e-01, -3.56947407e-02, 1.23156169e-02, -7.89485201e-02, 2.42000669e-01, 5.18658943e-02, 1.23368412e-01, -1.33664429e-01, -5.28409593e-02, -1.49152186e-02, -7.04273768e-03, 4.40860493e-03, -9.83215496e-02, 4.80295867e-02, 1.94351804e-02, 9.06426013e-02, -1.53706968e-02, 5.25671393e-02, 7.91645125e-02, 2.07034983e-02, 7.01766312e-01, 1.80371366e-02, -1.23936214e-01, 3.77195626e-02, -9.12794545e-02, -5.07131638e-03, -5.29812515e-01, 2.14143340e-02, 7.39834160e-02, -7.41400057e-03, 6.01231493e-02, 7.33223036e-02, 1.53090119e-01, 1.70849040e-01, -4.07319590e-02, -4.82572988e-03, 2.90474202e-02, 4.86663431e-02, -1.04625016e-01, -8.78558010e-02, -4.20586728e-02, 3.25415730e-02, -7.98427388e-02, -1.87286697e-02, -2.41589937e-02, 8.27142298e-02, 4.98151742e-02, 5.18750064e-02, 3.29844393e-02, -3.68855074e-02, 8.92899334e-02, -4.56710160e-03, 2.97619849e-02, -7.04986304e-02, -1.08520284e-01, -1.11746334e-01, -7.71660432e-02, 8.49972386e-03, 2.37586319e-01, -1.24874555e-01, -1.93465855e-02, 8.61749053e-02, 8.92835855e-02, -8.50701034e-02, -3.94290835e-02, -2.06257198e-02, -1.92599650e-02, 2.76334919e-02, 6.91349506e-02, -2.97043975e-02, 8.56228359e-03, 5.08836359e-02, -1.17148422e-02, -2.16257907e-02], dtype=float32), kernel: Array([[ 0.16482222, -0.22908561, 0.5409443 , ..., 0.52288747, 0.45133135, -0.3148359 ], [ 0.04791335, -0.19366321, 0.02344365, ..., 0.25754645, 0.15889917, -0.26091245], [ 0.21047738, 0.00189075, -0.053845 , ..., -0.10111952, -0.42335704, -0.05204519], ..., [-0.07512332, 0.07116441, -0.20204972, ..., -0.3296345 , 0.2802311 , -0.00193282], [-0.01015196, 0.04124757, 0.08690923, ..., 0.38333163, -0.18434097, -0.01834725], [ 0.0501605 , 0.7881087 , 0.21703115, ..., -1.006583 , -0.5440753 , 0.06839034]], dtype=float32), }, }, }, layernorm_after: { bias: Array([ 7.00707082e-03, -2.13932544e-01, -4.38091122e-02, -1.80086847e-02, 1.56905562e-01, 3.25518586e-02, -6.51721493e-04, -3.70227383e-04, -9.39783454e-03, -1.21371552e-01, 6.61288295e-03, -3.71838771e-02, 1.69692468e-02, -1.13847349e-02, -1.44199710e-02, -1.44837052e-03, 1.02954609e-02, 7.14356266e-03, -2.28633024e-02, -9.65952054e-02, -7.08255393e-05, 4.81518777e-03, -1.34146630e-05, 3.52103100e-03, -1.46182910e-01, -2.66583692e-02, -2.81199813e-02, 2.20761336e-02, 8.75673592e-02, 1.68776400e-02, 7.70330988e-03, -7.63349526e-04, 1.69382174e-03, -5.38112130e-03, -2.32931431e-02, 1.33482378e-03, 5.63706718e-02, -1.53671801e-01, -5.60674816e-04, 1.17010390e-02, 4.67808433e-02, -1.00002938e-03, 7.54884481e-02, 9.78838280e-03, -2.51426380e-02, -1.93388745e-01, -1.30395563e-02, -3.22579712e-01, -1.23171741e-02, -2.32061837e-02, -5.90802915e-02, 6.99069537e-03, 1.86737273e-02, -1.55908689e-01, 5.42053096e-02, -1.68696530e-02, -3.76862218e-03, 1.75712314e-02, -2.17723008e-02, -1.39841557e-01, -1.11271013e-02, -8.20189528e-03, -1.91661697e-02, -2.04071011e-02, 5.92404092e-03, 2.63817403e-02, 2.42956262e-02, 6.90133497e-02, -7.82949443e-04, 6.91150129e-03, -6.76295608e-02, -8.51555727e-03, -1.79697247e-03, 4.60966006e-02, -4.87765044e-01, 3.24066654e-02, -7.08101019e-02, -2.59833187e-02, -4.40584077e-03, 3.93147208e-03, 2.80292213e-01, -3.62146534e-02, 1.14013357e-02, 1.44051407e-02, 8.79329368e-02, -3.40585620e-03, 1.61548674e-01, -4.32909727e-02, 5.22663780e-02, -8.88153631e-03, -5.72965667e-03, 1.60463457e-03, -1.44764967e-03, -4.41680253e-02, -6.15086630e-02, 2.85431091e-03, -5.11140227e-02, 3.58232483e-02, 3.14552360e-03, -8.34190473e-02, -6.30146563e-02, 3.52296643e-02, 1.75646115e-02, -2.81428844e-01, 1.00661494e-01, -1.63044967e-02, -1.77673250e-03, -2.50048609e-03, 9.31597501e-02, -5.21637946e-02, -1.21001536e-02, -3.70397139e-03, -1.89869609e-02, 4.85324338e-02, -1.78271660e-03, -1.17816813e-02, 6.07670583e-02, -6.83511868e-02, -7.59183466e-02, 2.76686642e-02, -2.92036515e-02, 7.23615754e-03, -1.44946417e-02, 6.89900964e-02, -1.19854533e-03, -3.26084495e-02, -3.72714689e-03, 8.10674950e-02], dtype=float32), scale: Array([ 0.01037673, 0.20990598, 0.17576548, 0.21721113, 0.29847282, 0.69343257, 0.00376189, -0.01851019, 0.03572948, 0.19570692, -0.02536977, 0.42987674, 0.01018215, 0.15321532, 0.48395535, -0.00318469, 0.16068086, 0.01112385, 0.36739224, 0.4355385 , -0.01194211, -0.00359833, -0.086844 , 0.00597237, 0.5273556 , 0.03060911, 0.6654633 , -0.05266754, 0.36045882, 0.10801558, 0.29989985, -0.04426623, 0.00284036, 0.02850281, 0.08200129, 0.00828265, 0.4786764 , 0.32044822, 0.6007385 , 0.39719838, 0.38223657, -0.02341203, 0.02303379, 0.25033328, -0.08627673, 0.5241369 , 0.544966 , 0.3724897 , -0.32547915, -0.24456456, 0.32741693, 0.11249542, 0.07124001, 0.5731743 , 0.22054233, 0.68054676, -0.04105046, 0.07573894, 0.05682403, 0.4511571 , 0.51980895, 0.05435016, 0.06856246, 0.16396722, 0.00960637, 0.06122901, 0.23258007, 0.3978171 , 0.23067153, -0.0107969 , 0.517839 , 0.03140327, 0.02615155, 0.70969087, 0.1419986 , 0.37057772, 0.54277253, 0.18082201, 0.0020515 , -0.11159092, 0.43598834, 0.37816525, 0.586952 , 0.35433614, 0.50891733, 0.00721514, 0.27681717, 0.38934416, 0.4676223 , 0.18506213, 0.18672895, 0.01464603, -0.00548072, 0.72455907, 0.26698393, -0.00541573, 0.8330279 , 0.64272374, 0.00286223, 0.52491736, 0.5533957 , 0.3439355 , 0.32210883, 0.56023586, 0.5354366 , 0.3412869 , 0.05528033, -0.15407614, 0.28940812, 0.39378768, 0.5161475 , -0.03409952, 0.05442618, 0.390517 , 0.00593842, 0.06129869, 0.42658743, 0.4008259 , 0.46517026, 0.30656123, 0.2991743 , 0.03482211, 0.5670987 , 0.35471496, 0.08178211, 0.0831662 , 0.00438205, 0.29678687], dtype=float32), }, layernorm_before: { bias: Array([-1.00732700e-03, -3.65120143e-01, -3.34881805e-02, 4.26957064e-04, -7.94777123e-04, -6.86906278e-03, -1.02201905e-02, -4.80189286e-02, 1.55887365e-01, -1.59472153e-01, 5.88005059e-04, -4.74225767e-02, 1.99526129e-03, -6.20241184e-03, 2.79833134e-02, 7.60109648e-02, -4.03508246e-02, -5.76728676e-03, -8.25315190e-04, -5.45392111e-02, -9.78699885e-03, -2.21987873e-01, 8.89718998e-03, 1.56212691e-02, 2.51562544e-03, -2.31638588e-02, 2.57532690e-02, -3.37633025e-03, -4.75819921e-03, 2.06635101e-03, 4.15294990e-02, -1.22549701e-02, 9.92366020e-03, 1.33297034e-02, -6.88615516e-02, -2.13969056e-03, -9.33400914e-03, -1.47152513e-01, 1.47844050e-02, 2.68603060e-02, 1.06496513e-01, -1.40187237e-03, 2.46544197e-01, -1.18307155e-02, 1.22905318e-02, 7.02085486e-03, 3.16104256e-02, 1.29607348e-02, -1.19067011e-02, 5.89615945e-03, 1.45399803e-02, 7.92596638e-02, 8.44230223e-03, -5.16842827e-02, 2.47099735e-02, 7.68707599e-03, -7.68995145e-03, 3.25880721e-02, -3.79534140e-02, -1.32843507e-02, 1.00192919e-01, 8.90694335e-02, 1.43536124e-02, -1.85336508e-02, 4.20249486e-03, 4.04581055e-02, 1.56428572e-02, 5.54322731e-03, 7.59444525e-03, -9.50715330e-04, -5.35057345e-03, 2.70144927e-04, 1.31974250e-01, 2.08387990e-02, -5.05078256e-01, -4.19664755e-02, 2.06986954e-03, 2.65133400e-02, 1.08654350e-02, -2.48142332e-02, -2.32796840e-04, -3.82554834e-03, -3.06907180e-03, -3.17831822e-02, -2.74789296e-02, 1.76229049e-04, -5.93862869e-03, -2.54624914e-02, 5.30235134e-02, 3.77030671e-02, -3.30569521e-02, 6.81126863e-02, -3.34922522e-02, -5.41602597e-02, -4.98229451e-03, -1.29764492e-03, 1.27451606e-02, 5.30401291e-03, 8.72196928e-02, 5.26771834e-03, -5.91456294e-02, 2.01584827e-02, -1.72205925e-01, -2.79393774e-02, -3.19484659e-02, -3.31652700e-03, 3.22748870e-02, 1.10277152e-02, 7.98177347e-02, -2.52419740e-01, 6.41727597e-02, 2.78249923e-02, -1.18425563e-01, 5.46144024e-02, -1.38351889e-02, 1.97037379e-03, 3.38066697e-01, -1.05525143e-02, -5.22397608e-02, -8.39580037e-03, 1.55923087e-02, 2.35547218e-02, -4.73677404e-02, 7.04186484e-02, -5.95991965e-03, 5.85895148e-04, 4.35512178e-02, -9.74338278e-02], dtype=float32), scale: Array([-0.00892128, 0.22834587, 0.07191961, -0.03907548, -0.00996873, 0.35814652, 0.27738327, 0.11775097, 0.07957964, 0.2968416 , 0.01412186, 0.27139878, -0.00244969, -0.01754642, 0.31717873, 0.24255775, 0.34514305, -0.0094078 , -0.01727619, 0.2070495 , 0.23165925, 0.48358163, 0.08980764, 0.16053861, 0.07536255, 0.02379382, 0.3457654 , 0.22846563, -0.1335053 , 0.01941462, 0.26780826, 0.09502309, 0.14652625, 0.05376663, 0.22817616, 0.20199119, 0.2713218 , 0.26872477, -0.0762281 , 0.22205614, 0.31939635, 0.24618064, 0.17259929, 0.21590437, 0.02615085, 0.17664434, 0.36018652, 0.12768349, 0.20409435, 0.14295201, 0.18369466, 0.3224867 , 0.01613115, 0.11641789, 0.19955999, 0.08757862, -0.01346728, 0.198473 , 0.29986396, 0.36929435, 0.5931218 , 0.19321899, 0.11543731, 0.16761477, -0.00143133, 0.07413565, 0.2747692 , 0.02911899, 0.12630646, 0.02624349, 0.3049502 , 0.17227921, 0.35921425, 0.295701 , 0.13059482, 0.19860806, 0.01733009, 0.24637464, 0.00986639, 0.19075045, -0.00124613, 0.27278668, 0.01111688, 0.16424142, 0.18801676, 0.01323086, 0.00368139, 0.03970122, 0.26844364, 0.10092615, 0.09264332, 0.16377948, 0.22653754, 0.35064098, -0.05561429, -0.00500794, 0.07151927, 0.0050664 , 0.2468497 , 0.10480628, 0.17903912, 0.08759426, 0.41523352, 0.24769782, 0.11629754, 0.03568345, 0.20900775, 0.05634574, 0.4340352 , 0.3333243 , 0.41587278, 0.20580357, 0.3584831 , 0.2544934 , 0.01131326, 0.34364027, 0.22543208, 0.13177043, 0.19698657, 0.21796568, -0.03295131, 0.298913 , 0.47042173, 0.26397368, 0.02212258, 0.01400256, 0.3371711 , 0.22812258], dtype=float32), }, moe: { shared_experts: { intermediate: { bias: Array([-5.27199693e-02, -1.22556537e-02, -4.80708107e-02, -2.20634907e-01, -1.40253054e-02, -4.87366840e-02, -8.70805699e-03, -1.54133514e-02, -2.52253171e-02, -3.50418165e-02, -2.73905657e-02, -3.95800360e-02, -3.71573754e-02, -2.81724222e-02, -1.33444788e-02, -1.28413305e-01, -4.53637205e-02, -3.11292056e-02, -3.32505777e-02, -3.26416194e-02, -3.45684476e-02, -1.40969567e-02, -2.91586872e-02, -2.76421569e-02, -3.73606943e-02, -3.91720347e-02, -3.77491489e-02, -1.35208545e-02, -2.09600553e-02, -4.55045812e-02, -5.19248992e-02, -2.19188482e-02, -1.82508994e-02, -6.17397487e-01, -3.09374705e-02, -5.18258326e-02, -2.61735450e-02, -3.89497764e-02, -1.46954479e-02, -1.18027776e-02, -2.21383367e-02, -3.07989083e-02, -6.69780612e-01, -1.84117928e-02, -4.12670113e-02, -8.98960326e-03, -3.24395895e-02, -1.61407497e-02, -4.12646681e-02, -5.37053635e-03, -3.38159055e-02, -3.29089276e-02, -1.68235525e-02, -3.26590054e-02, -2.52911523e-02, -4.61294129e-02, -5.67609370e-01, -2.84224618e-02, -4.13801000e-02, -3.34288217e-02, -3.53450403e-02, -3.29895802e-02, -4.02423888e-02, -2.35914946e-01, -4.60832685e-01, -3.70765813e-02, -2.73666903e-02, -2.54514460e-02, -1.44379437e-02, -4.47799824e-02, -4.39045168e-02, -4.60769348e-02, -1.07184583e-02, -2.94511896e-02, -3.48258428e-02, -4.94903252e-02, -4.71179605e-01, -3.04534864e-02, -2.14969758e-02, -2.71185916e-02, -2.13122219e-02, -2.35180482e-02, -3.83645929e-02, -2.21622270e-02, -2.68899292e-01, -3.47404443e-02, -3.07555981e-02, -2.66567357e-02, -2.49998886e-02, -4.09555882e-01, -3.07542346e-02, -1.73584837e-02, -3.32305208e-02, -3.31670046e-02, -3.28688100e-02, -2.02093106e-02, -3.04487552e-02, -4.77275476e-02, -4.22687948e-01, -4.02031802e-02, -2.18092110e-02, -3.58968377e-02, -2.14504618e-02, -3.14717107e-02, -6.16603434e-01, -3.43027376e-02, -4.16996516e-02, -2.88160015e-02, -3.17607224e-02, -4.91070986e-01, -4.11182418e-02, -3.82331535e-02, -1.99981369e-02, -3.31944197e-01, -1.49236154e-02, -4.61634010e-01, -4.73840237e-02, -1.24957161e-02, -3.81090641e-01, -1.33936629e-01, -4.61227804e-01, -3.94086614e-02, -5.02103120e-02, -1.19854463e-02, -2.26863697e-02, -3.57204154e-02, -2.89868433e-02, -3.16134878e-02, -2.95881126e-02, -3.33401635e-02, -2.84984969e-02, -3.39026898e-02, -4.84661758e-01, -3.00496016e-02, -2.10581459e-02, -3.03259753e-02, -2.73086485e-02, -2.92658973e-02, -3.28680202e-02, -4.29259539e-02, -1.15742823e-02, -3.06628253e-02, -3.49430703e-02, -4.13970686e-02, -4.64555889e-01, -2.44929623e-02, -2.32402366e-02, -4.51661311e-02, -3.42191793e-02, -1.04636485e-02, -2.76410598e-02, -2.72743534e-02, -3.71849462e-02, -2.61768941e-02, -2.64907163e-02, -4.13466580e-02, -2.31192280e-02, -2.66561229e-02, -2.44059507e-02, -2.19917297e-02, -3.10962517e-02, -3.92389782e-02, -2.89020780e-02, -4.72296998e-02, -1.47718657e-02, -4.71781231e-02, -3.74658257e-02, -3.74986231e-02, -3.42181958e-02, -3.34076025e-02, -1.90843388e-01, -3.28308083e-02, -1.35448873e-02, -3.84903140e-02, -3.14540192e-02, 9.85380495e-04, -2.98315007e-02, -4.16581221e-02, -2.44194381e-02, -2.18599793e-02, -4.54461351e-02, -4.33417000e-02, -2.37555113e-02, -2.65070703e-02, -3.74873281e-02, -2.00963188e-02, -5.27748942e-01, -1.71016864e-02, -2.98697986e-02, -3.35949808e-02, -4.21402045e-02, -4.34817523e-02, -3.61697711e-02, -3.83502319e-02, -2.63728369e-02, -9.91544034e-03, -4.13909107e-02, -3.27453129e-02, -1.61904916e-02, -2.92313341e-02, -1.82820782e-02, -3.27837802e-02, -4.96267676e-02, -2.13364977e-02, -3.07123903e-02, -1.29112750e-02, -3.05922143e-02, -1.34589160e-02, -1.97699457e-01, -1.26856491e-02, -3.33952643e-02, 3.23457294e-03, -2.03445368e-02, -2.09414233e-02, -1.26234703e-02, -3.02408133e-02, -2.34196670e-02, -1.23696076e-02, -2.40369998e-02, -2.81809513e-02, -2.13961061e-02, -5.47036016e-03, -4.53693084e-02, -3.31229195e-02, -1.50107536e-02, -2.91419178e-02, -2.07277276e-02, -5.49518093e-02, -2.95787137e-02, -2.38886494e-02, -2.65640840e-02, -2.46801712e-02, -3.84779572e-02, -1.79381035e-02, -2.02893633e-02, -3.30808014e-02, -9.46734101e-03, -3.95300575e-02, -3.69772725e-02, -3.11741419e-02, -4.37073298e-02, -2.90284753e-02, -3.63519609e-01, -1.85389761e-02, -1.82310361e-02, -1.51738618e-02, -1.89224500e-02, -2.03151274e-02, -3.81886885e-02, -2.58216038e-02, -3.81475925e-01, -2.79287975e-02, -1.52051644e-02, -2.38528401e-02, -4.04975682e-01, -3.95358130e-02, -4.14295375e-01, -4.38631624e-02, -4.29423749e-02, -3.17317769e-02, -3.27443145e-02, -2.63639372e-02, -3.31052691e-02, 4.43929108e-03, -3.27373035e-02, -2.39749160e-02, -3.15703414e-02, -5.37155151e-01, -8.51269998e-03, -3.57643776e-02, -3.98143642e-02, -3.45447585e-02, -2.49514170e-02, -2.87994146e-01, -5.97163796e-01, -3.36527936e-02, -3.88488770e-02, -3.22511718e-02, -4.89461757e-02, -3.97258699e-02, -2.26812717e-02, -7.74491072e-01, -5.23366332e-01, -4.26757559e-02, -3.48140113e-02, -2.94348393e-02, -3.54919247e-02, -2.96318009e-02, -1.08983302e-02, -3.76659594e-02, -1.47500113e-02, -2.93985605e-02, -2.53436007e-02, 5.79031697e-03, -2.69875340e-02, -2.84397658e-02, -3.33693698e-02, -4.03800532e-02, -2.46412121e-02, -2.71790083e-02, -6.01157732e-02, -3.36032584e-02, -3.37207466e-02, -1.18618924e-02, -4.47471261e-01, -4.70100343e-02, -1.94270499e-02, -1.41834235e-02, -4.12988141e-02, -4.31584977e-02, -2.58529224e-02, -3.11732627e-02, -3.12253926e-02, -4.00089920e-02, -2.82256324e-02, -1.63722355e-02, -2.23478712e-02, -3.37487720e-02, -1.17873568e-02, -3.71523947e-02, -3.02173067e-02, -2.47102734e-02, -1.75078567e-02, -8.67900550e-01, -4.19426337e-02, -2.89364811e-02, -3.62409726e-02, -1.40926614e-02, -3.56511444e-01, -3.45515162e-02, -5.06763607e-02, -2.30538756e-01, -3.77509855e-02, -3.81864123e-02, -3.19356658e-02, -2.84604561e-02, -4.72619906e-02, -1.65800638e-02, -4.46446165e-02, -1.26604997e-02, -3.79219428e-02, -3.04708660e-01, -3.42885070e-02, -4.41885963e-02, -3.81699838e-02, -3.03119402e-02, -2.27376912e-02, -1.89236663e-02, -4.71340455e-02, -1.56659111e-02, -3.82196195e-02, -3.61527875e-02, -2.05292553e-02, -3.56830619e-02, 3.07818269e-03, -7.64196873e-01, -2.73544770e-02, -5.15536249e-01, -1.76008549e-02, -6.18065059e-01, -3.16055007e-02, -4.54795063e-02, -2.52228677e-02, -1.78863723e-02, -3.71731035e-02, -3.15613933e-02, -3.33131254e-02, -1.16747636e-02, -1.52025633e-02, -3.18961823e-03, -2.85964645e-02, -3.42308469e-02, -6.01763930e-03, -1.44698163e-02, -3.33204381e-02, -1.46625296e-03, -1.37048895e-02, -2.66424436e-02, -4.91908789e-02, -3.08587961e-02, -2.29315758e-02, -3.17254931e-01, -1.58655606e-02, -8.84249713e-03, -1.90528110e-02, -4.16713282e-02, -3.05165984e-02, -3.23779024e-02, 1.71483532e-02, -1.71599649e-02, -1.27734728e-02, -3.09068114e-02, -3.11801564e-02, -3.09810303e-02, -3.29174399e-02, -4.35743593e-02, -3.38067338e-02, -2.26877723e-02, -3.14960629e-02, -2.18670722e-02, -2.10534912e-02, -2.37145834e-02, -3.18468064e-02, -3.17289941e-02, -3.52053195e-02, -3.62415873e-02, -3.78505550e-02, -8.54883492e-01, -6.25598431e-01, -2.91614741e-01, -4.12686355e-02, -3.40597779e-02, -4.49921228e-02, -1.83722675e-01, -6.20228611e-03, -9.09710955e-03, -4.55179624e-02, -4.24469262e-02, -4.34849598e-02, -1.18993660e-02, -2.96284892e-02, -1.82864573e-02, -1.58976540e-02, -4.17148732e-02, -1.04806479e-02, -2.51498409e-02, -3.05920038e-02, -2.39100829e-02, -2.05667838e-02, -4.11339812e-02, -2.58969050e-02, -2.54813936e-02, -3.46504189e-02, -3.05222347e-02, -3.66321839e-02, -1.69235300e-02, -3.10963802e-02, -2.00775322e-02, -3.57228443e-02, -4.01586331e-02, -2.64529884e-02, -4.16483805e-02, -4.06530350e-02, -2.00617034e-02, -4.20974828e-02, -3.35381776e-02, -4.97709587e-02, -1.51588218e-02, -4.78560105e-02, -1.61089674e-02, -2.51727998e-02, -4.55499999e-02, -3.32853608e-02, -9.99737252e-03, -9.59883165e-03, -3.51412706e-02, -3.77242006e-02, -1.95016433e-02, -8.43567073e-01, -1.19444542e-02, -1.49743442e-04, -3.27145346e-02, -3.28630805e-02, -1.98399983e-02, -1.64248962e-02, -3.85820344e-02, -1.64614365e-01, -1.02842608e-02, -1.50990197e-02, -3.22756879e-02, -1.80973206e-02, -2.27656811e-02, -4.72140424e-02, -3.06061078e-02, -5.52861243e-02, -3.98167334e-02, -3.09152249e-02, -3.28242555e-02, -3.62482332e-02, -2.84483526e-02, -7.59550452e-01, -4.58953902e-02, -2.13591140e-02, -2.78457664e-02, -1.19789401e-02, -2.98056193e-02, -3.15071791e-02, 5.63531043e-03, -2.39895564e-02, -3.00265819e-01, -4.11561355e-02, -2.46453267e-02, -4.19642702e-02, -5.44620812e-01, -3.15308571e-02, -3.95948440e-02, -2.19081920e-02, -2.35356446e-02, -1.36993527e-02, -1.34584429e-02, -5.07402495e-02, -3.16087119e-02, -2.87754349e-02, -5.01885235e-01, -4.85699102e-02, -4.50458750e-02, -2.57593673e-02, -2.32719798e-02, -3.24436370e-03, -2.84067634e-02, -2.62353010e-02, -3.73448022e-02], dtype=float32), kernel: Array([[ 0.03366351, -0.01376447, 0.00334523, ..., -0.01204627, 0.0351628 , 0.0058507 ], [ 0.01479752, -0.00059022, 0.02888848, ..., 0.01372632, 0.05084606, -0.00852082], [-0.08748913, 0.06752201, -0.02948284, ..., -0.04957195, -0.01602156, -0.0564839 ], ..., [ 0.03158367, -0.0024418 , -0.01248567, ..., 0.00120247, -0.03067757, 0.05438066], [ 0.00710372, -0.07475934, -0.03510208, ..., -0.00362264, -0.01655151, 0.0695975 ], [ 0.03609428, -0.04006155, -0.0374678 , ..., -0.02961577, -0.03186451, 0.05374904]], dtype=float32), }, output: { bias: Array([-1.57430485e-01, 2.07714707e-01, -1.93849746e-02, -2.16451492e-02, -2.94376239e-02, -1.40355468e-01, 1.01594821e-01, 9.77267139e-03, 3.38900611e-02, 3.08969438e-01, -4.50306423e-02, 4.16773744e-02, -5.64103350e-02, 4.65862826e-02, -1.98888164e-02, 5.25933765e-02, 5.03250724e-03, -9.93339792e-02, -9.45335254e-02, 1.34056509e-01, -2.08574124e-02, -2.76607983e-02, -2.45273374e-02, -2.30302606e-02, 4.72312383e-02, 6.59246966e-02, -6.18701801e-02, -2.54601543e-03, 8.12924057e-02, 4.04679216e-02, 7.19875395e-02, -8.80277827e-02, 1.21765286e-01, -1.06022269e-01, 8.69699791e-02, -7.91017562e-02, -7.66793415e-02, -9.08406228e-02, -1.50119308e-02, -1.08254775e-01, 1.43383909e-02, 1.25184851e-02, -8.86707306e-01, 3.01385261e-02, -1.39143646e-01, 1.25063658e-01, 5.04919551e-02, 2.49843255e-01, 5.21366931e-02, -3.97394188e-02, -1.14855161e-02, -1.06901973e-02, -2.97776181e-02, 1.87453896e-01, 8.37227628e-02, -1.05806980e-02, -2.71216761e-02, -8.99265856e-02, 2.36086398e-01, 1.00231051e-01, 1.18218437e-01, -1.38503894e-01, -4.76376042e-02, -2.26873197e-02, -9.81818512e-03, -6.29501976e-03, -1.04944162e-01, -4.35994677e-02, 1.57965776e-02, 8.73260945e-02, 8.73774476e-03, 5.19699082e-02, 7.45332241e-02, -8.56231526e-03, 7.00146556e-01, 9.85437538e-03, -9.72239524e-02, 3.94646786e-02, -9.50414538e-02, -3.36449943e-04, -4.73328263e-01, 4.33811136e-02, 1.06056370e-01, -4.22528051e-02, 2.30107084e-02, 1.08815312e-01, 1.21224724e-01, 1.74949020e-01, -2.51618456e-02, -2.49235518e-02, 2.24952735e-02, 4.77656983e-02, -1.08213760e-01, -1.28404409e-01, -3.41621116e-02, 3.01731993e-02, -7.27362260e-02, -1.10188257e-02, -2.82108001e-02, 1.34670347e-01, 5.28050587e-02, 5.32444753e-02, 3.20014991e-02, 5.04108816e-02, -3.19805578e-04, -1.66307837e-02, 3.41970064e-02, -6.60975128e-02, -1.23339623e-01, -9.24760997e-02, -8.67600888e-02, 8.86741187e-03, 2.36912012e-01, -1.58255234e-01, -1.41610643e-02, 8.07640478e-02, 2.81964112e-02, -8.16835240e-02, -1.64333303e-02, -2.98798494e-02, -7.30156377e-02, 2.87766512e-02, 1.21695735e-01, -5.90394363e-02, 2.60551348e-02, 5.38477339e-02, -2.07343251e-02, -6.99748248e-02], dtype=float32), kernel: Array([[ 0.02169053, 0.02691332, -0.00372989, ..., -0.00424074, 0.03661725, 0.08953198], [-0.01738822, 0.01985764, 0.03393567, ..., -0.00211225, 0.01224655, 0.03731351], [ 0.00689401, 0.02467112, 0.01469113, ..., -0.01584834, -0.03345798, 0.01720266], ..., [ 0.01760239, 0.00760174, 0.01594303, ..., 0.00213138, 0.00272581, -0.03686259], [ 0.01555281, -0.08100978, 0.04178749, ..., -0.0257984 , -0.04736535, -0.03686064], [-0.01932741, 0.05431129, -0.04024049, ..., -0.00516084, -0.01021444, 0.02485615]], dtype=float32), }, }, }, }, 5: { attention: { attention: { key: { bias: Array([ 0.15380459, -0.23325492, 0.5298237 , -0.25394794, -0.1792779 , 0.2635233 , -0.2518766 , 0.51324993, 0.44011664, -0.15511836, 0.1065289 , -0.18103923, -0.24789087, 0.01313723, -0.00924131, 0.2846446 , -0.02708824, 0.01480816, 0.12186062, -0.18671922, 0.47530514, -0.44773427, -0.42629737, -0.25819948, 0.15633082, -0.48324105, -0.2818154 , -0.61912835, -0.29742548, 0.28758916, 0.07666907, 0.27997455, -0.5159455 , -0.00791557, 0.40676025, -0.13460469, 0.5236954 , -0.5017536 , 0.5868691 , 0.01062866, -0.10968728, -0.03771151, -0.33758804, -0.53288084, -0.6708519 , 0.60964614, 0.14677066, -0.52277976, -0.00805334, -0.3969676 , 0.5209981 , 0.34777972, 0.5903865 , -0.7917442 , -0.64988923, -0.63839513, -0.74101704, 0.11602865, 0.8335415 , 0.2829763 , -0.41182905, -0.3442916 , -0.47213373, 0.5067961 , 0.05412646, 0.11681292, 0.26332203, -0.25662488, 0.4626192 , -0.2888915 , 0.3559262 , 0.43773538, -0.4578352 , 0.32878822, -0.2354647 , -0.39989737, -0.21824938, 0.23775657, 0.28870043, -0.35698473, 0.2748483 , 0.05920567, -0.09002119, -0.35282713, 0.20067027, -0.54461175, -0.23671876, -0.26704383, -0.06359763, 0.01624733, -0.20986806, -0.0469152 , -0.21808156, 0.10429718, 0.28616387, -0.0075318 , 0.41809165, 0.21852025, -0.68503106, 0.15126543, -0.5385752 , -0.6387902 , -0.09934053, 1.1725092 , 0.9802906 , -0.24173768, -0.3697774 , 0.30808112, 1.2169337 , -0.4494893 , 0.80163956, 0.62732154, -0.03264387, 0.67577094, -0.39557546, -0.40416336, 0.0219003 , 0.2829875 , -0.28239524, 0.14352222, -0.8248839 , 0.02846357, 0.13515349, -0.19850218, -0.22375287, 0.2135571 , -0.27206707, 0.76276755], dtype=float32), kernel: Array([[ 0.26200238, 0.13167961, -0.11595184, ..., 0.0414915 , 0.11238165, -0.30494452], [ 0.22360708, 0.08118725, 0.02069189, ..., 0.00547806, -0.10924851, 0.26473263], [-0.16265303, 0.17587142, 0.12479865, ..., -0.07595218, 0.1396125 , -0.19043295], ..., [-0.26119086, -0.38923773, 0.35491627, ..., -0.7368683 , -0.49823397, -0.6397334 ], [ 0.13432133, -0.16010363, 0.22101043, ..., -0.06990113, 0.25519368, 0.09977636], [-0.12494502, -0.38168752, 0.16611753, ..., 0.36234066, -0.0669388 , 0.3607124 ]], dtype=float32), }, query: { bias: Array([-0.18479703, -0.35146004, 0.7537319 , -0.17177609, 0.01369478, 0.18720719, -0.17976424, 0.15347035, 0.08994126, 0.22886601, 0.18279251, -0.35825828, -0.06468849, -0.12611765, -0.028998 , 0.9971776 , -0.18668026, -0.04904906, -0.14856422, -0.24965602, 0.05141901, -1.1571982 , 0.05819758, 0.14917302, 0.15365446, -0.85294926, -0.08199202, -0.9097335 , 0.15537313, -0.26021054, 0.12598538, -0.06024593, -0.11726313, -0.20979708, 0.20502815, 0.22255987, 0.30414093, -0.11271469, 0.4672999 , 0.00509386, -0.13042028, 0.22407965, -0.10005572, -0.14989175, -0.4365904 , 0.2107629 , 0.18609835, -0.72566986, 0.18099388, -0.30312923, 0.5998623 , -0.15350953, 0.36957732, -0.70576656, -0.68291605, -0.420833 , -0.34906906, 0.06481054, 0.58362794, 0.31369215, -0.82122093, -0.25893688, -0.24382648, 0.19694354, 0.16259049, -0.11133604, -0.0934663 , -0.02774331, 0.21941422, -0.01251643, 0.12317675, 0.6149345 , 0.14704953, 0.37044725, -0.28755513, 0.1573624 , -0.44036826, 0.1894717 , 0.12188441, -0.49551368, 0.14024805, -0.03632037, -0.11464448, -0.6433361 , 0.15439317, -0.32633895, 0.00914915, -0.05114478, -0.1278209 , 0.33733654, -0.07761955, -0.3304572 , -0.59981114, 0.14654088, 0.06847122, -0.259691 , 0.25001425, 0.05990116, -0.645737 , 0.02534364, -0.6641346 , -0.17594564, -0.01261867, 0.5100829 , 0.06452097, -0.5573339 , 0.13375577, 0.25303102, 0.43233076, 0.15803403, 0.254334 , 0.03359455, -0.05677273, -0.11994728, -0.2779966 , -0.23615459, -0.04638248, -0.03050251, 0.3847538 , 0.0142238 , -0.26559195, -0.03787645, 0.02767896, -0.08072381, -0.01213382, 0.15982705, -0.07372479, 0.6117539 ], dtype=float32), kernel: Array([[-0.21102534, 0.13572834, 0.08487151, ..., -0.37551054, -0.08974288, -0.14673388], [-0.42533755, 0.19477351, -0.05581117, ..., -0.19027376, 0.11326433, 0.280313 ], [ 0.20709127, -0.02206374, 0.00242551, ..., -0.21323436, -0.1782914 , -0.49215955], ..., [-0.35879493, -0.13843112, 0.20357771, ..., -0.3228759 , -0.48138356, -0.60722345], [ 0.409117 , 0.09512895, -0.07367483, ..., -0.79642665, -0.3589279 , 0.36061507], [-0.1480074 , 0.24665533, -0.10206775, ..., 0.03880555, -0.12229352, 0.42183685]], dtype=float32), }, value: { bias: Array([-0.06382484, -0.10177656, -0.01783326, 0.07238453, 0.0269923 , 0.03472366, 0.00803764, 0.09564783, -0.02505345, 0.16271482, 0.01969427, -0.08568954, -0.02066216, -0.03939519, -0.09977195, -0.01792192, 0.02864774, 0.03915608, 0.02681843, 0.07789411, -0.12741898, 0.03238543, 0.01603028, -0.11089758, -0.08953978, 0.11356429, -0.03502819, -0.0710902 , -0.07928857, -0.07034726, -0.00949693, -0.11724807, 0.03667774, -0.0831453 , -0.08759701, -0.11416655, 0.05479722, 0.02118164, 0.0808093 , -0.04038942, -0.05000678, -0.06270798, -0.00748655, -0.0456011 , 0.09870113, 0.03644619, 0.06150641, 0.15579787, 0.05361883, 0.07626446, -0.02815951, 0.042865 , 0.06312688, 0.07170736, 0.185382 , 0.06517194, 0.0584194 , -0.07202493, -0.09540485, -0.08599403, -0.03540141, -0.10775873, 0.03609048, -0.06903887, -0.10958587, -0.09116477, -0.10746475, -0.05029682, 0.07927579, 0.10405283, 0.12020227, 0.03288461, -0.1196027 , 0.03575619, 0.0972458 , 0.05244093, 0.08934712, 0.12486867, 0.08823898, 0.0983156 , 0.10495333, -0.10362058, 0.09903006, 0.09088268, -0.12305731, 0.10125972, -0.08239742, -0.00916524, -0.02156379, -0.09864786, 0.10193784, -0.09551246, -0.05185878, -0.09967213, -0.10428255, 0.10507633, -0.14955677, -0.12777895, -0.07334802, 0.16633663, -0.04327126, 0.11767196, 0.08965011, 0.17071536, -0.0584787 , -0.09368757, -0.16054668, -0.16359754, 0.0472418 , 0.1223535 , -0.07023636, 0.24043013, 0.15963615, -0.01664593, 0.15456486, 0.11985014, 0.25587445, 0.17214614, -0.15644138, 0.02051393, -0.15587153, 0.03782795, 0.0117235 , 0.04015777, 0.17542166, -0.1388207 , 0.09689201, -0.11741312], dtype=float32), kernel: Array([[-0.06625638, -0.10676572, -0.11858384, ..., 0.12592293, 0.19599795, -0.10273615], [ 0.19458613, -0.08272579, -0.01620658, ..., 0.07205058, 0.00837682, 0.03907323], [ 0.17456186, -0.00289036, -0.24084125, ..., -0.03393131, -0.21872245, 0.21209157], ..., [-0.10240208, -0.01977782, -0.14529926, ..., 0.48570195, 0.29290506, -0.28927922], [ 0.5828709 , 0.14644368, 0.23230499, ..., -0.51522744, -0.00893875, 0.11504804], [-0.18426676, -0.12852795, -0.02424787, ..., -0.17730054, -0.39859936, 0.3791855 ]], dtype=float32), }, }, output: { dense: { bias: Array([-1.47393912e-01, 1.51925683e-01, -1.37644792e-02, -5.88501468e-02, -2.74660420e-02, 2.99488921e-02, 6.36566132e-02, -3.54291759e-02, -7.69458618e-03, 3.24805588e-01, -5.66644669e-02, 2.47724857e-02, -1.12583041e-02, -9.56679955e-02, 1.36414379e-01, 3.80160026e-02, 4.16892134e-02, -1.21877357e-01, -9.89358947e-02, 8.71492922e-02, -5.38453227e-04, 6.59572333e-02, -4.53343103e-03, -4.99130785e-02, 4.46245968e-02, 1.36791214e-01, -3.86863551e-03, 1.96640324e-02, 6.48279041e-02, 6.96821213e-02, -2.48288596e-03, -8.72585997e-02, 2.33484972e-02, -5.56293987e-02, 6.42355978e-02, -3.93086448e-02, -5.08882850e-03, -8.52516666e-02, 1.37223052e-02, -1.85753062e-01, 6.52511492e-02, -4.16962616e-03, -6.68387711e-01, -2.01383736e-02, -1.15884997e-01, 1.01936243e-01, 4.85421829e-02, 2.81445622e-01, 9.36387926e-02, -4.22131829e-02, -2.58780066e-02, -9.31250453e-02, -8.50194786e-03, 1.86161455e-02, 1.08278200e-01, -1.33628231e-02, -2.95319203e-02, -7.03121051e-02, 1.65721059e-01, 1.23157077e-01, 1.26239538e-01, 8.69110152e-02, -6.91822842e-02, -6.16661599e-03, -4.39871997e-02, 6.22018203e-02, -9.05643180e-02, 7.91532993e-02, -6.81676865e-02, -5.36848418e-02, 2.38579530e-02, 2.05037221e-02, 1.06868640e-01, -3.20243724e-02, 4.63792652e-01, 2.49073096e-02, -1.17389895e-01, 3.09424363e-02, -7.55871981e-02, -5.58793545e-02, -3.00298661e-01, 3.98461558e-02, -5.11545166e-02, -3.12989019e-02, 6.69162199e-02, 7.48850405e-03, 8.32765177e-02, 1.84600934e-01, 3.32334414e-02, 4.34078313e-02, 3.89995314e-02, 1.41597819e-02, -9.58840549e-02, 1.53720230e-02, -5.27216345e-02, 6.26715794e-02, -1.02486372e-01, -5.28356284e-02, -1.23486230e-02, 9.72601771e-02, -4.58076522e-02, 1.89271364e-02, 6.57725483e-02, 7.59906247e-02, -7.66134709e-02, -4.67620529e-02, 6.40749931e-02, 2.11690012e-02, -8.14582258e-02, -7.82966167e-02, 3.28965578e-03, -1.84342973e-02, 1.21189028e-01, -2.12921515e-01, -4.28700894e-02, -5.90704009e-02, 1.14371911e-01, -1.21928066e-01, -3.76502164e-02, -3.38824801e-02, -6.17372356e-02, -1.77653898e-02, 1.73333704e-01, 8.95729363e-02, -6.20543491e-03, 2.95204688e-02, 2.87007019e-02, -4.14825082e-02], dtype=float32), kernel: Array([[ 0.02049902, -0.1422533 , -0.4065625 , ..., 0.36780554, 0.4333429 , 0.19315735], [-0.03959692, 0.25549796, -0.06924722, ..., -0.55895936, -0.27878842, 0.00849131], [ 0.14230408, 0.490068 , -0.18883815, ..., -0.30614755, -0.30285606, 0.07531235], ..., [ 0.0692085 , 0.07044961, 0.61252284, ..., -0.23293641, -0.15725587, -0.13308391], [ 0.19255584, -0.04722506, 0.23870187, ..., 0.26104337, -0.1339734 , -0.28957063], [-0.11599459, 0.07947025, -0.35309428, ..., -0.22296615, 0.19523388, 0.4148677 ]], dtype=float32), }, }, }, layernorm_after: { bias: Array([-1.27752021e-01, -1.48293898e-01, -1.61956966e-01, 2.83768959e-03, -1.43960312e-01, 4.62501980e-02, 1.55017406e-01, -1.05546750e-02, 1.24534264e-01, 3.37903023e-01, -4.59039863e-03, -1.85061380e-01, 1.73028678e-01, 2.01043487e-02, 2.81281001e-03, 9.39656422e-02, -1.36595475e-03, -2.31539598e-03, -1.25028998e-01, 2.32474544e-04, -2.05569100e-02, -1.70720860e-01, 9.77594405e-03, 1.18461205e-02, 2.47885808e-01, -1.36246130e-01, -1.46172762e-01, -6.74006566e-02, 1.35634257e-03, -6.88059255e-02, -1.38106868e-01, -3.34430300e-02, 8.12808890e-03, 9.31725744e-03, 2.75329649e-02, -1.16437320e-02, -1.76476855e-02, -1.26601487e-01, -3.02024838e-02, -5.67115694e-02, 9.60008055e-02, 8.26168135e-02, -5.39703900e-03, 3.32480557e-02, -1.42446086e-01, 2.15325981e-01, 3.59231502e-01, 6.70243725e-02, 5.95594849e-03, 2.55793538e-02, 1.20438235e-02, -1.38192281e-01, -9.54014882e-02, -6.51674941e-02, 1.10238351e-01, -9.56654698e-02, 9.43305194e-02, -7.60812163e-02, -7.58638158e-02, 1.66560858e-01, 1.86092202e-02, 1.56081989e-01, 5.11778006e-03, -4.85944413e-02, 8.28391612e-02, 5.49042262e-02, -3.91565077e-02, -2.07471265e-03, 3.06699127e-02, 3.01516578e-02, -3.39646004e-02, -2.84999385e-02, 7.43485242e-02, 1.12809241e-02, -1.99580923e-01, -5.58756031e-02, 2.26514554e-03, -5.92570044e-02, -5.33170663e-02, 4.89857085e-02, 9.51478072e-03, -2.19953805e-02, -3.24405394e-02, -1.16561711e-01, -7.09283957e-03, -4.68143495e-03, 5.30386604e-02, -1.35984283e-03, 7.81681612e-02, -5.89879602e-03, -1.73093099e-03, -6.07467350e-03, -3.00015986e-01, -3.05419937e-02, -1.54022276e-02, -6.91811442e-02, -5.47736287e-02, 2.07600649e-02, 1.68445095e-01, -1.03444420e-02, -1.04541652e-01, 5.39848953e-02, -2.24056616e-01, 1.65632993e-01, -1.40168756e-01, -4.03944461e-04, -4.72264551e-02, 1.25392643e-03, 6.72694370e-02, -2.77178794e-01, -4.07134034e-02, 8.86196047e-02, 4.30512354e-02, -1.80928916e-01, -2.78730467e-02, 5.36768697e-02, 2.03331217e-01, -2.78739282e-03, 1.01259798e-01, 6.90696239e-02, -9.43698585e-02, 7.54763791e-03, 1.15448259e-01, 2.77357996e-01, 1.10673435e-01, 1.47797644e-01, -9.77296499e-04, -1.15336455e-01], dtype=float32), scale: Array([ 1.0143379 , 0.56084377, 0.61511797, 0.875825 , 0.7977206 , 0.86066914, 0.4466555 , 0.5817147 , 0.8288987 , 1.2778934 , 0.14663759, 0.92707175, 0.29109854, -0.5407728 , -0.00583301, 0.80144596, 0.39370012, -0.01780795, 0.55510044, 0.8787668 , 0.8118457 , 0.7212926 , 0.22898938, 0.00633279, 1.0344126 , 0.4334817 , 1.2401807 , 0.45261577, 0.04603433, 0.5680823 , 0.6459878 , 0.6229751 , 0.7747696 , 0.87726367, 0.4118688 , 0.02244997, 0.7460163 , 0.53923315, 0.5421808 , 0.90724677, 0.7915582 , 0.3890947 , 0.7099728 , 0.94361395, 0.44176197, 0.708961 , 0.51787186, 1.1014241 , 0.3872233 , 0.6767877 , 0.7677383 , 0.61901575, 0.73735505, 0.28867054, 0.81986 , 0.68635523, 0.40406612, 0.7550665 , 0.3736026 , 0.42338955, 0.51958174, 0.75038564, 0.10604075, 0.47208443, 0.53189516, 0.43309155, 0.37580943, 0.48088777, 0.4441973 , 0.5321363 , 0.5055118 , 0.98682684, 0.7596939 , 0.835741 , 0.10439195, 0.57524043, 0.43323192, 0.71470016, 0.38151345, 0.8561705 , 0.6086962 , 0.8476424 , 0.68529296, 0.81576556, 1.0155932 , 0.2647536 , 0.66374 , 0.8123001 , 0.43478364, 0.7506543 , 0.70008427, 0.01624053, 0.5848578 , 0.672363 , 0.5528674 , 0.17722914, 0.7533496 , 0.50877273, 0.87119836, 0.50191104, 0.70362985, 0.7272638 , 0.79176766, 1.0370609 , 0.6759593 , -0.02001736, 0.8079056 , 0.76534927, 0.6989189 , 0.55686975, 0.8860217 , 0.43745935, 0.6700682 , 0.90106094, 0.05214773, 0.5131126 , 0.7131446 , 0.76999676, 0.6244328 , 0.87544256, 0.54827476, 0.00163172, 0.72723454, 0.770018 , 0.62415475, 0.49478632, 0.32370028, 0.52392954], dtype=float32), }, layernorm_before: { bias: Array([ 5.46789803e-02, -1.76678717e-01, -3.22135426e-02, -1.92919653e-02, -2.71175429e-03, -5.60209937e-02, 3.75961289e-02, 2.42887228e-03, 2.20702612e-04, -1.47421882e-01, 1.25962566e-03, -9.49192513e-03, 3.36418934e-02, 1.43754203e-02, 2.79071257e-02, 3.28558236e-02, 1.25313839e-02, 5.57482056e-03, 2.42552720e-03, -4.17415686e-02, -6.24468969e-03, -7.01998994e-02, 6.89587668e-02, -7.22786319e-03, 1.22491186e-02, -1.73403710e-01, 4.65809219e-02, -2.40579285e-02, 4.88756225e-04, 1.74035388e-03, 6.00601882e-02, -2.83498946e-03, -4.87266015e-03, -6.68269256e-03, 2.96081766e-03, -4.06322069e-02, -5.13824681e-03, -5.18148541e-02, -6.03306864e-04, 3.28535922e-02, 5.79561479e-03, -5.60164126e-03, 2.76499480e-01, -5.52190188e-03, 1.08755060e-01, 2.78750118e-02, -1.74138229e-03, -4.13134582e-02, -1.12804314e-02, 1.25732794e-02, -2.00521722e-02, 3.98424082e-02, 1.83249754e-03, -1.12617195e-01, 8.08609067e-04, 6.40430721e-03, 4.37817909e-03, 2.31746268e-02, -2.95173544e-02, 5.21452923e-04, 5.27834482e-02, 4.03346457e-02, -6.75705774e-03, -1.17142848e-03, 2.27539521e-02, 8.85420591e-02, -6.18154998e-04, 2.34325184e-03, 2.68001352e-02, 3.51031497e-03, -3.19339558e-02, 1.70021728e-02, 5.05615212e-02, -3.64386290e-02, -5.41854799e-01, -3.14815044e-02, -8.37918895e-04, -5.96139133e-02, -5.71792945e-03, 2.48509813e-02, 5.07591176e-04, -9.80351865e-03, -4.27319445e-02, -4.39409167e-02, -2.25920565e-02, 5.03404811e-03, 5.27535053e-03, -2.32479889e-02, 8.92616529e-03, 2.46615335e-02, -1.04511445e-02, 4.72732633e-02, 2.06987560e-03, -4.62427586e-02, 2.54220869e-02, -2.62153684e-03, -1.93837471e-02, 3.46671464e-03, 9.97190922e-02, 5.05104400e-02, -1.72052216e-02, 5.25105570e-04, -1.76775366e-01, 3.32803018e-02, -2.44557224e-02, -1.54652279e-02, -1.59627572e-02, 3.77600156e-02, -2.54226532e-02, -1.87684055e-02, -8.18012469e-03, 6.22503310e-02, -9.38802143e-04, 4.75377999e-02, -9.10606887e-03, 2.95884646e-02, 1.39154634e-02, -4.97860415e-03, 5.38583286e-03, 2.14410182e-02, -7.47090429e-02, 2.04133317e-02, 1.69800371e-02, 5.70605770e-02, -5.42162033e-03, -4.40403894e-02, 3.08437683e-02, -1.20276608e-01], dtype=float32), scale: Array([ 3.09920963e-02, 1.43276036e-01, 1.49911180e-01, 2.15380549e-01, 8.57652703e-05, 2.60982364e-01, 3.85197014e-01, -2.69828960e-02, -3.57246934e-03, 1.77718669e-01, 1.78545862e-02, 2.83004701e-01, 4.92948219e-02, 2.25244924e-01, 1.80071101e-01, 3.60851258e-01, 2.13657439e-01, 6.50591101e-04, -1.27726868e-02, 8.39237347e-02, 1.44251525e-01, 3.37201595e-01, 2.32133791e-01, 3.55153927e-03, 1.30111605e-01, 3.19995940e-01, 1.58201158e-01, 1.93055466e-01, 1.13361664e-02, 1.06982812e-02, 2.68632799e-01, 1.73866183e-01, 2.49319479e-01, 2.33700909e-02, -3.93300736e-03, 1.54559046e-01, 7.23777190e-02, 2.99003601e-01, 2.09477082e-01, 1.43370494e-01, 2.27352500e-01, 2.16101669e-02, 1.23486273e-01, 3.68611127e-01, 2.57967263e-01, 3.01627338e-01, 2.64851768e-02, 3.02257270e-01, -8.79388452e-02, 1.15935393e-01, 3.20747286e-01, 3.84979308e-01, 2.91317515e-02, 3.35988641e-01, 1.00166025e-03, -2.34178808e-02, 1.65450841e-03, 9.69321802e-02, 1.97463453e-01, 2.78218836e-02, 3.64540905e-01, 4.77716655e-01, -1.46322930e-02, 3.61578852e-01, 2.67611772e-01, 1.56464770e-01, -1.12778105e-01, 3.55959013e-02, 2.19024077e-01, 5.52458549e-03, 2.59881735e-01, 2.20885038e-01, 2.75174618e-01, 2.36324608e-01, 1.31579578e-01, 2.25922078e-01, 2.05405220e-01, 2.81007886e-01, -3.42874392e-03, 1.95928365e-01, 1.01560242e-02, 1.41977131e-01, 2.52693713e-01, 2.46991277e-01, 1.85481176e-01, -2.18331348e-02, -1.25332624e-02, 1.17323473e-01, 6.74787387e-02, 1.07466944e-01, 1.33154258e-01, 1.68752119e-01, -5.74604236e-03, 2.33412385e-01, 1.67403102e-01, -5.18426392e-03, 1.57531217e-01, 1.38286993e-01, 3.68412942e-01, 2.31308982e-01, 1.48976266e-01, 7.62487238e-04, 3.20091337e-01, 1.83057398e-01, 2.31623769e-01, 2.69163251e-01, 1.83478698e-01, 3.07111442e-01, 3.13205153e-01, 2.52505481e-01, 3.30679387e-01, 3.39435637e-01, 9.45753418e-03, 4.02425140e-01, 6.86238613e-03, 2.50461936e-01, 3.17101419e-01, 4.53172848e-02, 6.39474615e-02, 2.83503503e-01, 1.75329089e-01, 1.88262612e-01, 1.82763785e-01, 3.37596089e-01, 3.82400723e-03, 2.11784661e-01, 2.38452464e-01, 3.09545726e-01], dtype=float32), }, moe: { shared_experts: { intermediate: { bias: Array([-3.9847344e-01, -3.6854276e-01, -4.5092517e-01, -4.3007284e-01, -4.8035741e-01, 1.8460776e-03, -4.3567291e-01, -4.2967001e-01, -3.3667952e-01, -4.1538629e-01, -3.6099383e-01, -4.3359479e-01, -4.4380239e-01, -5.5524331e-01, -3.6870196e-01, -2.4764657e-01, -4.7775912e-01, -4.1283187e-01, -3.4256911e-01, -2.0356853e-01, -4.6999124e-01, -5.6414276e-01, -3.3079070e-01, -3.8981372e-01, -4.5676839e-01, -8.1164964e-02, -2.3958890e-01, -4.1569802e-01, -3.3400738e-01, -3.1094922e-02, -4.0657026e-01, -3.2647237e-01, -5.5474406e-01, -5.1273751e-01, -4.6838763e-01, -4.5339519e-01, -3.2953310e-01, -4.2982268e-01, -4.1318712e-01, -4.8618275e-01, -3.3651689e-01, -3.8407576e-01, -3.7777162e-01, -4.4129571e-01, -2.5866541e-01, -4.4932291e-01, -4.2343855e-01, -3.6280292e-01, -3.8582063e-01, -3.7247223e-01, -7.4966386e-02, -3.5159770e-01, -4.0223166e-01, -1.5560038e-01, -3.4897429e-01, -3.5570183e-01, -2.3108204e-01, -3.0199406e-01, -5.0925249e-01, -4.2046854e-01, -5.7922322e-01, -4.1019771e-01, -4.2622930e-01, -4.4688463e-01, -4.6536162e-01, -3.6277786e-01, -2.4544826e-01, -3.0414537e-01, -4.6335497e-01, -3.0622986e-01, -3.6244193e-01, -1.3396668e-01, -4.9729797e-01, -4.0432706e-01, -1.7137310e-01, -1.5809147e-01, -4.3105653e-01, -2.7127942e-01, -3.0011219e-01, -2.2747953e-01, -3.7861067e-01, -2.8296992e-01, -4.9367398e-01, -1.4615911e-01, -3.4952867e-01, -2.4669057e-01, -4.6957275e-01, -3.5128793e-01, -3.9153334e-01, -4.2859459e-01, -4.0044680e-01, -4.3775627e-01, -3.7979439e-01, -5.2443427e-01, -1.5687215e-01, -4.4120738e-01, -4.7335675e-01, -3.9626312e-01, -3.6113608e-01, -8.2696810e-02, -2.6075220e-01, -1.9168627e-01, -4.8281315e-01, -1.7972828e-01, -5.2597886e-01, -2.7033496e-01, -5.4604769e-01, -3.4184673e-01, -4.3216091e-01, -4.4862813e-01, -2.6511189e-01, -4.5229965e-01, -3.8147661e-01, -3.2180479e-01, -3.1602433e-01, -4.3689847e-01, -5.1146489e-01, -3.5368246e-01, -3.7338638e-01, -3.0413544e-01, -2.4527813e-01, -4.7685048e-01, -2.5364643e-01, -3.0931181e-01, -3.2056129e-01, -3.0224264e-01, -3.6615273e-01, -3.6344633e-01, -4.7639742e-02, -4.2077422e-01, -4.8495576e-01, -3.2529086e-01, -3.9314100e-01, -3.9539710e-01, -4.4750559e-01, -2.1847221e-01, -3.4810659e-01, -3.5670453e-01, -4.8384792e-01, -4.6719027e-01, -2.9682928e-01, -4.2099711e-01, -4.7452945e-01, -4.3598908e-01, -4.5502350e-01, -4.7916898e-01, -5.1886410e-01, -2.9352316e-01, -4.6964645e-01, -4.7157702e-01, -2.2555082e-01, -3.9752248e-01, -7.7490084e-02, -1.6522288e-01, -4.9982330e-01, -4.2842680e-01, -3.2544607e-01, -3.0850667e-01, -4.0218577e-01, -4.1208234e-01, -4.0038344e-01, -3.7057555e-01, -2.2373217e-01, -3.4376973e-01, -3.1466734e-01, -5.2123147e-01, -5.3340238e-01, -2.8239700e-01, -3.3951777e-01, -4.2198521e-01, -4.4805121e-01, -2.5464025e-01, -1.5028036e-01, -4.1943482e-01, -4.2202288e-01, -3.9689219e-01, -3.7402752e-01, -1.6487882e-01, -3.0544364e-01, -4.0235436e-01, -4.1456652e-01, -3.5549703e-01, -4.5845604e-01, -3.7517163e-01, -4.1666454e-01, -2.5526059e-01, -4.9071726e-01, -3.1533489e-01, -3.4268734e-01, -4.6997485e-01, -3.2526866e-01, -4.7221246e-01, -3.2835072e-01, -5.4744834e-01, -4.1387323e-01, -2.6110759e-01, -4.3684092e-01, -4.7436625e-01, -3.4099048e-01, -3.7735420e-01, -4.5077652e-01, -5.1907909e-01, -3.7325633e-01, -2.2339475e-01, -2.8594923e-01, -4.6022233e-01, -3.7742871e-01, -3.5255402e-01, -1.6996916e-01, -3.3114195e-01, -2.1370201e-01, -4.9245229e-01, -4.8635155e-01, -4.3522072e-01, -3.8971159e-01, -4.3678322e-01, -4.6781665e-01, -2.9914606e-01, -4.2043447e-01, -2.9230228e-01, -2.6290324e-01, -2.3281787e-01, -5.2061099e-01, -3.1569782e-01, -5.0653201e-01, -4.7192627e-01, -3.8380232e-01, -4.1764271e-01, -3.3932805e-01, -4.6573785e-01, -4.0508986e-01, -1.6529684e-01, -3.9876980e-01, -4.5229152e-01, -2.8774762e-01, -3.5973960e-01, -3.5486862e-01, -2.4070273e-01, -2.9540777e-01, -2.9138458e-01, -3.0258924e-01, -3.0566293e-01, -3.6295635e-01, -2.6400667e-01, -3.0755830e-01, -2.9665276e-01, -3.2570139e-01, -3.8138995e-01, -2.5602007e-01, -3.7541685e-01, -4.3872544e-01, -4.1538125e-01, -4.2229000e-01, -3.4041786e-01, -3.0963835e-01, -4.1050071e-01, -5.6023502e-01, -5.6029701e-01, -3.5283986e-01, -2.6442611e-01, -3.5243306e-01, -3.4913927e-01, -2.1650518e-01, -4.0154454e-01, -3.7275749e-01, -4.7750309e-01, -3.6822703e-01, -4.5743442e-01, -4.8160750e-01, -4.7312003e-01, -3.9876688e-01, -5.0516546e-01, -5.4552037e-01, -3.4587166e-01, -1.6361688e-01, -3.0174541e-01, -3.3792850e-01, -4.7918764e-01, -4.8411125e-01, -4.3018821e-01, -4.6167120e-01, -4.1177577e-01, -4.5708910e-01, -4.5317286e-01, -2.9856035e-01, -3.5552701e-01, -3.3068407e-01, -4.0677980e-01, -4.2047119e-01, -2.2582465e-01, -5.4681146e-01, -3.3787546e-01, -4.4846484e-01, -3.3120909e-01, -3.4642521e-01, -3.9533862e-01, -3.4406123e-01, -4.5375788e-01, -5.0441438e-01, -3.3547118e-01, -2.8920132e-01, -1.0944715e-01, -3.5370487e-01, -3.3076125e-01, -2.8330564e-01, -3.6274710e-01, -2.7184573e-01, -3.5993487e-01, -4.0513000e-01, -3.0320236e-01, -2.5303522e-01, -4.7531182e-01, -4.6395004e-01, -4.2257047e-01, -4.8910224e-01, -3.6540136e-01, 1.3740684e-01, -3.0950975e-01, -4.7773069e-01, -3.0934384e-01, -4.1294369e-01, -4.5992157e-01, -4.8535463e-01, -4.4464013e-01, -3.5180721e-01, -3.6469987e-01, -4.2818370e-01, -5.0174421e-01, -3.8951018e-01, -3.7477994e-01, -3.4967715e-01, -4.3007034e-01, -4.4603419e-01, -3.4457707e-01, -4.6689948e-01, -3.0864301e-01, -4.9847844e-01, -3.7400711e-01, -4.7358903e-01, -3.5766226e-01, -2.8434107e-01, -2.7686146e-01, -3.4817812e-01, -3.6055216e-01, -4.3933406e-01, -5.2891338e-01, -2.7604723e-01, -5.1344776e-01, -2.9326063e-01, -5.0666833e-01, -3.3879602e-01, -4.9309245e-01, -3.0533355e-01, -5.3628761e-01, -4.3473902e-01, -4.1327205e-01, -1.9567779e-01, -4.3977755e-01, -4.7691789e-01, -3.4455177e-01, -4.1720390e-01, -3.4753558e-01, -4.0889883e-01, -4.2685688e-01, -3.7366074e-01, -1.6097990e-01, -3.0323794e-01, -4.8915049e-01, -3.5860199e-01, -3.2890522e-01, -3.6635777e-01, -3.0436921e-01, -4.3726847e-01, -1.7787141e-01, -3.3810359e-01, -4.7570062e-01, -5.1493889e-01, -4.9520645e-01, -4.8514831e-01, -4.0491089e-01, -5.2531552e-01, -4.0086076e-01, -5.0189817e-01, -4.7050786e-01, -2.1616998e-01, -3.5684329e-01, -4.2839581e-01, -3.8793033e-01, -2.3242010e-01, -3.5336223e-01, -3.3516747e-01, -3.7301928e-01, -4.4247723e-01, -4.8835176e-01, -2.0324853e-01, -3.9160007e-01, -3.3982515e-01, -2.5195253e-01, -5.4986537e-01, -3.1654030e-01, -4.6651766e-01, -4.7360784e-01, -2.8256953e-01, -3.3791304e-01, -3.5749507e-01, -3.0531624e-01, -4.9014235e-01, -4.1924736e-01, -2.0831045e-02, -3.5200366e-01, -2.2583617e-01, -4.5401311e-01, -3.8651881e-01, -4.5588291e-01, -4.9305251e-01, -1.7810422e-03, -2.6118660e-01, -3.1825066e-01, -2.4539576e-01, -2.5099796e-01, -5.3763103e-01, -3.6526870e-02, -2.3981811e-01, -3.0151102e-01, -4.7348249e-01, -2.2897437e-01, -4.1141531e-01, -4.8857701e-01, -3.2772517e-01, -5.7252842e-01, -4.5862877e-01, -3.6546475e-01, -3.6844012e-01, -2.0294371e-01, -4.8403230e-01, -4.8141652e-01, -1.4463230e-03, -4.9992949e-01, -3.6349079e-01, -2.1532026e-01, -4.1290241e-01, -4.8122567e-01, -2.8661153e-01, -4.4909769e-01, -3.6796397e-01, -2.6207677e-01, -3.9517501e-01, -4.4621187e-01, -3.6207435e-01, -4.8050162e-01, -3.7443826e-01, -2.3039897e-01, -3.5041386e-01, -3.9968762e-01, -3.6701423e-01, -3.8556710e-01, -3.4414786e-01, -4.8362854e-01, -3.8469809e-01, -4.8367801e-01, -3.7582186e-01, -4.8235339e-01, -3.0144668e-01, -3.7805808e-01, -3.9068311e-01, -4.6765688e-01, -4.5509905e-01, -4.1643125e-01, -4.3739045e-01, -5.5417305e-01, -3.9493406e-01, -4.5592216e-01, -3.2413527e-01, -4.4394735e-01, -2.5142363e-01, -2.9339787e-01, -4.6713012e-01, -2.5261605e-01, -2.6347214e-01, -3.7425596e-01, -2.1422139e-01, -4.4427580e-01, -3.5874227e-01, -4.1653034e-01, -3.3887890e-01, -3.0552921e-01, -2.4492532e-01, -4.0205300e-01, -3.2229105e-01, -4.8967350e-01, -2.1930854e-04, -3.7214684e-01, -3.0033603e-01, -4.7503376e-01, -3.7032735e-01, -3.4375238e-01, -3.8198787e-01, -3.3686790e-01, -5.5172253e-01, -3.0320832e-01, -1.3836212e-01, -3.9621928e-01, -3.8446391e-01, -4.3740526e-01, -3.8315192e-01, -4.0578344e-01, -3.2919899e-01, -4.7428545e-01, -4.4342765e-01, -4.8948532e-01, -2.3972549e-01, -3.4704226e-01], dtype=float32), kernel: Array([[ 0.10745826, 0.48113093, -0.04176594, ..., -0.36369812, 0.15021859, -0.05807315], [-0.58055264, -0.5796294 , -0.19744751, ..., -0.07468567, -0.51942426, -0.03569252], [ 0.01138834, 0.26800624, -0.01454838, ..., -0.8072465 , 0.5447465 , 0.36949006], ..., [ 0.36528957, 0.27883154, -0.3109063 , ..., -0.5552432 , 0.23742002, 0.05708292], [-0.09702465, 0.6521357 , 0.16876654, ..., -0.252267 , -0.09898252, -0.03783756], [-0.05449405, -0.4971733 , -0.02605094, ..., 0.03078811, 0.16231588, 0.4204838 ]], dtype=float32), }, output: { bias: Array([-1.16826370e-01, 1.19446300e-01, -3.04981880e-03, -1.48974463e-01, 5.59353642e-02, 7.24940300e-02, 3.09443232e-02, -3.97321768e-02, -1.61944836e-01, -1.69336453e-01, -5.43983765e-02, 1.14368267e-01, -1.82648357e-02, -5.84783889e-02, 2.02381879e-01, -1.69515777e-02, 4.80070449e-02, -1.26781985e-01, -6.17002137e-02, 3.90345929e-03, -1.18189584e-02, 1.06815845e-01, 2.18161605e-02, -5.02868928e-02, -1.54914372e-02, 2.92950004e-01, 3.89679447e-02, 4.18758281e-02, 6.82518259e-02, 8.17627087e-02, 2.73418054e-02, -7.99397752e-02, 2.45839432e-02, 1.14206569e-02, 3.90483811e-02, -3.62712778e-02, 2.89402343e-02, -7.93813691e-02, 1.43112317e-01, -1.52107775e-01, 1.80989935e-03, -1.87356304e-02, -5.44708908e-01, -9.66665521e-03, -5.63648678e-02, 1.56991724e-02, -2.02740412e-02, 5.39821014e-02, 1.57445461e-01, -2.57236585e-02, -4.68497947e-02, -2.43687704e-02, 9.51719135e-02, 7.29279891e-02, 1.93980023e-01, 1.36569049e-02, 6.77157715e-02, -6.52143881e-02, 2.89444447e-01, 1.13720141e-01, 1.87333062e-01, 1.36707956e-02, -7.90089220e-02, 1.85856335e-02, -5.92513643e-02, 7.04728812e-02, 8.90799426e-03, 1.34663433e-01, -6.09093085e-02, -2.86442898e-02, 3.39895934e-02, -6.57519400e-02, -6.73585013e-02, -5.29800951e-02, 5.72383285e-01, 7.09643438e-02, -1.11088164e-01, 3.76055352e-02, -4.64858226e-02, 1.60521493e-04, -2.10771397e-01, -1.34013280e-01, -9.27246362e-02, -2.77775893e-04, 1.43522322e-01, 4.07825224e-02, 5.58940060e-02, 1.22946128e-01, -1.75536349e-01, 2.35335454e-01, 8.74865577e-02, 1.05813751e-02, 1.50552183e-01, 9.60319340e-02, 6.48361864e-03, 6.70408085e-02, -1.03584118e-01, -6.56064600e-02, -5.26508018e-02, 1.28201723e-01, 3.76123935e-04, -2.51332112e-02, 8.39170814e-02, 3.41377631e-02, -2.44789012e-02, -2.70240922e-02, 5.39324842e-02, -2.40320619e-02, -9.49936584e-02, -1.08399363e-02, 8.49590078e-02, -4.56464440e-02, -1.49793148e-01, 6.02403209e-02, -2.90313717e-02, -4.08411808e-02, 2.29514278e-02, -1.53553560e-01, -5.11574633e-02, 3.26815434e-02, 1.73677616e-02, -2.31411960e-02, 1.70279935e-01, -1.79555595e-01, -7.74298981e-02, 9.48442426e-03, 4.52735312e-02, 9.07007884e-03], dtype=float32), kernel: Array([[ 0.49678656, -0.04703587, 0.14592949, ..., 0.14652656, 0.4102155 , 0.43552795], [ 0.12093473, 0.28034988, 0.05580731, ..., -1.0471447 , 0.00430787, -0.37116638], [ 0.4650048 , 0.25468072, 0.04566461, ..., -0.30497208, -0.02476864, 0.01837519], ..., [ 0.13562478, -0.04827686, 0.8059429 , ..., -0.31421518, 0.24673045, -0.4773905 ], [-0.73090583, -0.39090383, 0.1884505 , ..., 0.3404823 , 0.5766392 , 0.37859675], [-0.57174814, 0.14778472, -0.11844227, ..., -0.2646688 , -0.18360172, -0.04887927]], dtype=float32), }, }, }, }, }, }, layernorm: { bias: Array([ 4.99482080e-02, -4.29396838e-01, 1.81105509e-02, 4.06332692e-05, 1.38564035e-01, -2.67413799e-02, 4.30011488e-02, -3.71168442e-02, -5.17193019e-01, -9.74547490e-03, -6.02933764e-02, 8.66996311e-03, 3.20597798e-01, -1.68491941e-04, 1.20097917e-04, -8.34563673e-02, 9.34075788e-02, -6.27656728e-02, -3.47385721e-05, -4.00960334e-02, 1.16589768e-02, 2.07135342e-02, 5.25931716e-02, -5.63471504e-02, 1.14345705e-04, -2.30510355e-04, 2.13737637e-02, -1.45103643e-03, 1.10674717e-01, 3.27529460e-02, 1.30938485e-01, 2.17286926e-02, -6.83023154e-06, -9.22439562e-04, -5.39473398e-03, -4.33185026e-02, -9.40410607e-03, -1.07025966e-01, -1.36162853e-03, 1.07634941e-03, 2.64154427e-04, -1.61221977e-02, 1.59604579e-01, 6.28793612e-04, 2.51234043e-03, -1.61393564e-02, -3.47336605e-02, -2.62861013e-01, -1.80254143e-03, 1.19618746e-02, -1.74341142e-01, -2.35029664e-02, 1.11634769e-01, -7.43765607e-02, 2.52770900e-04, 5.80298016e-03, 1.65572765e-05, -1.99696115e-05, 8.98374594e-07, 2.72183418e-02, 5.92560282e-05, 2.25074757e-02, -7.86222667e-02, 3.12634259e-02, -5.19049093e-02, 7.74146914e-02, -1.94988661e-02, 3.86598520e-04, 3.62606696e-03, 1.13942362e-02, -5.74776307e-02, 1.42280420e-04, -7.51663325e-03, -2.08205208e-01, -3.37208897e-01, 6.53023422e-02, -9.40454900e-02, 2.59398657e-04, -1.21128708e-01, 6.60678595e-02, 5.90708017e-01, -3.01386812e-04, -4.50036081e-04, -4.51468586e-05, 1.67313181e-02, 1.51245928e-04, 1.40742473e-02, 5.68821088e-06, 1.60795447e-04, 1.45528553e-04, 6.64302963e-04, 4.23284993e-02, -2.40530949e-02, 2.40640657e-06, -1.34843877e-02, 4.14691959e-03, -1.12099377e-02, -5.02210781e-02, 2.03482993e-02, 2.34702706e-01, 7.58242968e-05, -5.44176670e-04, -2.74804551e-02, 2.38151588e-02, -4.01919484e-02, -3.65443416e-02, -7.20889375e-06, -2.77027546e-04, -8.81933421e-02, 1.69912830e-01, 2.06964463e-02, 1.79356318e-02, -1.09324465e-04, 6.39357895e-04, 3.84408049e-02, 6.90272776e-04, -3.20791632e-01, 6.30142495e-06, 4.83152457e-02, -2.25914060e-04, -3.39954719e-02, -4.70114350e-02, 2.85215719e-05, 1.06935140e-05, -1.23181306e-02, 1.01582222e-01, 3.10630426e-02, -4.47405800e-02], dtype=float32), scale: Array([ 6.8868792e-01, 3.1249523e-01, 9.1828078e-01, -2.3013241e-04, 3.7263566e-01, 2.3251632e-01, 6.5610683e-01, 1.1425257e-01, 3.4526226e-01, 1.4145033e-02, 7.2997391e-01, 8.0623746e-02, 2.9320392e-01, -4.1230087e-05, -6.6874339e-04, 5.7173258e-01, 6.9446439e-01, 1.2802645e+00, -6.2002582e-05, 1.1703534e-01, 4.1233909e-01, 8.1602544e-01, 5.0138348e-01, 5.7336593e-01, 7.1710587e-05, 1.3667290e-05, 1.9022560e-01, 6.5288413e-01, 4.8426083e-01, 4.8938426e-01, 6.1096561e-01, 4.0365598e-01, 2.9291890e-05, 1.8095745e-03, 4.2584249e-01, 1.0999548e+00, 2.1008846e-01, 3.8091195e-01, 4.8326021e-03, -1.2936177e-04, -3.4556305e-04, 6.0092252e-01, 3.3731986e-02, 2.1269962e-03, 7.6969045e-01, 3.1354058e-01, 5.6419522e-01, 3.5723236e-01, 1.6803181e-01, 1.7196296e-01, 6.4958000e-01, 5.5659479e-01, 2.6263377e-01, 2.4760671e-01, -5.8708456e-04, 3.1410387e-01, -1.7643184e-04, 2.0361553e-04, -2.0005717e-05, 6.7649913e-01, -1.2435010e-04, 3.2834163e-01, 6.4273733e-01, 4.8385906e-01, 8.3392429e-01, 2.8222600e-01, 1.5083265e-01, -5.6320438e-03, 3.3680016e-01, 2.1632074e-01, 4.0965131e-01, 4.5164741e-04, 1.3090445e-01, 7.1241659e-01, 8.9404747e-02, 4.2915529e-01, 6.3035327e-01, -3.9689415e-04, 6.1234730e-01, 2.9691941e-01, 1.6158453e-01, 1.7273755e-04, 1.4721676e-04, -1.1007604e-04, 1.3981619e-01, 1.9772084e-04, 4.9367583e-01, 2.4138608e-06, -8.6720851e-05, 3.0593758e-03, 2.4073297e-01, 1.1165603e+00, 1.5985191e-01, 3.0023075e-04, 2.5869986e-01, 9.9650544e-01, 2.4187727e-01, 4.2451209e-01, 4.7070310e-01, 4.9373725e-01, -3.1352194e-04, 1.4548376e-04, 1.2102582e+00, 2.8354323e-01, 4.2492944e-01, 3.4629729e-01, 6.0579823e-06, -1.6931001e-03, 7.3870343e-01, 6.7030573e-01, 1.5534639e-01, 5.8719957e-01, -1.9493036e-04, 1.6623877e-03, 7.6086062e-01, 3.6656085e-01, 3.9590457e-01, 7.1911036e-06, 7.4346775e-01, -2.0176137e-04, 2.7498129e-01, 6.6088724e-01, -2.2207441e-06, -7.9591937e-06, 2.0538671e-01, 1.2334855e+00, 2.2934711e-01, 1.6446222e-01], dtype=float32), }, }, }) FrozenDict({ classifier: { bias: Array([ 0.13959047, -0.15330061, 0.23963794, 0.26991433, 0.14193657, -0.07303967, -0.15755221, -0.00429095, -0.1597117 , 0.01030699], dtype=float32), kernel: Array([[-0.21733038, -0.04500001, 0.6502397 , ..., -0.16637883, -0.19785312, -0.19238858], [-0.05325412, 0.08365323, 0.04513923, ..., 0.04858669, -0.2313862 , 0.15936573], [ 0.15941077, -0.5403379 , 0.17727152, ..., 0.23605543, -0.5477616 , 0.400377 ], ..., [ 0.14339526, -0.40216786, -0.11720804, ..., -0.4565188 , 0.55370927, -0.3628201 ], [ 0.1330832 , -0.05788376, 0.05994943, ..., -0.05280861, -0.35830075, -0.07267395], [ 0.07371017, 0.0797717 , 0.05526711, ..., -0.15986513, -0.19333237, -0.11412273]], dtype=float32), }, vit: { embeddings: { cls_token: Array([[[ 4.21749912e-02, -7.63344541e-02, -9.38936546e-02, 3.23799951e-03, 1.69068575e-01, 2.25809757e-02, 3.44559725e-04, -3.61712719e-03, -8.03779662e-02, 3.50343175e-02, -2.25252569e-01, 8.41705650e-02, -1.40322876e+00, -6.44051209e-02, 3.56393009e-02, 1.55636281e-01, 1.44712580e-02, -1.67391926e-01, 1.13313071e-01, 2.03049585e-01, -1.21489443e-01, 3.01832557e-01, -6.39758036e-02, 2.43316695e-01, 8.19623619e-02, 3.64291817e-01, 2.20934585e-01, 4.44887951e-02, -1.20811634e-01, 7.45030791e-02, 2.88756996e-01, 2.45113447e-01, -1.02346905e-01, -1.44495424e-02, -2.92423535e-02, 1.27275869e-01, -4.68343794e-02, 6.18743040e-02, 1.62268832e-01, 7.68043697e-02, 8.22397470e-02, 4.15349565e-02, 4.29975651e-02, 3.34680080e-01, 1.23265073e-01, -1.82213470e-01, -2.61384398e-01, -2.86995154e-02, 4.78795953e-02, -7.12198839e-02, -9.28327814e-02, -1.04730994e-01, -2.08872005e-01, 5.08915447e-02, -2.92177856e-01, -4.63173315e-02, -1.39276087e-01, -6.45741746e-02, 4.31298822e-01, -2.79950589e-01, -3.42224717e-01, -2.57990032e-01, 7.88487419e-02, 1.75719306e-01, -1.91833928e-01, -1.60736710e-01, 2.24118829e-01, -1.42223332e-02, -1.56884298e-01, -1.34610429e-01, -7.02320859e-02, -5.77486083e-02, -2.67209888e-01, -1.95046023e-01, 1.72840059e-01, 5.77186942e-02, 1.49908558e-01, 1.20706670e-01, 2.60736823e-01, -4.85861525e-02, 1.50602728e-01, -1.42758638e-01, 2.74297837e-02, 2.43728027e-01, 2.83344448e-01, -3.49160563e-03, -1.21281445e-01, -1.72878504e-01, -1.20296307e-01, -5.02357781e-02, 8.33122134e-02, -1.96546650e+00, 1.50719002e-01, 2.28078086e-02, -1.70492530e-01, 4.20110017e-01, -9.01997983e-02, -3.95133868e-02, 1.77724794e-01, 1.20884374e-01, -3.86571921e-02, -7.12739974e-02, 1.50522381e-01, 1.34570688e-01, 3.98968756e-01, -1.69036195e-01, 7.98634291e-02, -2.08218116e-02, -1.60120368e-01, -2.48059444e-02, -1.43607566e-02, 3.75894368e-01, 8.90320763e-02, -1.83609471e-01, 3.58941674e-01, -1.86423570e-01, 2.57358737e-02, 1.14504859e-01, -1.51107222e-01, -1.81861663e+00, -1.02884192e-02, -3.45049873e-02, 7.34117031e-02, 6.87620267e-02, 6.22403771e-02, 3.00719798e-01, 6.51065707e-02, 3.67412001e-01]]], dtype=float32), patch_embeddings: { projection: { bias: Array([-0.23706354, -1.6540319 , -0.56307954, 0.26430997, 0.45925793, 0.10770348, -0.64220166, -0.256316 , -0.70986354, 0.00280434, -0.8621466 , 0.33027068, -0.09428401, -0.18285578, 0.48031166, 0.31876177, 0.1595892 , -0.7937292 , 0.41048172, 0.7553594 , -0.11688971, 1.0029606 , -0.19344132, 0.9868722 , 0.33903363, 1.4018133 , 0.6315837 , 0.15693058, -0.43901056, -0.03160467, 1.4104275 , 0.8133011 , -0.8685446 , 0.09167413, -0.38324404, 0.4127237 , -0.0992351 , -0.8858117 , 0.49157575, 1.1184502 , 0.29291674, 0.09339268, 0.10110624, 1.3492149 , 0.23373768, -0.62969214, -1.1953127 , -0.32950965, 0.16054444, -0.04147059, -0.32538444, -0.44066548, -0.81780577, 0.3041742 , -0.88412094, -0.2175792 , -0.7703008 , -0.29075882, 1.4584533 , -0.90483326, -1.0301116 , -1.9739118 , 0.30468434, 0.47119713, -2.507297 , -0.4836628 , 0.7882562 , 0.13242652, -0.64575255, -0.5046555 , -0.2928836 , -0.19443981, -1.8653952 , -0.6447174 , 0.53096473, 0.53206563, 0.53241354, 0.5417067 , 0.87263006, -0.22854666, 0.57786745, -0.3563041 , -0.03160271, 0.5421835 , 1.0412699 , -0.1696869 , -0.4523725 , -1.6923577 , -0.46177727, -0.20988323, 0.2240901 , -0.9807764 , 0.61122894, -1.4953612 , -0.6922171 , 1.6215447 , -0.32785478, -0.6734319 , 0.5917648 , 0.33969748, -0.27886963, -0.50457156, 0.5443144 , 0.5749297 , 2.0102262 , 0.15956564, 0.03995752, 0.18428732, 0.1132739 , 1.1407704 , 0.10695937, 0.6835315 , 0.15768398, -0.5695771 , 0.45638716, -0.6475178 , -0.24609064, 0.36820015, -0.7532235 , -0.60841304, -0.13496049, -0.49064922, 0.31223625, 0.27192888, 0.27554023, 1.1431948 , 0.30906013, 1.071581 ], dtype=float32), kernel: Array([[[[ 0.1773554 , -0.0076261 , 0.39377195, ..., 0.09380915, -0.01814741, 0.2283536 ], [-0.2979896 , 0.02483123, 0.03855339, ..., 0.09292422, 0.0840548 , 0.02128509], [-0.11292487, -0.09941795, -0.21962844, ..., -0.00123785, 0.14886236, 0.08236729]], [[ 0.32237047, -0.10110885, 0.5543121 , ..., 0.20536377, -0.33312383, 0.189913 ], [-0.23798265, -0.02816244, 0.15939426, ..., 0.1943017 , -0.09701379, -0.03432931], [ 0.10851432, -0.19066858, 0.04841118, ..., -0.01213947, -0.04722768, 0.09524001]], [[ 0.2550473 , -0.04364953, -0.47635067, ..., 0.23234586, -0.365624 , -0.01943434], [-0.2977616 , 0.00419303, -0.8656642 , ..., 0.30416554, -0.04954646, -0.1717201 ], [ 0.11049867, -0.17219873, -0.6994334 , ..., 0.08241771, 0.03823497, 0.0737647 ]], [[ 0.32358578, 0.17112552, 0.14320694, ..., -0.16366176, -0.2593694 , 0.07770303], [-0.1454706 , 0.12558867, -0.0796358 , ..., -0.1381298 , -0.02053097, 0.09441353], [ 0.05893492, -0.00259268, 0.02598703, ..., -0.20935844, 0.1816749 , 0.30294153]]], [[[ 0.14045075, -0.10724663, 0.1179781 , ..., 0.03702212, 0.30327222, 0.0930538 ], [-0.30190122, 0.11180247, -0.158114 , ..., 0.0602173 , 0.28998795, -0.29604238], [ 0.15810204, -0.02881167, -0.5261955 , ..., 0.16892877, -0.02681437, -0.18120374]], [[ 0.1801053 , -0.17351657, 0.55432343, ..., -0.04310204, 0.20605011, 0.15464272], [-0.35629642, 0.1292508 , 0.4196015 , ..., -0.00466895, 0.40400398, -0.22713828], [ 0.3233468 , -0.09626149, 0.11579885, ..., 0.07394816, -0.05250774, 0.03030762]], [[ 0.00362034, -0.27520794, -0.2702903 , ..., -0.0462058 , 0.11103056, -0.17394325], [-0.5339498 , 0.0243352 , -0.31235537, ..., 0.11845686, 0.4861063 , -0.42657182], [ 0.18858455, -0.18266024, -0.27887428, ..., 0.160679 , 0.1009312 , -0.00895783]], [[ 0.23181222, -0.16900635, 0.17172067, ..., -0.2783022 , 0.0531309 , -0.14735658], [-0.17012398, 0.0160596 , 0.26340246, ..., -0.15997212, 0.34474295, -0.18495837], [ 0.25120145, -0.10030471, 0.4254995 , ..., -0.14183548, 0.27963412, 0.15028709]]], [[[ 0.04808923, -0.0793573 , -0.05729656, ..., -0.06481453, -0.08433866, 0.25621825], [-0.21680164, 0.12764114, -0.13568583, ..., -0.07495075, -0.22111697, -0.15526403], [ 0.3291076 , -0.05329537, -0.51923084, ..., 0.05456135, -0.5494265 , -0.08629624]], [[ 0.00642845, -0.02851976, 0.48139414, ..., -0.3939782 , -0.02460362, 0.4114907 ], [-0.33011124, 0.21700603, 0.6363917 , ..., -0.34058657, 0.07507597, -0.00235403], [ 0.4467702 , -0.08816491, 0.338514 , ..., -0.20894945, -0.44596782, 0.22598238]], [[-0.12848432, -0.17561927, -0.29103017, ..., -0.3294456 , 0.0409772 , 0.05005151], [-0.47656342, 0.09599458, 0.01909353, ..., -0.15514864, 0.3270731 , -0.2028388 ], [ 0.28001264, -0.17020059, 0.09817386, ..., -0.0626152 , -0.07981837, 0.1589364 ]], [[ 0.18718387, -0.22170076, -0.12480265, ..., -0.2592998 , -0.3301834 , 0.06356978], [-0.05257114, -0.01859191, 0.32042652, ..., -0.23058891, -0.06912984, 0.0211558 ], [ 0.38780808, -0.15261608, 0.6172939 , ..., -0.2062019 , -0.08611296, 0.31126183]]], [[[-0.17952503, -0.05265503, -0.25038588, ..., 0.2504428 , -0.02090847, 0.28380206], [-0.2667295 , 0.07449401, -0.06040205, ..., 0.07079565, -0.10550356, 0.09134477], [ 0.09433349, -0.00351469, -0.22650217, ..., 0.02396235, -0.11385609, -0.00377663]], [[-0.0575775 , 0.03152074, -0.16277562, ..., 0.3681371 , 0.01227211, 0.08380491], [-0.17844254, 0.10643482, 0.21974468, ..., 0.26805398, 0.11603563, -0.11773686], [ 0.38371345, -0.10176778, 0.20368902, ..., 0.1509468 , -0.02080609, -0.06920536]], [[-0.18148401, -0.06984355, -0.85413545, ..., 0.47789526, 0.04577804, -0.23575552], [-0.3161512 , 0.02919929, -0.4008322 , ..., 0.44068208, 0.26976907, -0.33418196], [ 0.20420933, -0.1428987 , -0.11891793, ..., 0.30973166, 0.22936389, -0.15018184]], [[-0.08356225, -0.13858244, -0.32757443, ..., 0.142595 , -0.351965 , -0.0224065 ], [-0.17430148, -0.03812229, 0.26883033, ..., -0.06516638, -0.17817669, 0.04958347], [ 0.09899019, -0.05975993, 0.6693031 , ..., -0.12235346, 0.04438766, 0.1941795 ]]]], dtype=float32), }, }, position_embeddings: Array([[[ 0.033918 , -0.0929377 , -0.10530281, ..., 0.28548247, 0.08699364, 0.3533416 ], [-1.1857578 , -0.05044684, 0.04907949, ..., 0.22999725, 0.11194737, -0.5982713 ], [-0.8083621 , -0.45285803, -0.35289305, ..., 0.1278877 , -0.05470289, -0.3990702 ], ..., [ 0.01943148, -0.21790016, 0.20034477, ..., 0.13447031, -0.12896743, 0.67419606], [-0.5797325 , 0.09393977, -0.10601121, ..., 0.5656874 , -0.4588539 , 0.3854702 ], [-0.5565649 , 0.41388488, 0.21124075, ..., 0.8872757 , -0.52099276, -0.30215293]]], dtype=float32), }, encoder: { layer: { 0: { attention: { attention: { key: { bias: Array([-0.03415372, 0.45620176, -0.6126655 , -0.51253253, 0.4352034 , 0.649004 , 0.68406653, -0.46161497, -0.42964068, 0.8848562 , 0.61135024, -0.67950135, -0.08493523, -0.29988992, 0.17360933, 0.6337579 , 0.08232927, -0.59048873, -0.31515497, 0.09602205, 0.14608672, -0.06844742, 0.5664715 , -0.6412251 , 0.44181392, -0.40813568, 0.6399837 , 0.41530073, -0.66080755, -0.01420364, -0.19360113, 0.14690916, -0.9573473 , -0.3561616 , -0.18794109, 0.5325726 , -0.5846602 , -0.73184437, -0.35172847, -0.27912554, 0.33543518, -0.21184462, 0.35979107, -0.12768705, 0.3412426 , 0.35638675, 0.02133832, -0.08022975, -0.15245748, -0.20536591, -0.11586879, -1.1425315 , -0.02779106, 0.32926312, -0.26967174, 0.1869237 , -0.04547051, -1.0505885 , 0.9415589 , -0.5067646 , -0.4434777 , 0.31517163, -0.48155847, 0.53517747, -0.40972438, 0.16880448, 0.17367116, 0.29588723, 0.05554008, -0.48776305, -0.31501368, -0.10502674, 0.13945673, 0.55460685, 0.51986885, -0.39894032, -0.2494641 , 0.05012343, -0.04834368, 0.29485628, 0.19806208, 0.3374114 , 0.11331975, 0.36502162, -0.01847783, -0.39011198, 0.3790581 , 0.02779981, 0.651827 , 0.46813282, 0.29877353, -0.43409863, 0.15295495, -0.35798907, -0.43806 , -0.26218414, -0.45440933, 0.11940431, -0.4274145 , 0.18455578, 0.00287167, 0.33210322, 0.32568368, -0.33072832, 0.08617178, -0.24881381, -0.07380535, 0.23978767, -0.12747152, -0.7031561 , 0.10753731, 0.27640837, 0.0658379 , -0.00729593, 0.15286936, 0.14529614, 0.20602514, -0.18859431, 0.08264172, 0.09762905, -0.1777643 , 0.21521233, 0.68952733, 0.35572144, -0.25402895, -0.12086129, 0.33329782, -0.27421075], dtype=float32), kernel: Array([[-0.13037314, -0.3519865 , -0.7311013 , ..., -0.14157563, -0.098704 , 0.08183759], [ 0.36583012, 0.46603498, 0.07223602, ..., 0.3872796 , -0.01522054, 0.1815025 ], [ 0.4379942 , -0.2856487 , -0.14937577, ..., 0.2555367 , 0.143062 , -0.10283173], ..., [-0.08320192, -0.2315665 , 0.183584 , ..., -0.08580804, -0.13162506, 0.07868645], [ 0.42390886, -0.22044583, 0.4916073 , ..., 0.57441944, -0.04296773, 0.02346938], [ 0.09392057, -0.64890575, -0.21604928, ..., 0.09232247, 0.13443615, 0.11167435]], dtype=float32), }, query: { bias: Array([-0.06905509, -0.44198722, 0.30581513, 0.14314957, -0.15857255, 0.15910107, -0.14257118, 0.11895534, -0.13231936, -0.28292307, -0.1729494 , 0.1297792 , 0.29567292, 0.1645278 , -0.07054784, -0.1081412 , 0.2331662 , -0.02096415, 0.00768806, -0.13316594, -0.25273684, -0.09793813, 0.04979343, 0.23559354, -0.19012289, 0.08803113, -0.03696692, -0.29770622, 0.39657098, 0.1723672 , 0.12946679, 0.12689376, 0.3535053 , -0.02396139, 0.01831555, -0.45482886, 0.0035749 , 0.0489781 , -0.19142908, 0.04616706, -0.10055473, 0.00193944, -0.0123636 , -0.1297169 , 0.06566181, -0.12390063, 0.01915232, -0.10427503, -0.26933163, 0.01553594, -0.02893843, 0.49531803, 0.30729276, 0.01837071, 0.172419 , -0.05943993, -0.29665995, 0.46334326, -0.13110274, 0.49487928, 0.21740474, -0.25742924, 0.03184005, -0.16928338, 0.04680282, -0.10523521, 0.07657482, 0.147745 , -0.09269993, 0.01065696, 0.05368982, -0.18941729, 0.10891443, 0.20821781, 0.15085766, 0.51667964, -0.15096618, 0.34021842, -0.03054576, 0.26622248, -0.36788234, 0.29284194, -0.18616207, 0.09047277, 0.3589835 , -0.09647398, -0.10208427, 0.09295312, 0.03126433, -0.27378327, 0.08303533, -0.17912042, -0.06608632, -0.05499709, -0.09980894, 0.15720649, 0.37134722, 0.39746282, 0.172588 , 0.04140972, 0.58430547, -0.250469 , -0.3090034 , 0.35163316, -0.09326122, 0.18299101, -0.02789885, -0.16457352, 0.21470276, 0.20549582, -0.27671963, -0.4622744 , 0.19656613, 0.31474093, -0.0242331 , -0.01637263, 0.08560522, 0.40495184, -0.14052592, -0.24214344, -0.27252468, 0.15204628, -0.19363014, -0.13963734, -0.02372985, -0.13916686, 0.21864863, 0.13198575], dtype=float32), kernel: Array([[-0.02407378, -0.08052919, -0.78489935, ..., 0.0214601 , -0.21603775, 0.07814599], [ 0.1441562 , 0.39637634, -0.01425371, ..., 0.44336745, 0.02438617, 0.04486706], [ 0.54582417, -0.38129556, -0.09409017, ..., 0.07300002, 0.08039478, -0.01730129], ..., [-0.40244198, 0.23577765, -0.09792052, ..., 0.06759664, 0.0144011 , -0.21112448], [ 0.52981406, 0.07420167, 0.320484 , ..., 0.3295426 , -0.08885895, -0.11461516], [ 0.2322218 , -0.26398233, -0.4191451 , ..., 0.24738975, -0.05001807, 0.01805569]], dtype=float32), }, value: { bias: Array([-2.94787940e-02, -2.72796545e-02, 6.14140369e-02, -2.86302455e-02, 7.17062429e-02, -3.11819911e-02, -4.36154753e-02, 3.26400585e-02, -2.82624215e-02, 1.26899080e-02, -7.74416178e-02, 4.88068024e-03, 2.13063625e-03, -7.00014010e-02, 6.05225153e-02, -4.30300795e-02, 1.79692768e-02, -2.71751042e-02, 3.53089422e-02, -1.95429940e-02, -1.21456282e-02, -2.05201493e-03, -3.62878200e-05, -1.08139282e-02, 7.40849450e-02, 1.50353080e-02, -7.06956685e-02, 8.49624351e-02, -7.26162344e-02, -5.55856004e-02, -7.41069913e-02, 7.11530168e-03, -2.33412739e-02, 6.24122694e-02, 1.00617625e-01, 4.78886962e-02, 6.93868250e-02, 8.44263211e-02, 6.89995065e-02, -1.49511993e-02, 6.76077977e-02, 2.99593173e-02, -6.50391206e-02, -3.10420450e-02, -4.87265997e-02, -7.07542598e-02, 3.83699648e-02, -1.07949436e-01, 3.88861708e-02, 1.07101694e-01, -7.05710873e-02, 6.53532194e-03, 5.76992594e-02, 8.30659941e-02, -1.04475126e-01, 8.40004310e-02, -9.89107862e-02, 8.94425064e-02, -3.12575921e-02, -5.92782833e-02, -8.53339434e-02, -5.87267503e-02, 3.41176279e-02, -6.32282570e-02, -4.08555977e-02, -7.24948272e-02, 4.16365713e-02, -2.20883206e-01, -4.90566827e-02, -1.89475138e-02, -1.50290072e-01, 2.09986553e-01, 3.20502929e-02, 8.28777160e-03, 6.13699071e-02, -2.50448077e-03, -2.15128988e-01, -3.07804886e-02, -1.82573318e-01, 9.74187329e-02, 8.86107534e-02, -6.78683072e-02, -6.89225420e-02, -3.00914403e-02, -1.05735838e-01, -2.17407808e-01, 1.63544074e-01, 1.00356489e-01, -6.03550719e-03, 3.53857167e-02, -7.28843063e-02, 1.15036063e-01, -1.29502550e-01, 5.06461645e-03, 1.20808132e-01, 1.27421441e-02, -3.18243355e-02, -5.57740144e-02, -6.90745190e-02, -1.28188869e-02, 1.20238643e-02, -4.42795120e-02, -2.79449914e-02, -3.33387293e-02, 7.23095536e-02, -2.58487016e-02, -1.62459761e-02, 1.07147522e-01, -6.11629374e-02, 7.07705468e-02, -5.56649491e-02, 3.85964476e-02, 9.34836396e-04, -1.59531720e-02, -7.20297545e-02, 8.05353299e-02, 1.81769580e-02, -3.95291820e-02, -7.38437474e-02, -9.54268277e-02, 1.34773731e-01, 9.29158255e-02, -5.70693836e-02, 2.48423647e-02, -8.44348520e-02, -7.13706389e-02, -7.08770528e-02, 7.93647394e-02], dtype=float32), kernel: Array([[-0.19559653, 0.10921917, -0.24855891, ..., -0.3095954 , -0.11501747, 0.3495971 ], [ 0.2759333 , -0.0968355 , 0.25864232, ..., -0.07446828, -0.01653024, 0.04277551], [-0.53325766, -0.17772451, -0.19101648, ..., -0.09837775, 0.07115 , 0.39424804], ..., [ 0.36891153, -0.11530672, 0.24386573, ..., 0.05493925, -0.2263004 , -0.10157143], [ 0.06181217, 0.21771231, -0.52629954, ..., -0.0283559 , -0.11670491, -0.19776788], [-0.20297265, -0.24884045, 0.21022551, ..., 0.13422379, 0.22147001, 0.18063086]], dtype=float32), }, }, output: { dense: { bias: Array([-0.08976492, 0.35623267, -0.28853896, 0.1584903 , -0.04931528, 0.41930285, -0.12198956, -0.1676379 , -0.07629202, 0.44505563, -0.13279814, -0.2606873 , -0.38957864, 0.2600788 , -0.15301487, -0.7227959 , 0.06558075, -0.02327237, -0.01965788, 0.3900318 , 0.14463983, 0.33909434, -0.02560525, 0.25102413, 0.26336563, 0.07151037, -0.01791747, -0.32692116, -0.29931375, 0.26871207, -0.45228213, -0.07384168, -0.2372598 , 0.16236512, 0.20462725, -0.04373658, 0.18107706, 0.5327047 , 0.11542407, -0.20473395, -0.29505363, -0.1296326 , -0.50376815, 0.23728877, 0.16542588, 0.20167176, 0.05926127, 0.1154462 , -0.14579555, -0.08487356, 0.07978057, -0.27115968, -0.5287936 , 0.07883674, -0.07367402, -0.04930678, -0.5660592 , 0.14252122, -0.05342535, -0.03159878, 0.18108739, -0.06234799, 0.13124788, 0.00128543, -0.24720064, 0.13804607, -0.5026876 , 0.53043497, 0.07013435, 0.22524473, 0.17431633, 0.22486666, 0.01429994, -0.02526889, 0.65786517, -0.12362218, -0.124548 , 0.02234903, -0.16374543, -0.03648356, -0.79906535, -0.21115243, -0.20314318, 0.24117418, -0.42399853, 0.37275848, 0.11345251, -0.07611627, 0.09416302, -0.16610427, 0.22861136, -0.22463125, 0.21705595, 0.09253941, -0.2043051 , -0.06828043, -0.27955458, -0.00722787, -0.13780643, -0.44847608, 0.18490234, 0.09589677, 0.2274974 , -0.04555204, -0.13882257, -0.17222178, -0.24243575, 0.09366386, 0.16359329, 0.15961671, 0.08576287, 0.09707799, 0.31846833, -0.27176753, 0.12482687, -0.35437414, -0.04145558, -0.4648349 , 0.1269856 , -0.09939937, 0.00209725, 0.00080232, 0.20968738, -0.18917999, -0.12978964, 0.13607804, 0.00752292, -0.00256601], dtype=float32), kernel: Array([[ 0.01726927, 0.12866902, -0.1439863 , ..., -0.26570368, -0.0510007 , 0.47683123], [ 0.15968257, -0.1014777 , -0.26105717, ..., 0.30918548, 0.48335168, 0.01941068], [-0.05697384, 0.05050297, -0.3797851 , ..., -0.70562077, 0.11496508, 0.02426186], ..., [-0.37025422, -0.41289684, -0.09372032, ..., 0.3905355 , -0.44608384, -0.10514601], [ 0.03965069, -0.21772908, -0.06686966, ..., 0.5678079 , -0.5982024 , 0.03439448], [ 0.31467485, 0.03231593, 0.6761289 , ..., 0.3369738 , -0.16577064, 0.09921929]], dtype=float32), }, }, }, layernorm_after: { bias: Array([ 7.65252188e-02, 6.29107952e-01, 8.02198052e-02, 8.70596059e-03, 4.38350849e-02, -1.07495952e-02, -5.31466417e-02, -1.02623366e-01, 2.12361347e-02, 6.80099875e-02, -2.28481810e-03, 8.53950828e-02, -5.73644787e-02, 2.06265952e-02, -7.53715783e-02, -1.43298656e-01, -2.74074376e-02, -1.09807447e-01, -6.17579147e-02, -7.90846869e-02, -9.07771438e-02, 8.38840082e-02, -1.48849720e-02, -2.24489588e-02, -9.19756591e-02, -1.33614585e-01, 8.40165392e-02, -2.04390988e-01, 7.47799221e-03, 1.38857782e-01, -5.69530539e-02, 9.17964336e-03, -1.55208111e-01, 2.73925383e-02, -1.95245608e-03, 5.41600175e-02, -2.93773953e-02, 3.53689879e-01, -1.23494409e-01, -1.56066343e-01, 8.71012285e-02, 2.55195685e-02, -3.10201854e-01, -8.39867163e-03, -3.08605190e-02, -5.28353490e-02, -1.40786827e-01, -5.47004975e-02, -1.13185709e-02, 1.86418276e-02, -7.84755684e-03, -4.15783050e-03, -7.23032653e-02, -1.61183357e-01, 1.90100819e-02, -4.55883006e-03, -2.70947274e-02, 1.69784650e-02, -2.28463719e-03, 4.21662033e-02, -9.98298172e-03, 7.62146562e-02, -1.09199071e-02, 7.20275193e-02, 1.14038222e-01, 2.85849470e-04, -2.99424119e-02, 1.37919337e-01, -1.45379841e-01, 6.62748562e-03, 2.23827194e-02, -5.66348387e-03, 5.01772054e-02, 1.41351158e-03, 5.96981421e-02, -2.08293200e-01, -5.95040731e-02, 5.53276669e-03, -8.04201216e-02, -1.67969614e-01, 3.61396512e-03, 5.54506434e-03, -2.92952433e-02, 4.65981402e-02, 1.59924058e-03, -2.43443865e-02, -1.17824182e-01, -3.83222140e-02, -3.87748890e-02, -6.59047961e-02, -8.08075536e-04, -3.35978210e-01, -4.24195863e-02, 3.55881304e-01, 3.02827614e-03, -5.55533350e-01, 1.54737878e-04, -1.47938222e-01, -2.13265121e-02, -1.14879206e-01, -3.36717785e-04, 5.57862930e-02, 1.95077602e-02, -8.35356787e-02, 2.89721284e-02, -1.48645073e-01, -2.94460828e-04, 8.76041055e-02, -1.90158889e-01, 1.32130921e-01, 1.11547068e-01, -8.59021768e-02, -4.04770300e-03, -2.04068184e-01, 2.25875482e-01, -7.98009858e-02, 1.03451028e-01, -6.95376145e-03, -8.43686834e-02, -1.78507604e-02, -9.83250737e-02, -7.10767433e-02, -6.19554799e-03, -5.51361814e-02, -8.06146394e-03, 4.24989313e-01, -2.02743262e-02, -7.10928217e-02], dtype=float32), scale: Array([ 4.92835701e-01, 7.25533962e-01, 1.97780564e-01, 7.19367981e-01, 3.85366410e-01, 1.70830980e-01, 4.03196029e-02, 3.59184593e-01, 1.64954796e-01, 5.99162340e-01, 1.94611892e-01, 4.47059900e-01, 1.33249581e-01, 1.17341705e-01, 2.57417858e-01, 2.13419840e-01, 1.17401130e-01, 1.44011736e-01, 4.55426246e-01, 3.37985694e-01, 3.68331194e-01, 1.58645034e-01, 4.11303312e-01, 2.80974478e-01, 2.08782077e-01, 3.16983044e-01, 2.54319280e-01, 5.07932603e-01, 6.26261353e-01, 3.31194192e-01, 5.77102304e-01, 5.91307692e-03, 2.17876673e-01, 1.83340341e-01, 7.28062959e-03, 1.43880785e-01, 3.79934996e-01, 8.64668906e-01, 5.03845096e-01, 3.30553919e-01, 3.66227090e-01, 1.23913087e-01, 4.04930204e-01, -5.25120134e-03, 2.16952696e-01, 2.82419056e-01, 2.41874039e-01, 2.18594924e-01, 2.72880584e-01, 1.63890943e-01, 5.69228344e-02, 1.57258492e-02, 3.11221033e-01, 3.75806093e-01, 3.45861703e-01, 3.33030730e-01, 1.86815023e-01, 2.87897736e-01, 1.79882497e-02, 3.26225400e-01, 3.56188752e-02, 3.17295760e-01, 5.93220115e-01, 3.10718954e-01, 1.92407712e-01, 9.49200336e-03, 2.15880141e-01, 4.15735185e-01, 3.10675055e-01, 5.28817117e-01, 1.04173638e-01, 1.57576382e-01, 3.02691996e-01, 3.79474498e-02, 7.07599940e-03, 5.08240998e-01, 3.50080490e-01, 3.61189306e-01, 1.87464565e-01, 4.86818105e-01, 3.34662758e-03, -1.36662647e-03, 1.20725386e-01, 5.04414022e-01, 3.98086966e-04, 9.34311152e-02, 1.42500699e-01, 3.84391755e-01, 1.07679166e-01, 3.01436037e-01, -1.05553423e-03, 4.41559106e-01, 1.26928464e-01, 4.32251424e-01, -6.94631459e-03, 5.02396822e-01, -3.20078339e-04, 1.94142312e-01, 8.77228081e-02, 1.18769050e-01, 4.47774172e-01, 3.87618184e-01, 4.12855119e-01, 3.15822661e-01, 4.97932345e-01, 6.33932471e-01, 6.15001982e-03, 2.09667221e-01, 2.26229250e-01, 1.05634868e+00, 5.23372293e-01, 4.29101616e-01, 2.27578685e-01, 6.58651769e-01, 2.97994912e-01, 1.07822992e-01, 4.83628422e-01, 4.72095191e-01, 3.23185503e-01, 1.00680351e-01, 4.04555202e-01, 1.64712802e-01, 6.41667992e-02, 1.81472510e-01, 5.06515145e-01, 4.70199496e-01, 2.70896405e-01, 4.36434001e-01], dtype=float32), }, layernorm_before: { bias: Array([-0.05890432, 0.07051883, 0.09445145, -0.07912993, -0.14302658, -0.01429262, 0.03426607, 0.00171352, 0.11276561, -0.05662137, 0.13313785, -0.04794319, 0.00357568, 0.01197018, -0.07405796, -0.06490814, -0.0245945 , 0.14022568, -0.05081064, -0.12518148, 0.00789185, -0.24005446, 0.02322139, -0.17626956, -0.03338344, -0.33740795, -0.13774505, -0.07997898, 0.04349966, -0.01520896, -0.39623648, -0.150468 , 0.11885046, -0.01181321, 0.00808169, -0.20219389, 0.0043777 , -0.08942726, -0.16421181, -0.18974544, -0.07742172, -0.01755824, -0.02627599, -0.4378331 , -0.19054742, 0.2720697 , 0.25448796, 0.01773962, -0.01873611, 0.0169231 , 0.03406947, 0.0349858 , 0.09072868, -0.05627641, 0.14739008, 0.01965971, 0.09791738, 0.03601414, -0.28183195, 0.15014632, 0.22542529, 0.3072184 , -0.09403451, -0.08351854, 0.25167593, 0.04495538, -0.11564947, -0.00397131, 0.10662062, 0.05204451, 0.01583207, 0.01414629, 0.23943776, 0.06893958, -0.08743864, -0.06693614, -0.07541402, -0.19011301, -0.17251883, 0.01803088, -0.09193094, 0.04894969, -0.00502513, -0.09108073, -0.1768744 , -0.01100582, 0.10178719, 0.22854435, 0.04437195, 0.00859537, -0.03013818, 0.26522186, -0.0835686 , 0.0909194 , 0.08486014, -0.39586094, 0.03491055, 0.02060359, -0.14571564, -0.18893251, 0.04928007, 0.05034845, -0.15555385, -0.15786135, -0.56710386, 0.01834173, -0.04088113, -0.0075858 , 0.04828192, -0.1228609 , -0.00563954, -0.3955512 , -0.06544469, 0.09136705, -0.10234332, 0.09786685, 0.00981475, -0.05797027, 0.08497368, 0.18863663, -0.00070434, 0.03551389, -0.04786862, -0.03968854, -0.02606042, -0.2953443 , -0.03344315, -0.2232286 ], dtype=float32), scale: Array([0.45335642, 0.28725365, 0.25888288, 0.51014036, 0.25393826, 0.14649442, 0.45231742, 0.07753127, 0.3569898 , 0.67538893, 0.2085993 , 0.17448373, 0.01005436, 0.07782084, 0.26946452, 0.14676209, 0.22900493, 0.29659525, 0.13212137, 0.1925922 , 0.04533805, 0.26069018, 0.15647988, 0.20724292, 0.08556735, 0.296176 , 0.20237958, 0.6578139 , 0.11472861, 0.3509395 , 0.38030067, 0.19699657, 0.30957213, 0.1079342 , 0.12253352, 0.54774934, 0.06868377, 0.49437425, 0.34611228, 0.33752495, 0.24485146, 0.11372902, 0.13047746, 0.40855378, 0.49101192, 0.5738504 , 0.32902393, 0.2079778 , 0.09577054, 0.11233962, 0.17488489, 0.10529607, 0.13811632, 0.19155163, 0.16904642, 0.10060844, 0.2304541 , 0.20629783, 0.21513213, 0.1885996 , 0.2502167 , 0.3126915 , 0.3413607 , 0.1449984 , 0.41228867, 0.09932974, 0.14989796, 0.13950281, 0.24986736, 0.15920106, 0.06112578, 0.10076807, 0.25724828, 0.11337321, 0.1927693 , 0.24177718, 0.14134662, 0.4839469 , 0.20330684, 0.1124863 , 0.17910473, 0.1306797 , 0.09654108, 0.13909315, 0.19018278, 0.12305983, 0.3541507 , 0.31824607, 0.13942654, 0.08698463, 0.08841392, 0.17479059, 0.15011418, 0.45322958, 0.17411204, 0.29569057, 0.10512718, 0.3171436 , 0.21283199, 0.51158214, 0.42643854, 0.18322334, 0.3302157 , 0.37901062, 0.44037428, 0.50349325, 0.1318383 , 0.13000993, 0.14093749, 0.5986485 , 0.1306848 , 0.36673996, 0.2426187 , 0.19750752, 0.14503735, 0.22176953, 0.1925638 , 0.15475766, 0.17507745, 0.22627683, 0.10930963, 0.37122646, 0.12475897, 0.12880948, 0.11971601, 0.2903781 , 0.1446715 , 0.20750394], dtype=float32), }, moe: { shared_experts: { intermediate: { bias: Array([-0.4722571 , -0.43928552, -0.5569621 , -0.3837599 , -0.47573015, -0.4316669 , -0.762032 , -0.5649565 , -0.31526604, -0.42409667, -0.35039356, -0.5466656 , -0.08653939, -0.50116634, -0.65188706, -0.47724712, -0.29790252, 0.32602024, -0.04504937, -0.40849677, -0.11292349, -0.25168103, -0.45442316, -0.34808317, -0.38236308, -0.3788954 , -0.19183853, -0.30315536, -0.16824003, -0.4485127 , -0.30100617, -0.39639288, -0.16540907, -0.5450078 , -0.4645357 , -0.36592278, -0.5585429 , -0.34257036, -0.19063933, -0.44469953, -0.42737913, -0.36188826, -0.2907807 , -0.02146477, -0.23154494, -0.20253362, -0.69757545, -0.39887667, -0.44025323, -0.39642838, -0.387052 , -0.24472845, -0.09177764, 0.05788963, -0.04718702, -0.21287683, -0.29563203, -0.3253591 , -0.43122077, -0.1849469 , -0.5141688 , -0.29721844, -0.47572163, -0.3446094 , -0.26447332, -0.47591433, -0.19857107, -0.6629566 , -0.36350504, -0.60418856, -0.4550368 , -0.40432703, -0.35896167, -0.33915696, -0.41339654, -0.23302272, -0.05349919, -0.47071078, -0.32239798, -0.3542714 , -0.25527537, -0.3346329 , -0.49991933, -0.22825265, -0.45629248, 0.01691399, -0.34854415, -0.44188562, -0.03925477, -0.3642511 , -0.40468022, -0.33773935, -0.49415597, -0.5592706 , -0.2977098 , -0.32126102, -0.16516496, -0.40793797, -0.8022191 , -0.44280693, -0.1008477 , -0.22637095, -0.16032524, -0.45757398, -0.4884214 , -0.48628846, -0.4652781 , -0.51417524, -0.25536695, -0.33155087, -0.5587205 , -0.37767413, -0.30986476, -0.47982594, -0.51307243, -0.38967907, -0.24242653, -0.25831246, -0.1458375 , -0.58483136, -0.3381596 , 0.27927315, -0.7453308 , -0.4533877 , -0.15844217, -0.45824784, -0.468726 , -0.4619875 , -0.4371935 , 0.00825728, -0.31616744, -0.4558102 , 0.0993996 , -0.08013049, -0.17536967, -0.52361816, -0.44187388, -0.25537458, -0.19660369, -0.30101493, -0.09137805, -0.41055298, -0.46738333, -0.57409215, -0.34877914, -0.52286893, -0.55365384, -0.33294562, -0.47346646, -0.3105509 , -0.44117743, -0.3502154 , -0.39879954, -0.3967916 , -0.47747064, -0.24431925, -0.39359137, -0.22136432, -0.35870054, -0.29925522, -0.34952432, -0.49339256, -0.42442355, -0.383239 , -0.3228079 , -0.59250957, -0.3766234 , -0.48536432, -0.49891275, -0.10365842, -0.21082799, -0.5129332 , -0.39634183, -0.4714169 , -0.3876767 , -0.2642752 , -0.1780093 , -0.08995035, -0.5045371 , -0.31991875, -0.3657458 , -0.51370513, -0.5282006 , -0.43932983, -0.44844607, -0.4636747 , -0.541842 , -0.28016394, -0.5640383 , -0.28513128, -0.44549647, -0.43519762, -0.42721725, -0.57123166, -0.40650263, -0.12434424, -0.2684522 , -0.13034356, -0.41602165, -0.2656361 , -0.63166195, -0.24252081, -0.49084774, -0.66598415, -0.21980134, -0.20069678, -0.3857888 , -0.5750186 , -0.28702593, -0.42044863, -0.43332833, -0.05599573, -0.17978549, -0.33969852, -0.42026928, -0.19741155, -0.2731103 , -0.5647213 , -0.61824954, -0.44372314, -0.16089505, -0.6141717 , -0.27717924, -0.39058053, -0.30320698, -0.3017265 , -0.36256546, -0.29453766, -0.7215134 , -0.36851448, -0.45460737, -0.25023326, -0.24756469, 0.5198105 , -0.49648827, -0.7883031 , -0.40549907, -0.44536465, -0.34158024, -0.4282217 , -0.45166638, -0.285461 , -0.5351079 , -0.5246095 , -0.38692436, -0.4037612 , -0.68792176, 0.09384751, -0.3728323 , -0.54188645, -0.14435874, -0.13926642, -0.48824954, 0.103251 , -0.5347266 , -0.12768686, -0.2596208 , -0.33724293, 0.41711676, -0.46974605, -0.31142974, -0.46029773, -0.38704032, -0.4837437 , -0.4762127 , -0.48931426, -0.27646464, -0.7709319 , -0.11419559, -0.3580196 , -0.41691482, -0.523539 , -0.7911689 , -0.14548191, -0.32703182, -0.22812605, -0.5490665 , -0.27162734, -0.47042787, -0.27687344, -0.29244688, -0.14437722, -0.42082107, -0.46603575, -0.3757466 , -0.43942484, -0.35704526, -0.50712955, -0.06310245, -0.51886773, -0.36505163, -0.29938784, -0.47636163, -0.798845 , -0.39453545, -0.42888972, -0.34777957, -0.42616373, -0.37013018, -0.2707561 , -0.22923009, -0.4484783 , 0.0781149 , -0.54366326, -0.31296468, -0.46913856, -0.26365936, -0.05434812, -0.43664017, -0.262903 , -0.21987678, -0.29727325, -0.53905886, -0.2881995 , -0.2697021 , -0.56999284, -0.23401402, -0.25733256, -0.48550016, -0.53915155, -0.29186913, -0.41333848, -0.11731004, -0.1876421 , -0.38366327, -0.69185966, -0.54369533, -0.13784738, -0.45933396, -0.47800148, -0.42821413, -0.37634867, -0.42712605, -0.06954894, -0.445592 , -0.40655133, -0.3772172 , -0.7594351 , -0.5565112 , -0.25855637, -0.13048875, -0.3732955 , -0.30561906, -0.5940574 , -0.41309616, -0.314704 , -0.3010439 , -0.5837942 , 0.40323326, -0.24861278, -0.4630232 , -0.538137 , -0.37435848, -0.46322653, -0.48135036, -0.26899594, -0.17752594, -0.46429896, -0.62554 , -0.4089612 , -0.34856302, -0.07396294, -0.4918398 , -0.44181666, -0.21933144, -0.41612074, -0.46780694, -0.25013933, -0.3507257 , -0.31338322, -0.49207342, -0.573551 , -0.25965518, -0.27235296, -0.46362886, -0.572643 , -0.36132613, -0.5891415 , -0.43028024, -0.58089757, -0.5628406 , -0.23266539, -0.48305756, -0.4171379 , -0.30668163, -0.800837 , 0.16001745, -0.32928717, -0.7041144 , -0.28114614, -0.1339794 , -0.1539137 , -0.48452973, -0.2916893 , -0.31748015, -0.41399214, -0.5892567 , -0.6904999 , -0.39504105, -0.05394559, -0.4220559 , -0.08093858, -0.55065817, -0.42636526, -0.36694276, -0.6491974 , -0.16998939, -0.6064266 , -0.49446252, -0.25082645, -0.3862861 , -0.5665641 , -0.12244188, -0.59443516, -0.43923312, -0.03984782, -0.4539737 , -0.2804616 , -0.28886226, -0.41679534, -0.7231176 , -0.44628784, -0.17072082, -0.55711204, -0.33631635, -0.3513167 , -0.44696072, -0.09838888, -0.5331628 , -0.3244455 , -0.22034623, -0.39987078, -0.52567166, -0.18180668, -0.1563768 , -0.15176131, -0.34147656, -0.39736265, -0.22072102, -0.16304936, -0.4934699 , -0.35189024, -0.38949656, 0.13233016, -0.2961783 , -0.03038953, -0.19008778, -0.43870625, -0.14041252, -0.46124077, -0.6101991 , -0.78746367, -0.41358054, -0.0965796 , -0.17454897, -0.46205354, -0.5193903 , -0.43612164, -0.338933 , -0.2509465 , -0.25011972, -0.32999432, -0.5068436 , -0.40737084, -0.59392023, -0.08752135, -0.6887468 , -0.02706476, -0.34656492, -0.45208535, 0.12221402, -0.07045163, -0.6397726 , -0.32004473, -0.2506177 , -0.41133684, -0.20769219, -0.3077098 , -0.48824766, -0.59370387, -0.3106976 , -0.09882683, -0.23733851, -0.5397758 , -0.384874 , -0.48532894, -0.35305282, -0.44897798, -0.57191414, -0.23201816, -0.40165707, -0.45165244, -0.30947968, -0.57236695, -0.40093747, -0.610941 , -0.4310733 , 0.00399699, -0.17124976, -0.33956885, -0.59238917, -0.4834356 , -0.23509572, -0.3706481 , -0.46995562, -0.30272713, -0.42109653, -0.16173697, -0.3515516 , -0.46951428, -0.24467751, -0.37127313], dtype=float32), kernel: Array([[-0.16517928, -0.06662413, -0.32296306, ..., 0.17395326, -0.00140762, -0.21535204], [-0.2497043 , -0.32800463, -0.54542446, ..., -0.37079784, -0.4540639 , -0.2762963 ], [ 0.20276228, -0.2351077 , -0.4339836 , ..., -0.07039763, 0.09680251, -0.00943627], ..., [-0.0865024 , -0.07723805, -1.0022317 , ..., -0.22161727, 0.2631157 , -0.2520337 ], [-0.06232752, 0.41687462, 0.43454677, ..., 0.17987384, -0.22243185, 0.2732002 ], [-0.1515276 , 0.08726911, -0.16446725, ..., -0.1428107 , -0.12378144, -0.24571244]], dtype=float32), }, output: { bias: Array([-2.27567896e-01, 3.73193085e-01, 1.41862273e-01, 8.60335305e-02, 6.99756369e-02, -1.08260542e-01, -7.94294663e-03, 1.12044938e-01, -1.97965074e-02, 1.03512242e-01, 1.25780692e-02, 2.14173347e-02, -1.95132956e-01, -1.66688226e-02, -5.57698980e-02, 2.21790038e-02, 8.23988672e-03, -1.56303987e-01, 2.77717095e-02, -6.28630817e-02, -1.10570341e-01, 1.39788479e-01, -9.08712447e-02, -7.11519644e-03, -1.21463031e-01, 1.24595746e-01, 6.99105812e-03, 6.04419075e-02, 1.16523303e-01, -3.08515457e-03, 1.71046168e-01, -4.62258421e-02, 1.37897981e-02, -9.23820660e-02, 1.22136414e-01, -5.12558557e-02, -4.16860264e-03, 5.67508861e-02, -8.22613947e-04, -1.72597811e-01, -1.73500534e-02, -1.02644004e-01, -7.16797352e-01, 6.50356859e-02, -9.02397409e-02, -7.97297060e-02, -3.20093259e-02, 7.92354718e-02, -5.91707882e-03, -5.96906338e-03, 2.83249887e-03, 4.21253145e-02, -1.67518064e-01, -2.99427286e-02, -5.09011634e-02, -3.20116989e-02, -1.33028731e-01, 2.27317363e-02, 1.29082099e-01, 1.18409574e-01, 1.57172829e-02, -2.19231278e-01, -2.51264381e-03, -4.79261391e-02, 4.22019213e-02, -7.78303593e-02, -7.12049752e-02, -8.38479102e-02, -3.88463922e-02, -3.15606929e-02, -6.03214689e-02, 4.97607961e-02, -3.79606709e-02, 7.08331587e-04, 9.65211987e-01, 1.56220675e-01, -2.14344375e-02, -2.15863492e-02, -1.17833465e-01, 1.19182810e-01, -2.71626860e-01, 1.97731983e-03, 3.98320071e-02, 2.02119559e-01, 6.09681681e-02, -3.32502718e-03, 1.39364108e-01, 5.88153526e-02, -4.45023738e-02, -6.55459762e-02, 4.42181528e-02, -8.78035650e-02, -4.80993986e-02, -1.09087467e-01, -9.66937765e-02, -1.30174085e-01, 2.12008785e-03, 2.63354164e-02, -9.37493294e-02, 7.85378218e-02, 1.73281968e-01, -7.77669847e-02, 1.33560196e-01, -1.09741293e-01, 1.85674503e-01, 1.03994340e-01, -1.89410765e-02, -1.18796170e-01, -1.62400916e-01, 2.00676285e-02, -1.06416032e-01, 1.79987997e-02, 1.19247243e-01, -2.42678579e-02, 1.64977908e-01, -7.41590792e-03, -1.76154330e-01, 2.93482076e-02, -6.35338202e-02, 2.07892321e-02, 8.92439187e-02, 2.59270482e-02, 2.71555986e-02, -7.91539550e-02, 9.35807526e-02, 5.84603362e-02, 1.93760265e-02, 1.72683045e-01], dtype=float32), kernel: Array([[ 6.6307461e-01, -2.6014799e-01, -8.3183780e-02, ..., -2.6699215e-01, -1.0033758e-01, -2.1524315e-01], [-1.4059058e-01, -4.3626988e-01, -8.1478050e-03, ..., -3.1092042e-01, 3.1286913e-01, -3.7179798e-02], [ 7.0444113e-01, -1.0556934e-05, -2.0829076e-02, ..., -9.0309918e-01, 3.2471472e-01, 5.0782524e-02], ..., [-7.5270250e-02, 3.7796192e-02, -5.1213449e-01, ..., 3.4989411e-01, -1.0396782e-01, -5.7098109e-01], [ 9.8383605e-02, -1.5289782e-01, 3.3700535e-01, ..., 5.9063655e-01, -9.2052268e-03, -1.1524218e-01], [ 1.1272128e-01, -6.6285774e-02, -2.5734550e-01, ..., -1.2502365e-01, 9.1840588e-02, 2.3722002e-01]], dtype=float32), }, }, }, }, 1: { attention: { attention: { key: { bias: Array([ 0.08042084, 0.17798816, 0.05449522, 0.09923156, 0.02864883, -0.06706466, 0.12987332, 0.05871709, 0.05153077, 0.05436414, 0.05116336, -0.14068684, -0.11056656, -0.00991776, 0.00717054, 0.0770971 , 0.11563851, 0.02029003, 0.02723394, -0.12056217, -0.06265891, -0.05970899, 0.14154014, -0.03163632, 0.0481686 , 0.02711394, 0.04412205, 0.02307438, -0.05139432, 0.03842278, -0.07357287, -0.08965195, -0.08623472, 0.03718801, 0.84410733, 0.4109549 , 0.3907228 , -0.44974568, -0.3517945 , 0.03430397, -0.07853877, -0.30442524, 0.5081588 , 0.15563154, 0.17845473, 0.14964798, 0.01935043, 0.78861445, 0.13657925, 0.9571119 , -0.15077294, 0.04331086, -0.23139133, -0.09555604, 0.06358018, -0.59850585, -0.37260893, -0.9738547 , -0.01462804, 0.12405673, 0.07412421, -0.01379338, 0.05929387, 0.7805736 , -0.36394337, 0.47818112, -0.43866792, 0.63309664, 0.1147754 , 0.45001298, 0.3213029 , 0.44093522, 0.51016414, -0.18746474, -0.13791506, 0.04778931, -0.49275073, 0.04839996, -0.38444966, -0.4882343 , -0.6615728 , 0.09630081, -0.29930323, -0.34863642, 0.77798736, -0.55292934, -0.56388927, 0.09340773, -0.4621951 , 0.26494777, 0.31161246, 0.44366807, 0.6213311 , 0.01337955, 0.14917652, -0.4876238 , 0.06336308, -0.12776731, -0.07288839, -0.18272795, 0.23383974, -0.059283 , -0.08504926, -0.12375498, 0.3620917 , 0.04436552, 0.15867496, 0.12362888, -0.09505215, 0.18897517, 0.08316641, -0.18332016, -0.26461446, 0.17972536, 0.23397446, -0.04752706, -0.04319599, 0.12071403, 0.25849587, 0.06566656, 0.19910227, 0.22742352, -0.22645345, -0.14635295, -0.2507506 , -0.18631294, -0.05247986, 0.12347726], dtype=float32), kernel: Array([[-0.06023952, 0.297179 , -0.02699618, ..., -0.1145321 , -0.08048212, -0.10587901], [ 0.6934901 , 0.47458088, 0.61580837, ..., -0.13140583, -0.14733373, 0.26471645], [ 0.09990517, -0.17357504, 0.05461919, ..., -0.11821933, -0.24399522, -0.15253429], ..., [-0.2205276 , -0.23215702, -0.24366312, ..., -0.03175567, 0.22550093, 0.27891138], [-0.24111255, -0.27693352, -0.21988523, ..., 0.30499363, 0.37258562, -0.13957877], [-0.103215 , -0.14666593, -0.04492585, ..., -0.6473914 , -0.23865867, 0.19981082]], dtype=float32), }, query: { bias: Array([-0.11081476, -0.09144565, -0.06050301, -0.03483055, 0.03535065, -0.1849559 , -0.04721399, -0.07624152, -0.10357637, 0.22812082, -0.3447222 , 0.0331145 , 0.05764522, -0.13265188, 0.1981749 , -0.03139308, -0.07606025, 0.13599226, 0.02934942, 0.12725802, 0.07556148, 0.07594807, -0.04966472, -0.15265177, -0.35979733, 0.01576409, -0.07025045, -0.19974303, -0.22990641, 0.0952646 , -0.33106107, 0.04587565, -0.41658264, 0.33157024, -0.51854265, 0.34642565, -0.38225 , 0.21841964, 0.4542027 , -0.3708585 , -0.17867757, 0.38574484, -0.40515402, -0.18270488, 0.15764216, -0.09051255, -0.47678202, -0.5485276 , 0.17880292, -0.42515096, 0.11444188, 0.27703342, -0.23375264, -0.17498691, 0.13194664, 0.06895838, 0.32982063, 0.17495483, -0.16514482, 0.16266893, 0.24224894, 0.49473688, -0.7105683 , -0.22717172, -0.4628581 , -0.02517562, 0.13951837, 0.04581514, -0.02814059, 0.06052845, 0.89534795, -0.01086807, -0.2937633 , 0.4263962 , 0.01343843, 0.9783011 , 0.23017581, 0.7163039 , 0.41632223, 0.17262603, -0.15068051, 0.850449 , -0.16111681, -0.02732765, -0.18101065, 0.47148392, 0.18522678, -0.06222672, 0.15925601, -0.07961265, 0.71117395, 0.02433823, -0.35966122, -0.09352748, -0.06117439, -0.13183966, -0.08041007, -0.27098748, -0.71445733, -0.5658294 , 0.51616377, -0.56782836, -0.68260497, -0.5732899 , 0.5585706 , -0.14336808, 0.59159863, -0.21151459, 0.42952493, 0.46055183, 0.46279824, -0.4195606 , -0.5513547 , 0.34171253, 0.69069153, -0.25621027, -0.46899697, 0.588077 , 0.6255269 , 0.5819053 , 0.54691887, 0.59542644, -0.5749214 , -0.57993734, 0.42467687, -0.70540833, -0.34917867, 0.3106283 ], dtype=float32), kernel: Array([[-0.11731137, -0.1564569 , -0.11458346, ..., 0.02826559, -0.03335454, 0.14976554], [-0.04609706, -0.04719561, -0.02786452, ..., -0.30554634, -0.20783764, -0.11992735], [ 0.0489803 , 0.12078558, 0.0699265 , ..., 0.26866522, -0.40512353, -0.5030211 ], ..., [-0.00423968, -0.00703923, -0.03773196, ..., -0.10513086, -0.145968 , 0.07945077], [ 0.07510082, -0.13710845, 0.03422854, ..., 0.15533802, 0.8349194 , -0.10965192], [-0.07718561, 0.09582806, -0.06439573, ..., -0.12381844, -0.9082412 , 0.04665521]], dtype=float32), }, value: { bias: Array([-0.11375186, 0.01737056, -0.11132707, -0.01888963, 0.02005741, -0.00092973, -0.11860004, 0.14434907, -0.06270761, 0.14292052, 0.14930397, 0.06040736, 0.1552559 , -0.06054825, 0.13105005, 0.05230521, 0.03080181, -0.14906318, 0.04209505, -0.11033 , -0.02365249, -0.14162254, -0.00175851, 0.03008844, 0.04629046, 0.10597341, 0.02212737, 0.13431458, -0.03534039, 0.00220095, -0.05828169, 0.01887055, 0.07417873, -0.18334955, 0.05120145, 0.03194632, 0.09620124, 0.07511445, 0.14970838, -0.3099061 , -0.00785524, 0.21368372, -0.24593364, -0.20003676, 0.29439747, 0.02457003, -0.0362221 , -0.04669455, -0.11111037, -0.09324432, -0.007548 , -0.05471316, -0.1160496 , -0.08286496, 0.10508952, -0.29490575, 0.00155351, 0.13688038, 0.055782 , -0.241635 , 0.19978897, 0.12800941, -0.08591252, -0.17519936, 0.0762416 , 0.03798862, -0.04639675, -0.1959964 , -0.25277418, 0.169555 , -0.06163629, -0.11278319, -0.01159456, -0.06765992, 0.0005254 , -0.06396206, -0.00113279, 0.1249745 , -0.02429713, -0.01882967, -0.04579956, -0.02704 , 0.00707008, 0.02282375, -0.10874218, -0.01254413, 0.05641572, -0.06401654, 0.17181516, 0.0763936 , -0.07932342, -0.12056571, 0.0820564 , -0.01728764, 0.04256084, -0.05678854, 0.11785327, 0.2377933 , -0.19259618, -0.08974701, -0.19949402, -0.09832341, 0.12747605, 0.01983653, 0.00941677, 0.11338363, -0.20597233, -0.04107258, 0.02300076, -0.04743035, 0.11633945, 0.12718917, 0.23441505, 0.01317911, -0.19377421, -0.06671913, 0.04473834, 0.00467375, -0.15131958, -0.07110944, -0.16257668, -0.08920506, 0.03026637, -0.0935519 , 0.06253995, -0.19525133, -0.21808124, -0.19407861], dtype=float32), kernel: Array([[-0.01158661, 0.04303771, -0.05590956, ..., -0.05333043, -0.00572822, -0.01724763], [ 0.2719801 , 0.14965016, -0.02898863, ..., -0.06263707, -0.35550225, -0.28345698], [-0.05632893, -0.26717782, -0.07165924, ..., -0.43430653, 0.0922191 , 0.10500429], ..., [-0.04451123, -0.00324844, 0.15866533, ..., -0.15973136, -0.06277192, -0.10269681], [-0.28674033, 0.40732047, -0.23387024, ..., 0.5245572 , -0.20356956, -0.19302332], [-0.02055671, -0.35077375, 0.13905095, ..., 0.08351879, 0.4497927 , 0.44072792]], dtype=float32), }, }, output: { dense: { bias: Array([-2.29397908e-01, 3.42801064e-01, 1.12294301e-01, -1.40454136e-02, 3.03802863e-02, -1.64602742e-01, 6.72679991e-02, 9.77464989e-02, -2.96701267e-02, 7.89122283e-02, 5.59587255e-02, 1.48079200e-02, -1.85802028e-01, -3.27114910e-02, -2.40307394e-02, 6.82050884e-02, 5.04146097e-04, -1.56787038e-01, 6.75788894e-02, -1.31789781e-02, -4.55502272e-02, 1.27612054e-01, -7.01643378e-02, -2.48179883e-02, -1.31983072e-01, 1.38503700e-01, 1.17344763e-02, 7.56531805e-02, 6.57847598e-02, 2.69352719e-02, 2.21325293e-01, -5.05222380e-02, 3.61935459e-02, -1.04897447e-01, 5.24883904e-02, -8.26861486e-02, 1.44010643e-02, 8.25399533e-02, -6.71743648e-03, -1.94458306e-01, -2.49852352e-02, -8.23389366e-02, -6.81263685e-01, 8.11951086e-02, -8.49441960e-02, -1.40838427e-02, -1.44380983e-02, 6.43714517e-02, -1.57576092e-02, -3.63134891e-02, -1.25548737e-02, 1.41190467e-02, -1.39931515e-01, 1.46125898e-01, -4.10604011e-03, -3.28917056e-02, -1.47544041e-01, -7.30510205e-02, 1.31265923e-01, 3.08194682e-02, 5.00784516e-02, -2.18488395e-01, 2.87581724e-03, -3.31245400e-02, 2.70526409e-02, -1.38555646e-01, -1.16920797e-02, -7.85453171e-02, 1.44379027e-02, -8.10271129e-02, -7.89678767e-02, 6.87262276e-03, 2.95094848e-02, -2.96989549e-02, 8.52790475e-01, 1.68656245e-01, -4.95392680e-02, 1.01274252e-03, -1.17389508e-01, 1.42783284e-01, -2.95234203e-01, -3.66869681e-02, 2.10161004e-02, 1.61923394e-01, 1.00733116e-01, 9.56858695e-02, 1.62279665e-01, 6.97396919e-02, -6.76168874e-02, -1.21831730e-01, 2.96244565e-02, -8.43596309e-02, -4.85159941e-02, -1.54231772e-01, -8.46877992e-02, -9.01621133e-02, -7.94285350e-03, 9.12197307e-03, -5.20062111e-02, 2.70500332e-02, 1.75196305e-01, -7.51989707e-02, 1.59212708e-01, -8.15638751e-02, 1.15850195e-01, -1.62196823e-03, 6.47357618e-03, -1.16396412e-01, -1.10036433e-01, -1.06435921e-02, -1.69102713e-01, 2.04750933e-02, 1.00554541e-01, -3.42080109e-02, 1.30484760e-01, 1.22546069e-02, -1.68031573e-01, 3.20073776e-02, -6.57493770e-02, -7.96901435e-03, 1.29959419e-01, 8.22199881e-02, 4.45892625e-02, -4.60687764e-02, 6.53466284e-02, 2.84257364e-02, -7.09084561e-03, 8.53643045e-02], dtype=float32), kernel: Array([[-4.33085859e-01, -5.61205745e-01, 2.93745458e-01, ..., -1.06319696e-01, -3.59772116e-01, -5.50748073e-02], [ 1.29990995e-01, -1.94321834e-02, 3.05669427e-01, ..., 1.81368724e-01, 9.14699510e-02, -6.21524341e-02], [-2.22445935e-01, 1.21082552e-01, -5.40154229e-04, ..., -1.46840677e-01, -1.43376559e-01, 9.97718945e-02], ..., [ 6.94694817e-01, 8.00504535e-03, 7.26794839e-01, ..., -2.44273514e-01, -3.41350943e-01, -1.44506514e-01], [ 6.64600372e-01, -1.74710259e-01, 2.86397934e-01, ..., -4.65431027e-02, -2.18760711e-03, -2.81117141e-01], [ 5.85957587e-01, -1.68455601e-01, 3.16843003e-01, ..., -4.89029028e-02, -3.35241156e-03, -2.94326484e-01]], dtype=float32), }, }, }, layernorm_after: { bias: Array([ 6.88922033e-02, 5.80978803e-02, 3.85873437e-01, -3.11470646e-02, 1.69555983e-03, -2.32034996e-01, -1.58923380e-02, -3.62700596e-02, 5.75650949e-04, -3.82705927e-01, 2.31035739e-01, 6.68624789e-02, -1.72114998e-01, -5.66142239e-02, 2.09674109e-02, -2.80936039e-03, -2.27225497e-02, -3.59404653e-01, 1.31325588e-01, -3.94533155e-03, -1.76445603e-01, 4.06457812e-01, -3.71316727e-03, -2.46738438e-02, -1.81620270e-01, 4.64766249e-02, 7.05432147e-02, 7.28050619e-02, -5.19782789e-02, -4.45098169e-02, 3.48823667e-01, 6.42984435e-02, -3.04025542e-02, -9.68929306e-02, -5.14351986e-02, -3.97855928e-03, -3.20420787e-02, 2.67196506e-01, -7.22298101e-02, -2.07284018e-01, 4.90039028e-02, -2.19781891e-01, -1.35466352e-01, 8.54400694e-02, 1.19789563e-01, -1.32402414e-02, -2.06071258e-01, 3.45563710e-01, 1.00530580e-01, 8.49714726e-02, 3.91122419e-03, 2.15060711e-02, 8.78436714e-02, 2.07931370e-01, 3.97193991e-02, -1.13610223e-01, 3.73787954e-02, 1.23064490e-02, -9.43080038e-02, -1.86623499e-01, -3.85700941e-01, -1.63304899e-02, 1.10437632e-01, -1.03100136e-01, -6.19614311e-03, -3.90111618e-02, 1.53294563e-01, -1.00014314e-01, 8.98484327e-03, -1.33218288e-01, 1.19500153e-01, -1.21143341e-01, 2.10582778e-01, 1.27065793e-01, -1.65820807e-01, 2.54214227e-01, 5.12329042e-02, -9.36500803e-02, 5.69578595e-02, 3.65398824e-01, -1.66889839e-02, -8.60542282e-02, -2.17619090e-04, 7.38569945e-02, 1.74466029e-01, 7.53093883e-02, -1.50982007e-01, -1.61455601e-01, -4.24303383e-01, -1.41928092e-01, 1.67686343e-02, -2.84632683e-01, 1.45200789e-01, 5.28667821e-03, -2.16334872e-02, 4.74276505e-02, 1.93778221e-02, -1.02245426e-02, 8.27698484e-02, -4.25329745e-01, -9.56311524e-02, 2.45452020e-02, 5.58077872e-01, 8.93760026e-02, -3.42835188e-02, -7.27823749e-03, -4.62576747e-02, -1.11996412e-01, -2.86399245e-01, 3.93988378e-03, -5.40265851e-02, 1.06320893e-02, -1.11137971e-01, 9.71371382e-02, 7.98522159e-02, -7.13857710e-02, -4.83553410e-01, 1.70313239e-01, -1.70816466e-01, 8.59201849e-02, -3.64008136e-02, -2.24960689e-02, -3.12829800e-02, 1.58451021e-01, 9.86519852e-04, -3.24616969e-01, -2.68537730e-01, -1.29793763e-01], dtype=float32), scale: Array([ 0.16587867, 0.62729955, 0.78539467, 0.33312872, 0.02448054, 0.52365166, 0.09352965, 0.57490396, -0.00181935, 0.62166315, 0.5622861 , 0.632528 , 0.32434413, 0.33608988, 0.09364542, 0.00295282, 0.20884101, 0.4969771 , 0.49006283, 0.3390643 , 0.76553243, 0.75735354, 0.64736444, 0.20501752, 0.5099314 , 0.39772737, 0.4760952 , 0.2673762 , 0.5248422 , 0.34622046, 0.67513955, 0.12790139, 0.78136003, 0.46376252, 0.17142247, 0.02035132, 0.35290378, 0.79245657, 0.43180364, 0.58383363, 0.28102416, 0.6248626 , 0.09244277, 0.38200957, 0.36086422, 0.44281945, 0.62160456, 0.49688524, 0.47265998, 0.4368852 , 0.00439632, 0.24748251, 0.2077039 , 0.71143705, 0.55966383, 0.76935214, 0.2611507 , 0.3572642 , 0.4780699 , 0.5534745 , 0.5650403 , 0.49476576, 0.6574258 , 0.38081974, 0.15224549, 0.2871514 , 0.5269491 , 0.5443608 , 0.47709846, 0.6731636 , 0.48256594, 0.57757723, 0.52946705, 0.27599993, 0.11646109, 0.7459179 , 0.6025415 , 0.4426929 , 0.150529 , 0.6974664 , 0.07860193, 0.63189274, 0.6666647 , 0.8469711 , 0.4019909 , 0.45879367, 0.22790541, 0.5310602 , 0.46144342, 0.46353984, 0.3125676 , 0.6395497 , 0.5902147 , -0.00431166, 0.51785195, 0.44832194, 0.2982607 , -0.02292351, 0.48743743, 0.42023692, 0.5205952 , 0.08753084, 0.76318944, 0.42676944, 0.6018406 , 0.7163417 , 0.55152804, 0.8593687 , 0.4082306 , 0.4901796 , 0.64518034, 0.5525667 , 0.34930795, 0.46910506, 0.6457558 , 0.54612505, 0.61357117, 0.49350467, 0.41991618, 0.55927193, 0.4409146 , 0.15714881, 0.31889114, 0.26255667, 0.45123503, 0.56282735, 0.50075245, 0.34856102], dtype=float32), }, layernorm_before: { bias: Array([ 2.82420870e-02, -2.29402691e-01, -2.96053607e-02, -1.30823404e-02, -2.65389867e-03, 2.99529568e-03, -3.38511716e-04, -4.37224060e-02, 2.60172427e-01, -2.05475893e-02, 1.60881188e-02, -1.69427078e-02, -5.21531154e-04, 7.41502224e-03, -1.61934067e-02, 3.54598239e-02, -1.85820572e-02, 2.71367803e-02, -4.88283485e-03, -6.32894114e-02, 3.68127339e-02, -6.04641587e-02, 1.19915558e-03, 1.77571445e-03, -7.46919066e-02, -1.53914511e-01, -3.68062826e-03, 1.60319414e-02, 6.28713518e-02, 3.06911282e-02, -2.31577735e-02, -6.57155272e-03, 6.15396686e-02, 2.63886862e-02, -4.85325791e-02, 7.88616296e-03, 8.01346614e-04, -1.55313745e-01, -2.11913697e-03, -6.54404704e-03, 5.55190295e-02, -7.31622195e-03, 1.96012199e-01, 5.94329368e-03, -2.78697144e-02, -4.93782759e-02, 3.01586390e-02, 6.05354905e-02, -1.21998333e-03, 3.04064993e-02, 1.04687298e-02, 1.09375557e-02, -2.71287002e-03, -2.32986733e-01, 1.44290384e-02, -2.30109710e-02, 4.33547981e-02, 5.82204685e-02, -2.50215475e-02, 1.38790637e-01, -6.68904185e-02, 1.24691881e-01, -4.73514060e-03, -3.82121443e-03, 3.83599102e-02, 8.80813003e-02, -1.65379103e-02, -7.19220552e-04, 3.60673331e-02, 1.00804691e-03, -1.36750117e-02, -2.13624742e-02, 3.65178101e-02, 2.25900616e-02, -4.44582880e-01, -2.65965145e-02, -5.42465337e-02, -7.62177110e-02, 6.57133060e-04, -1.34424558e-02, 3.63113172e-02, 7.55314380e-02, 8.09766643e-04, -2.51245238e-02, -2.74020876e-03, -4.82323244e-02, -7.02568442e-02, 1.68840366e-03, 1.02340458e-02, 1.46568000e-01, -2.30909362e-02, 4.56218012e-02, -6.14786409e-02, -2.31754649e-02, 1.12327132e-02, 8.98616686e-02, 3.33565250e-02, 3.43459705e-03, -1.15024485e-02, -9.24066082e-02, -3.95509861e-02, -2.51406804e-03, -9.44837704e-02, -6.91622263e-04, -3.78696583e-02, -7.13908523e-02, -6.70393333e-02, 2.14871876e-02, -3.32830548e-02, -9.49056670e-02, 3.31703201e-02, 5.51439868e-03, -1.63990129e-02, 1.68498412e-01, -1.52823254e-01, 1.28563598e-01, 2.69770622e-02, -2.52538477e-03, -6.20844103e-02, -2.31217127e-03, -1.12146828e-02, 2.15321942e-03, -1.07783070e-02, 7.04224780e-03, -5.70042757e-04, -1.27387140e-02, -1.50637852e-03, -1.26768842e-01], dtype=float32), scale: Array([ 0.02225583, 0.3582427 , 0.18810613, 0.2897796 , 0.01008445, 0.20093079, 0.00262845, 0.18910973, 0.20207141, 0.05451531, 0.13650979, 0.22285123, 0.04530312, 0.00609608, 0.10985828, 0.09655713, 0.13278736, 0.08787937, 0.02501552, 0.16963154, 0.22782324, 0.20419084, 0.23176458, 0.04793768, 0.12091924, 0.15503103, 0.12642743, 0.07698647, 0.37561363, 0.1734934 , 0.22910334, 0.00436596, 0.25682634, 0.24328624, 0.2714059 , 0.06567708, 0.02274263, 0.567844 , -0.00143492, 0.16284508, 0.18691777, 0.3501844 , 0.11066452, 0.01387185, 0.17211044, 0.31377098, 0.1559427 , 0.31400132, 0.03055096, 0.1480449 , 0.06916483, 0.05761943, -0.00319004, 0.38499779, 0.0325634 , 0.3114719 , 0.07862399, 0.19379751, 0.12478653, 0.41179135, 0.26195255, 0.3847175 , 0.16452418, 0.00272222, 0.15891619, 0.11142635, 0.14923933, 0.10400768, 0.21697171, 0.12220434, 0.24840005, 0.16391885, 0.09268359, 0.12174353, 0.12482438, 0.2716236 , 0.22829536, 0.2231431 , -0.0092565 , 0.03284183, 0.06326266, 0.15956563, 0.06667496, 0.22676413, 0.01954163, 0.2572471 , 0.14380816, 0.3025213 , 0.13769881, 0.38677344, 0.05833767, 0.11660744, 0.1784911 , 0.31127843, 0.0378647 , 0.11842007, 0.13203348, 0.08775622, 0.22929348, 0.1685133 , 0.17730162, 0.14410907, 0.3452061 , -0.01377543, 0.44221473, 0.43522978, 0.29845122, 0.13120209, 0.45809656, 0.19538262, 0.46633697, 0.1401029 , 0.05973453, 0.4215374 , 0.16866335, 0.27390364, 0.03285838, 0.01355824, 0.18128489, 0.2254643 , 0.03559769, 0.00747606, 0.18407959, 0.02931737, -0.00183865, 0.0155194 , 0.2910443 , 0.2715314 ], dtype=float32), }, moe: { shared_experts: { intermediate: { bias: Array([-2.87663639e-01, -2.04487324e-01, -3.44864547e-01, -2.00529769e-01, -5.94829321e-01, -1.03927039e-01, -2.77930766e-01, -4.94999826e-01, -2.08718747e-01, -3.45035255e-01, -2.67087758e-01, -4.77189213e-01, -1.66359320e-01, -4.48330075e-01, -4.78699297e-01, -2.21549183e-01, -2.41500542e-01, -3.03187490e-01, -4.03688133e-01, -3.62672597e-01, 3.11330687e-02, -4.75372255e-01, -1.33399740e-01, -1.58057332e-01, -1.38183832e-01, -3.53402674e-01, -4.83101279e-01, -3.49856049e-01, -4.62018728e-01, -2.20034316e-01, -1.45977482e-01, -1.44924179e-01, -3.43084693e-01, -2.04671875e-01, -2.16533989e-01, -5.23977339e-01, -2.12659031e-01, -2.74191111e-01, -2.65450865e-01, -4.38724220e-01, -4.05285835e-01, -4.25657749e-01, -2.83261627e-01, -1.07905425e-01, -3.10509741e-01, -3.69468153e-01, -5.25859118e-01, -3.74539018e-01, -4.44122553e-01, -2.76227951e-01, -3.62340897e-01, -3.20163608e-01, -5.22766829e-01, -3.68136406e-01, -6.11795008e-01, -1.88745782e-01, -4.54058141e-01, -1.91686556e-01, -4.25439298e-01, -5.22909224e-01, -3.05993110e-01, -3.57376486e-01, -3.10534716e-01, -2.63750911e-01, -5.45675814e-01, -4.23284829e-01, -1.21132560e-01, -3.99448633e-01, -1.06209874e-01, -3.28158796e-01, -5.71545482e-01, -1.91518441e-01, -5.93747199e-01, -3.57770473e-01, -2.59463459e-01, -3.54470700e-01, -5.28468788e-01, -3.18399936e-01, -4.58652407e-01, -2.15082556e-01, -4.01729345e-01, -5.17883539e-01, -3.26544613e-01, -2.71224737e-01, -3.98273647e-01, -3.58481169e-01, -4.33161110e-01, -4.17540729e-01, -4.41764146e-01, -4.13900316e-01, -4.20035869e-01, -4.61230427e-01, -5.54932415e-01, -2.82214522e-01, -4.63347554e-01, -1.69810832e-01, -4.96465772e-01, -3.57517421e-01, -3.08119923e-01, -5.05688906e-01, -4.42314088e-01, -2.96192914e-01, -3.34470212e-01, -1.58352673e-01, -1.96208924e-01, -1.71411768e-01, -2.78193712e-01, -4.14353907e-01, -5.30355275e-01, -6.04073517e-02, -3.42649519e-01, -3.25455546e-01, -2.80562103e-01, -3.16360086e-01, -1.88339218e-01, -1.45719707e-01, -2.42292494e-01, -4.38852787e-01, -3.38206112e-01, -3.96864712e-01, -3.94322276e-01, -5.04216313e-01, -4.47647184e-01, -3.14450145e-01, -4.03099447e-01, -3.31715465e-01, -5.11624277e-01, -2.97003418e-01, -1.71126753e-01, -5.23082614e-01, -3.97670537e-01, -3.33513230e-01, -2.73618609e-01, -2.16470793e-01, -3.16209495e-01, -3.54464471e-01, -4.11175519e-01, -3.47123802e-01, -4.30214375e-01, -2.96357393e-01, -3.50862324e-01, -4.91615891e-01, -2.99721777e-01, -3.46929997e-01, -6.17211461e-01, -3.17567289e-01, -2.59017050e-01, -5.43685555e-01, -3.07671517e-01, -3.49160552e-01, -5.83933473e-01, -2.99324632e-01, -3.56959522e-01, -4.75278974e-01, -4.69556153e-01, -4.00068551e-01, -3.57878387e-01, -2.61400610e-01, 5.89218475e-02, -4.75838035e-01, -3.18645447e-01, -2.91967630e-01, -3.57156008e-01, -5.22750139e-01, -1.99007168e-01, -4.69590366e-01, -3.96272272e-01, -2.64699906e-01, -3.60923529e-01, -1.91638634e-01, -3.21910053e-01, -2.48550743e-01, -3.98402393e-01, -4.65990812e-01, -3.43011886e-01, -3.76989871e-01, -3.99559021e-01, -3.55907410e-01, -4.88500834e-01, -3.14257860e-01, -5.45107484e-01, -1.72966227e-01, -3.69181097e-01, -2.42310494e-01, -3.32116067e-01, -3.86646032e-01, -2.56473958e-01, -3.48980218e-01, -3.98971230e-01, -2.64990151e-01, -4.63962048e-01, -1.67646125e-01, -3.03776145e-01, -2.27759063e-01, -4.02701110e-01, -3.71156693e-01, -4.49759185e-01, -4.83259469e-01, -4.44558293e-01, -3.49739552e-01, -4.34326917e-01, -2.64975369e-01, -2.89425910e-01, -4.49308783e-01, -2.49016702e-01, -4.31563139e-01, -4.08892900e-01, -3.69399726e-01, -6.46372512e-02, -3.25122327e-01, -1.55182466e-01, -1.11164600e-01, -4.79935288e-01, -5.89126289e-01, -5.05477190e-01, -3.91809165e-01, -4.87700224e-01, -5.42157114e-01, -5.01625121e-01, -3.88096064e-01, -4.09615874e-01, -3.30858119e-03, -3.16691220e-01, -4.25819814e-01, -2.07181290e-01, -2.52645940e-01, -4.78967577e-01, -4.66646552e-01, -5.08263230e-01, -2.38753840e-01, -3.27672154e-01, -5.24233878e-01, -3.54793280e-01, -3.08880091e-01, -2.62713552e-01, -3.00624251e-01, -4.21428025e-01, -4.42182183e-01, -5.11559665e-01, -2.42620170e-01, -3.54224473e-01, -3.06683362e-01, -4.52630949e-04, -4.44990546e-01, -3.27576101e-01, -2.59946615e-01, -5.11295557e-01, -6.64835870e-01, -1.85168549e-01, -5.48847735e-01, -2.54403502e-01, -4.44106311e-01, -2.09086508e-01, -5.46322465e-01, -4.56374437e-01, -2.26774529e-01, -3.03409159e-01, -2.95624912e-01, -4.80216295e-01, -1.95244759e-01, -2.66378999e-01, -3.54911506e-01, -3.88011873e-01, -2.92912245e-01, -4.73809302e-01, -2.48538107e-01, -2.53374189e-01, -5.34175217e-01, 1.01788407e-02, -4.37775195e-01, -3.74520659e-01, -4.16397095e-01, -2.86437750e-01, -1.17193200e-01, -4.65466827e-01, -2.45054185e-01, -1.55902386e-01, -2.60219306e-01, -3.81529987e-01, -4.36873585e-01, -3.58277500e-01, -4.33071882e-01, -4.30247694e-01, -3.75352204e-01, -3.81595612e-01, -4.86366630e-01, -4.01012272e-01, -2.53684759e-01, -3.21343958e-01, -5.21693230e-01, -2.32155509e-02, -3.01609904e-01, -2.36861959e-01, -3.18232119e-01, -1.21864028e-01, -3.91450346e-01, -5.88354170e-01, -3.05314749e-01, -3.78017843e-01, -9.56703201e-02, -3.85766625e-01, -2.07299024e-01, -8.47784728e-02, -3.33295763e-01, -4.15781707e-01, -4.48703825e-01, -1.40794486e-01, -1.18951991e-01, -2.74386436e-01, -4.96934414e-01, -3.64418119e-01, -3.70397896e-01, -1.35903224e-01, -1.35731950e-01, -3.99609268e-01, -2.93618023e-01, -3.04051191e-01, -5.70618927e-01, -5.02060175e-01, -2.81283766e-01, -6.31259501e-01, -3.06242526e-01, -2.12949291e-01, -4.86519247e-01, -1.59262896e-01, -4.03478205e-01, -1.81495771e-01, -4.62975919e-01, -3.86971891e-01, -5.12782514e-01, -2.76577592e-01, -2.94954062e-01, -2.17922330e-01, -4.69827503e-01, 1.06356489e-02, -3.15419227e-01, -3.53538573e-01, -5.48389792e-01, 5.30533865e-02, -3.23666394e-01, -3.68639082e-01, -3.19764733e-01, -4.70012695e-01, -9.86519307e-02, -3.48372728e-01, -2.50590980e-01, -3.00890654e-01, -4.49694991e-01, -3.42396975e-01, -2.40296051e-01, -1.83449000e-01, -4.94159639e-01, -3.83832961e-01, -2.35955358e-01, -4.39356416e-01, -2.83128679e-01, -5.06890595e-01, -4.84923810e-01, -3.48398417e-01, -8.36336315e-02, -3.36799473e-01, -4.19064090e-02, -5.25771677e-01, -4.51774269e-01, -4.36057091e-01, -4.80761558e-01, -3.56559277e-01, -4.32838291e-01, -4.25217807e-01, -2.08370864e-01, -6.56363890e-02, -5.52178204e-01, -4.09472764e-01, -1.91283375e-01, -1.69247791e-01, -3.70922804e-01, -4.36929941e-01, -1.94023415e-01, -2.98483998e-01, -1.78429320e-01, -2.56660700e-01, -3.49405706e-01, -2.60902107e-01, -1.27692908e-01, -2.62808502e-01, -3.31191778e-01, -5.31970501e-01, -2.73841202e-01, -3.46037507e-01, -2.07595572e-01, -3.45578700e-01, -3.25063765e-01, -3.59042734e-01, -5.38096130e-01, -3.52915227e-01, -2.89237618e-01, -4.53390926e-01, -3.57132286e-01, -2.32824370e-01, -1.73414394e-01, -3.02039832e-01, -4.06891137e-01, -3.11633587e-01, -3.86790335e-01, -4.44779336e-01, -4.03608739e-01, -3.27035785e-01, -5.77829838e-01, -3.05345178e-01, -5.01044929e-01, -4.90038365e-01, -3.42383653e-01, -2.16015473e-01, -2.34810472e-01, -3.94850314e-01, -3.71235818e-01, 1.26175291e-03, -4.32831019e-01, -1.67813487e-02, -4.22147512e-01, -3.73436451e-01, -4.40798849e-01, -4.10787642e-01, -4.72505748e-01, -2.18411952e-01, -3.72730881e-01, -3.14533114e-01, -2.04394564e-01, -2.78175622e-01, -2.42158204e-01, -3.91372234e-01, -5.00931203e-01, -2.58081257e-01, -4.08814311e-01, -4.02594358e-02, -3.35627794e-02, -2.13214949e-01, -3.36978287e-01, -2.62033820e-01, -6.90276861e-01, -3.95373791e-01, -4.39152598e-01, -2.85017580e-01, -4.47889686e-01, -4.36591983e-01, -3.10838968e-01, -4.41035658e-01, -2.79656291e-01, -1.40883625e-01, -3.65119666e-01, -1.61859736e-01, -1.76689908e-01, -3.66094023e-01, -4.24499989e-01, -4.19989884e-01, -2.08415791e-01, -2.32176647e-01, -3.55920881e-01, -2.76795477e-01, -4.64629948e-01, -2.86160856e-01, -5.59463799e-01, -3.76166940e-01, -3.82688463e-01, -4.46783721e-01, -3.94609421e-01, -3.84682864e-01, -2.80228592e-02, -3.13305557e-01, -3.56978267e-01, -4.11862016e-01, -6.72479212e-01, -2.63180643e-01, -2.53385037e-01, -4.69251305e-01, -4.94098127e-01, -3.67969900e-01, -5.01329899e-01, -4.63324428e-01, -4.17908072e-01, -2.42769625e-03, -3.66800427e-01, -4.42783326e-01, -5.09920537e-01, -3.79778743e-01, -2.11185992e-01, -4.76333320e-01, -2.97944933e-01, -5.35430849e-01, -5.54302871e-01, -1.78685069e-01, -5.05887330e-01, -3.60490650e-01, -2.97754109e-02, -4.16262656e-01, -2.02613860e-01, -4.54910725e-01, -5.27239859e-01, -3.19837511e-01, -3.31910402e-01, -5.04021525e-01, -5.96606016e-01, -2.31208146e-01, -5.22311389e-01, -3.61870646e-01, -2.52372891e-01, -5.43789089e-01, -3.68237197e-01, -2.00233698e-01, -3.31239879e-01, -3.89182925e-01, -1.77743852e-01], dtype=float32), kernel: Array([[-0.03629679, 0.12771748, 0.30917847, ..., -0.02645334, 0.2853967 , 0.25290728], [-0.05366539, -0.08199085, -0.77184826, ..., -0.28385365, -0.08720582, -0.2480316 ], [-0.30405623, -0.14440256, -0.16176778, ..., -0.6061087 , -0.53592414, -0.557683 ], ..., [-0.5002675 , 0.6923438 , -0.39955443, ..., 0.2793648 , -0.0890847 , 0.13734408], [ 0.42013338, -0.1837182 , 0.20519936, ..., 0.59244996, 0.15519385, -0.04555327], [ 0.2887559 , 0.03648978, 0.19605577, ..., -0.3638592 , -0.05200996, 0.14020972]], dtype=float32), }, output: { bias: Array([-2.03454956e-01, 2.89009154e-01, -5.26833301e-03, 1.99230332e-02, 7.02989474e-03, -8.10585693e-02, 1.01039574e-01, 1.28230199e-01, 1.65420137e-02, 1.76779047e-01, 2.31052637e-02, -2.84619331e-02, -1.72500789e-01, 8.57431665e-02, -2.58707665e-02, 8.68742466e-02, 3.48051847e-03, -8.32902640e-02, 8.47522356e-03, -6.26579765e-03, 6.64773211e-02, -5.39215617e-02, -1.33164413e-02, -2.61053070e-02, -1.25059590e-01, 9.69652552e-03, -6.74541369e-02, 5.93836829e-02, 6.43064156e-02, 6.59576431e-02, 1.18919395e-01, -1.21684916e-01, 1.34139672e-01, -7.79727027e-02, 7.02831373e-02, -7.30708465e-02, 2.56273188e-02, 8.22603703e-03, 5.56182116e-03, -1.27773434e-01, -1.63553078e-02, -2.40529538e-03, -7.88959324e-01, 2.98491977e-02, -1.15960285e-01, -1.11549925e-02, 1.14158385e-01, 4.73596305e-02, -4.17886041e-02, -5.26564419e-02, -9.78346448e-03, 4.26121093e-02, -1.87094569e-01, 8.54555890e-02, -4.69494089e-02, -3.20936963e-02, -1.27747595e-01, -3.98493744e-02, 1.56673312e-01, 5.93540259e-02, 1.29704997e-01, -2.26318836e-01, -4.20836620e-02, 1.89136285e-02, 2.25093830e-02, -1.10592782e-01, -8.14836994e-02, -1.74993854e-02, 6.80994093e-02, -1.73572097e-02, -1.06341034e-01, 2.98510771e-03, -6.83057960e-03, -4.16511074e-02, 8.99981439e-01, 1.04607493e-02, -1.10422350e-01, 4.36907373e-02, -1.15947202e-01, -4.37050452e-03, -4.16922748e-01, 3.20061632e-02, 4.93779741e-02, 2.31584487e-03, -1.06097618e-02, 8.79805684e-02, 1.83927685e-01, 1.49958774e-01, 1.10785790e-01, 1.96547806e-02, -6.48459792e-03, 2.37252992e-02, -1.06167868e-01, -1.58290878e-01, -3.44699137e-02, -5.64344190e-02, 2.31592474e-03, 1.24252474e-04, -6.33904487e-02, 1.05357006e-01, 2.25357682e-01, -6.24567755e-02, 3.09208464e-02, -1.09267943e-01, 1.43063843e-01, -6.33983240e-02, -1.62469558e-02, -1.03259414e-01, -8.94914567e-02, -3.48297022e-02, -1.44557551e-01, 1.30005326e-04, 1.44872010e-01, -1.74515136e-02, 4.52921912e-02, 1.08484700e-01, -9.68603417e-03, -4.61588614e-03, -3.65241021e-02, -3.43756750e-04, 1.57616019e-01, 8.44166130e-02, 2.49973107e-02, -9.32366401e-02, 2.59006135e-02, 4.68501188e-02, 5.38958088e-02, 1.02374971e-01], dtype=float32), kernel: Array([[-0.35964 , 0.58528113, 0.12339137, ..., -0.2728965 , -0.35934824, -0.06503713], [ 0.35246447, -0.02949085, 0.5403422 , ..., -0.09529549, 0.11235058, -0.22131431], [ 0.13894935, -0.3717892 , -0.66237545, ..., -0.41706035, 0.52529556, 0.16840486], ..., [-0.09083151, 0.03931169, 0.2619683 , ..., -0.20559356, 0.15253511, 0.3468645 ], [-0.02631908, -0.0662327 , 0.12175545, ..., 0.10028415, -0.3420218 , 0.01938576], [-0.15720125, 0.16154028, -0.15429054, ..., -0.42136976, 0.29129225, 0.06028228]], dtype=float32), }, }, }, }, 2: { attention: { attention: { key: { bias: Array([ 4.91330713e-01, 9.67662875e-03, 5.60255200e-02, 8.97967637e-01, -3.06834251e-01, 1.06595111e+00, 5.69995821e-01, -9.72940326e-01, -1.00492425e-01, 2.51781523e-01, -3.73477489e-01, 1.62789464e-01, -3.40229303e-01, 8.37876360e-05, 4.15999711e-01, 8.95448923e-02, -1.16417743e-01, 1.02886930e-01, -1.22432733e+00, -2.46076301e-01, -5.80214441e-01, -4.96384725e-02, -2.68343747e-01, -3.29451412e-01, 2.24202454e-01, -6.01632036e-02, -6.19756997e-01, -3.03309470e-01, -1.28122672e-01, -2.58336246e-01, 4.15848672e-01, 6.79572225e-02, 1.68727383e-01, -1.08576640e-01, 9.50729072e-01, 1.18212245e-01, 2.77229160e-01, 4.02590066e-01, 2.45219976e-01, -2.20667735e-01, -6.55377328e-01, -2.22052753e-01, -2.34352529e-01, 7.21003652e-01, -2.47502819e-01, -8.39031518e-01, 7.14816153e-01, 5.66149652e-01, 1.96338549e-01, -3.21684837e-01, 1.26060262e-01, -1.13100424e-01, 7.06214011e-02, -3.80029410e-01, -5.25692463e-01, 5.33015609e-01, -2.28610292e-01, 8.77432287e-01, 4.54767972e-01, 1.05058357e-01, -1.01594612e-01, -4.25737798e-01, 8.72221589e-01, -4.27799642e-01, -7.26574242e-01, 3.97354126e-01, 5.40428281e-01, -6.92076981e-01, -7.33269453e-01, -8.38262439e-01, -9.10226703e-01, -2.79593736e-01, 5.23640335e-01, 2.84005404e-01, 3.39674622e-01, -2.09556028e-01, -1.83250681e-02, 2.20996439e-01, 8.39932729e-03, -1.35142893e-01, -4.47591990e-01, -5.44816703e-02, 3.56293529e-01, 3.84467632e-01, -4.40692216e-01, 1.00856110e-01, 8.52475286e-01, 3.95925105e-01, -5.06571472e-01, 4.37813222e-01, -8.72387230e-01, -4.80205119e-01, -6.85112596e-01, -6.57971025e-01, 3.89101595e-01, -2.59548306e-01, -2.97444463e-01, 2.12126691e-02, -8.14053118e-02, 1.06517822e-01, 2.81973779e-01, 7.66788945e-02, -4.01952624e-01, -1.77236781e-01, 6.38018996e-02, -3.04998636e-01, -3.08628559e-01, 1.56093314e-01, 1.61672279e-01, 1.33270577e-01, 1.59121871e-01, -1.46263152e-01, 1.54846618e-02, 2.81419247e-01, -3.13295014e-02, 2.87094891e-01, -3.95765811e-01, 4.69570488e-01, -4.26991880e-01, 2.32855052e-01, -1.62792206e-01, -3.81025262e-02, 4.74744802e-03, -7.95326605e-02, 1.28198788e-02, 1.01442777e-01, -4.87101451e-02, -3.22961807e-02], dtype=float32), kernel: Array([[-0.20145601, -0.22677487, -0.05235473, ..., -0.06058222, -0.01446399, 0.16699584], [ 0.22185628, 0.15144752, 0.23851915, ..., -0.10337625, -0.07871008, 0.01390354], [ 0.01399281, 0.06500424, -0.15749013, ..., 0.14727156, -0.08356368, 0.03307934], ..., [ 0.01401845, -0.0230756 , -0.0378951 , ..., -0.40631384, -0.10683019, 0.05925401], [-0.00425362, -0.07440671, 0.03620811, ..., 0.21968801, -0.18367934, 0.02989441], [-0.1020968 , 0.5364823 , 0.00217419, ..., -0.05559423, 0.13003536, -0.06085972]], dtype=float32), }, query: { bias: Array([ 0.3528763 , 0.3471857 , 0.21516396, 0.44737092, -0.4285367 , 0.3081475 , -0.25559098, -0.5126486 , 0.21324182, -0.32549337, 0.02813848, -0.38898697, -0.34589097, -0.19787419, 0.28709203, -0.24419545, -0.26928642, 0.30525193, -0.32151905, 0.0448105 , 0.51535946, -0.11308531, 0.3212626 , 0.4922951 , 0.26803052, -0.38731664, -0.2744102 , 0.39860168, 0.4643956 , 0.20418273, 0.14548776, 0.0657926 , -0.21296258, 0.15750653, -0.3648503 , 0.61559546, -0.35865372, 0.0193048 , 0.31175947, 0.2440965 , 0.318281 , -0.7344339 , -0.88320374, -0.05781611, 0.18718946, 0.20290868, -0.23830453, 0.5799299 , 0.01096278, -0.4863952 , -0.03866521, 0.30802044, 0.44045374, -0.03443577, -0.02167755, -0.01554048, -0.511046 , 0.41503018, -0.58751154, 0.5616692 , -0.26840666, -0.20692529, 0.71333766, 0.2290065 , 0.15057702, 0.1327177 , 0.31366843, 0.3856127 , -0.18231715, -0.24914798, 0.321846 , -0.3574352 , -0.12077462, 0.36877826, 0.38871482, -0.3660267 , 0.7790568 , 0.06110919, -0.22040306, -0.23536305, -0.07361323, -0.47640693, 0.53327733, 0.49116883, -0.5596567 , -0.7786464 , -0.1766858 , -0.14165151, -0.4430259 , 0.38748688, 0.02364437, -0.46849003, -0.23776187, -0.45196384, 0.65864664, 0.39194986, 0.07788196, 0.10206337, 0.15233228, 0.01135532, 0.08919958, -0.0229385 , -0.00552564, -0.6363855 , 0.04719624, 0.07559558, 0.0246234 , 0.32256266, -0.02410899, -0.37407702, -0.063107 , -0.24087183, -0.0432428 , 0.01937803, -0.12409952, 0.38090006, -0.8962724 , 0.51235485, -0.21019219, -0.06553419, -0.0576391 , -0.00670896, 0.0239524 , 0.00629916, 0.46627957, 0.02785698, 0.03592508, 0.06080098], dtype=float32), kernel: Array([[ 0.09377078, 0.06110698, -0.01986536, ..., -0.09585967, -0.00882398, -0.09038808], [-0.1099542 , -0.0294884 , -0.3200316 , ..., 0.14632395, -0.085163 , -0.12561767], [-0.17357844, -0.08655021, -0.17681202, ..., 0.2461566 , -0.11222455, 0.1451854 ], ..., [ 0.01483 , 0.12659054, 0.3020677 , ..., -0.1301052 , -0.02523333, 0.0047435 ], [-0.21112014, -0.10351861, 0.12283702, ..., 0.04758199, -0.02335739, -0.20571615], [ 0.04751823, -0.08418679, 0.021736 , ..., 0.23970562, -0.10594929, -0.2381722 ]], dtype=float32), }, value: { bias: Array([-0.20210104, -0.22905709, -0.14039448, -0.03683947, 0.12594987, -0.02302479, 0.00906336, -0.00965133, -0.07539421, 0.19282112, -0.14035326, -0.02629162, -0.0613457 , 0.19251512, 0.22437336, 0.01865598, 0.09472503, 0.07302232, 0.23662573, 0.1634112 , -0.25386894, 0.03644704, -0.14476314, -0.15108086, 0.30487335, -0.01728077, -0.03989496, 0.10778196, 0.01612386, -0.02545331, 0.10517047, 0.00122259, 0.06249418, 0.1109454 , 0.07476445, -0.00254933, -0.08234914, 0.00984051, -0.18144692, 0.16224049, 0.0791403 , 0.03824679, 0.05325351, 0.0459534 , -0.01707704, -0.05914249, -0.07880631, 0.09950723, 0.01833864, -0.12028188, 0.0201016 , 0.17571941, 0.08609623, -0.07171538, 0.27483281, 0.19407238, 0.01660765, 0.03474303, 0.09089345, 0.19815543, 0.21818227, 0.1436114 , 0.00318472, 0.06330492, -0.06290373, 0.03819067, -0.06003238, -0.03846627, 0.07632727, -0.05140862, -0.07411028, 0.21224949, -0.07434353, -0.09569198, -0.01517388, 0.00378666, 0.0954181 , 0.02467517, -0.0480962 , -0.07227837, 0.08442853, -0.11686811, 0.04613666, -0.04021059, 0.03575824, 0.02964796, 0.09947117, 0.07083944, -0.06642204, -0.07676494, 0.02942543, -0.0550985 , -0.06826791, 0.06714635, -0.05162644, -0.03433613, -0.00067884, 0.05411651, -0.01232108, 0.10868998, 0.05962406, -0.00136539, 0.01625578, -0.00632739, -0.0961687 , -0.07040943, 0.02730714, 0.00544707, 0.01023827, 0.03284573, 0.02555647, 0.0085297 , 0.03900068, 0.02747485, -0.03433363, -0.01524182, -0.00976253, -0.06111204, 0.10030263, 0.04285727, -0.00326952, -0.04434157, 0.11089424, 0.04277305, 0.02115745, 0.02765508, 0.08399435, 0.08146379], dtype=float32), kernel: Array([[-1.31283207e-02, 5.85139990e-02, 6.22775187e-05, ..., 4.85775769e-02, -4.02368233e-02, -6.80026598e-03], [-2.73164004e-01, -3.41370434e-01, -2.39238873e-01, ..., 4.36691791e-01, 3.35174829e-01, 8.86783656e-03], [ 6.34363815e-02, 3.22204083e-01, 2.63729244e-01, ..., -9.32745263e-03, -1.88997626e-01, 2.92390492e-02], ..., [-1.07346885e-01, -1.71600416e-01, -1.54479533e-01, ..., -2.66532987e-01, -7.03294575e-02, -3.63519728e-01], [-1.77037895e-01, -1.31959140e-01, 3.59883085e-02, ..., 1.12166613e-01, -3.25531811e-01, 4.15854841e-01], [ 1.20714135e-01, 1.70483068e-01, 6.37787461e-01, ..., -3.24400365e-01, -7.82848775e-01, -1.08967312e-01]], dtype=float32), }, }, output: { dense: { bias: Array([-1.42582014e-01, 3.51325125e-01, -5.49227633e-02, -1.36979753e-02, -8.89001694e-03, -4.82381359e-02, 7.59436563e-02, 4.62903231e-02, 1.25182196e-02, 1.74479246e-01, 5.47183445e-03, -4.21341211e-02, -1.15092970e-01, 5.72893657e-02, -5.69992810e-02, 5.97054660e-02, 2.31458675e-02, -9.98827890e-02, -1.63127463e-02, -5.68029238e-03, 1.06309690e-01, -5.58492132e-02, -2.88810022e-02, -5.41146100e-02, -1.25681654e-01, 1.55297220e-02, -1.05181850e-01, 2.84042470e-02, 8.00593048e-02, 1.12994172e-01, 6.64846674e-02, -8.10639188e-02, 6.96494877e-02, -7.10230693e-02, 2.48000082e-02, -6.40962422e-02, 5.03854305e-02, 2.99889389e-02, 2.71755550e-02, -1.32961541e-01, -9.42786504e-03, 6.68036118e-02, -7.92975008e-01, 2.92515382e-02, -1.21656477e-01, 6.97143525e-02, 9.77309272e-02, 9.14023891e-02, -1.06936611e-01, -4.60654832e-02, -3.59379910e-02, 6.73950538e-02, -3.20891030e-02, 1.82818487e-01, -4.29846235e-02, 1.71033782e-03, -2.07776949e-01, -8.03176984e-02, 1.26594633e-01, 1.53843045e-01, 9.50956270e-02, -1.38820022e-01, -5.31725921e-02, 3.35701555e-02, 5.83558269e-02, -1.29202589e-01, -4.29277681e-02, 5.83481155e-02, 1.43310845e-01, 3.71531360e-02, -1.16031744e-01, 1.25720399e-03, -3.66386138e-02, -5.68410717e-02, 8.81127954e-01, -3.54486227e-04, -1.14806622e-01, 5.00139184e-02, -1.25719443e-01, -2.92788483e-02, -5.60584605e-01, 1.90249607e-02, 7.62136094e-03, 3.29709910e-02, -1.73091292e-02, 1.49499878e-01, 1.71786591e-01, 1.50196671e-01, -4.66331327e-03, 7.90596679e-02, -6.07569050e-03, 1.27625940e-02, -6.54113144e-02, -1.46687955e-01, -2.00468884e-03, -7.76169971e-02, -5.61596686e-03, -2.51500830e-02, -5.23540825e-02, 1.42120197e-01, 1.58276126e-01, -3.11181620e-02, 4.16391157e-02, -9.11307335e-02, 1.01979673e-01, -1.40924305e-01, -2.65695825e-02, 3.79197043e-03, -9.74692628e-02, -5.42117134e-02, -1.30008578e-01, 3.88463400e-02, 1.63258657e-01, -1.50325231e-03, 3.45914699e-02, 1.69843420e-01, 1.55994203e-02, -1.33964028e-02, -1.95804201e-02, 1.02751255e-02, 1.84371233e-01, 7.10138083e-02, 5.72973862e-03, -1.13018960e-01, -8.22554808e-03, 5.45389391e-02, 1.77288230e-03, 1.88416634e-02], dtype=float32), kernel: Array([[ 0.15479796, 0.53571814, 0.51493394, ..., -0.80009085, -0.05134518, 0.13081257], [-0.00908494, 0.42273548, 0.710739 , ..., -0.3385195 , -0.41763583, 0.50498915], [ 0.21792167, 0.02829238, -0.15942883, ..., 0.14649649, -0.3748477 , 0.5899068 ], ..., [ 0.69713473, -0.10411429, 0.20231156, ..., 0.13196032, -0.09708058, -0.21109731], [ 0.5224463 , -0.00511254, -0.9390049 , ..., 0.06903978, 0.24796766, 0.5909974 ], [-0.6673977 , 0.24624978, -0.7719052 , ..., -0.09019069, -0.29529244, -0.12763341]], dtype=float32), }, }, }, layernorm_after: { bias: Array([-2.69522518e-02, 7.07723275e-02, -1.92383816e-03, -1.69017464e-01, 6.28127530e-03, 1.35174885e-01, 1.43263070e-02, -1.49136512e-02, 1.43651441e-02, 2.47675721e-02, 2.31170952e-01, -1.36354938e-02, -5.00643514e-02, 4.64095622e-02, 7.30278119e-02, 2.58173402e-02, -7.96374157e-02, -9.20170024e-02, -1.74471997e-02, 1.98375121e-01, 2.97931284e-01, 1.71703979e-01, -1.74371853e-01, -8.47198628e-03, -2.85758138e-01, 2.43080053e-02, -2.33928546e-01, 1.65304869e-01, 1.08051397e-01, -3.64561914e-03, -4.06746566e-01, -2.43781097e-02, -1.94358621e-02, 2.50640996e-02, 1.21675162e-02, 3.38391075e-03, 4.25222097e-03, -1.16181098e-01, -6.87912405e-02, -1.43533750e-02, 6.30778596e-02, 5.16471975e-02, 2.78629586e-02, -6.81551844e-02, -2.16658652e-01, 1.19643852e-01, -1.77933127e-02, 4.19648476e-02, -6.94157332e-02, 8.80410895e-02, 3.07484251e-03, -2.26102751e-02, 2.71407124e-02, -1.37352645e-02, 1.24339305e-03, -1.10182655e-03, -4.25179452e-02, 3.90366204e-02, -6.09224401e-02, 7.32898619e-03, -5.39776146e-01, 8.10408741e-02, 1.43810920e-02, -1.46246385e-02, 1.65752345e-03, -2.01134533e-01, 1.59140024e-02, 4.93869092e-03, 4.72723007e-01, -1.37103930e-01, 1.31060192e-02, -4.23336178e-02, 4.09131646e-02, 3.04093435e-02, 5.28761335e-02, 3.03985029e-01, -2.57239580e-01, 1.94385331e-02, 2.98021808e-02, -3.17842066e-02, -3.57343435e-01, 6.69872388e-02, -2.05376260e-02, 6.03192970e-02, -2.77649582e-04, 4.66996022e-02, 2.09306199e-02, 1.26473635e-01, -1.99992478e-01, 6.02058368e-03, -9.60124508e-02, -6.86570778e-02, -2.69254763e-02, -1.16648778e-01, 6.80445060e-02, 8.54630917e-02, 6.16764985e-02, -6.65853033e-03, -6.19218759e-02, -3.73522252e-01, -1.32694215e-01, 4.03534174e-02, 5.64134538e-01, 5.32403849e-02, -7.38940388e-02, -9.58362669e-02, -1.27810821e-01, 5.41193448e-02, 1.32223934e-01, 4.17461758e-03, 7.13659152e-02, 3.29776555e-02, 3.19321036e-01, -3.77637222e-02, 2.86701359e-02, 9.89994928e-02, 5.24967327e-04, 6.47896230e-02, -9.30845514e-02, -8.97478983e-02, 3.75827640e-01, 7.06133619e-03, -1.70939237e-01, -2.22015311e-03, -8.06940794e-02, -1.80535465e-01, -1.17046207e-01, -5.05085103e-02], dtype=float32), scale: Array([-2.16153208e-02, 4.99568492e-01, 1.90679863e-01, 3.98646802e-01, 3.23043484e-03, 8.04649472e-01, 1.73618346e-01, 6.32202104e-02, 6.03341032e-03, 7.60933161e-01, 5.64841628e-01, 3.30200821e-01, -1.13521464e-01, 4.35642749e-01, 7.02156186e-01, 1.86426133e-01, 2.69546956e-01, 4.07335073e-01, 3.06657463e-01, 7.59951055e-01, 8.35915983e-01, 5.73645175e-01, 3.52393031e-01, 2.92183794e-02, 5.95538437e-01, -3.08835153e-02, 5.68952084e-01, 6.18548512e-01, 7.05874801e-01, -1.11141577e-01, 7.34429657e-01, 1.59030482e-01, 6.69286311e-01, 3.07584852e-01, 2.97907770e-01, 2.45770905e-02, 3.42297733e-01, 6.38300121e-01, 4.57696557e-01, 3.39488059e-01, 4.27823365e-01, 6.61583364e-01, 2.51028594e-02, 6.87740266e-01, 6.38814211e-01, 6.12671137e-01, 3.40550840e-01, 3.15654457e-01, 5.78372836e-01, 5.64573705e-01, 5.55405932e-05, 3.71645719e-01, 8.48703533e-02, 9.91551802e-02, 4.97666523e-02, 5.78866065e-01, -1.33863851e-01, 2.80352771e-01, 5.00911117e-01, 1.85352847e-01, 7.25611448e-01, 4.00964379e-01, 2.84439921e-01, 3.20623189e-01, -1.70927703e-01, 4.05740440e-01, 3.79946142e-01, 2.70140201e-01, 5.38314104e-01, 6.14893734e-01, 5.95924675e-01, 5.98044157e-01, 1.48603588e-01, 3.81722450e-01, 5.22187948e-01, 7.07299054e-01, 4.85696197e-01, 1.27313018e-01, 7.40394592e-02, 3.15272629e-01, 3.43401968e-01, 6.31168902e-01, 2.14417100e-01, -3.38677377e-01, 2.39464089e-01, 4.71792251e-01, 3.14170390e-01, 8.54388118e-01, 6.38605714e-01, 1.02670453e-01, 4.03134346e-01, 3.42762381e-01, 4.40316945e-01, 5.73590755e-01, 3.09652478e-01, 1.55366600e-01, 6.81986809e-01, -7.89772999e-03, 5.20319641e-01, 6.24004126e-01, 5.56326687e-01, 3.05760831e-01, 9.59146619e-01, 5.92343211e-01, 3.85424703e-01, 4.82657462e-01, 5.25808513e-01, 4.87956852e-01, 3.75198275e-01, -6.13166541e-02, 5.11129975e-01, 6.06624246e-01, 7.52410650e-01, 1.93235308e-01, -5.42348363e-02, 3.70484799e-01, 6.81148842e-03, 7.82989919e-01, 6.00687087e-01, 4.27850395e-01, 9.20885563e-01, 2.06756860e-01, 7.61731803e-01, 6.10257149e-01, 2.84399986e-01, 6.83825493e-01, 5.32033622e-01, 1.40386716e-01], dtype=float32), }, layernorm_before: { bias: Array([ 1.7358573e-02, -2.1321216e-01, -3.0296935e-02, -8.5394399e-04, -3.9229371e-02, 5.5817175e-03, -2.7252757e-03, -4.4029057e-02, -2.0724675e-03, -1.2441806e-01, -2.2284750e-02, -2.7261129e-02, -2.7341412e-03, 3.6942791e-03, 4.5538493e-03, 7.4563054e-03, -6.2132953e-03, 9.7167321e-02, -2.5345799e-02, -3.9951671e-02, -4.3273875e-03, -1.4203915e-03, 6.3152872e-02, 1.3384975e-03, -5.5942591e-02, 6.3980678e-03, -3.2894940e-03, 2.6343842e-03, 2.8435083e-02, 2.8260322e-02, 1.2178370e-02, -5.3195474e-03, -2.4072591e-03, 4.1922098e-03, -7.3268535e-03, 7.5856894e-03, 3.8502638e-03, -9.8595031e-02, 2.4951894e-02, -9.5122019e-03, -5.4523430e-04, 4.9118581e-03, 2.0277210e-01, -1.6341466e-02, 3.9525043e-02, -2.2266436e-02, -2.0717401e-03, -1.2440056e-01, 2.4503253e-03, 1.1633219e-02, 3.5871702e-04, 1.0992530e-03, 7.6659650e-02, -6.7069583e-02, 9.7138679e-04, -2.9395521e-02, -8.4895557e-03, -1.3592694e-02, 2.9208256e-02, 1.7238649e-02, 5.7587404e-02, 4.4692371e-02, 1.6647557e-02, -6.6133849e-03, -3.3748765e-03, 9.0554915e-03, 1.8321996e-03, 1.5999395e-02, -1.3479884e-02, 2.9679830e-03, -2.7784597e-02, 3.4300985e-03, 4.0611736e-02, -4.6936791e-02, -7.2169447e-01, 8.0935545e-03, -1.8306768e-02, -5.0263423e-02, -3.8130886e-03, 1.9901623e-03, 2.5550249e-01, 6.1443527e-03, -2.2893531e-02, -1.7245837e-02, 2.9439591e-02, -4.3969110e-02, -6.7670122e-03, -1.3685081e-02, -9.9776837e-04, 9.0234309e-02, -1.4077160e-02, 2.6768299e-03, -3.8174929e-03, -1.4334331e-02, 3.4874912e-02, 8.0644749e-02, 8.6499676e-03, -1.7143000e-02, 7.9911202e-03, 2.8301086e-02, -4.1625272e-02, -3.0587490e-03, -5.6571878e-02, 9.0596341e-03, -7.6397270e-02, -4.5915369e-02, -6.2590851e-03, 3.4895688e-02, 6.1382476e-02, -5.6433260e-02, 1.0131227e-02, 1.5290984e-03, -5.1695153e-02, 1.3336526e-01, 1.1029670e-02, -2.2941723e-02, 2.4411927e-03, -2.4650490e-02, -1.1010250e-02, -7.5843465e-03, -1.1662335e-02, 3.2463686e-03, -1.6854508e-03, 2.7790260e-02, 7.3739677e-03, -9.9853233e-02, 1.0033315e-02, -9.9916525e-02], dtype=float32), scale: Array([ 7.94857554e-03, 1.53508723e-01, 7.40691870e-02, 1.35285795e-01, 2.26536706e-01, 1.69742465e-01, 8.65431502e-04, 3.19571525e-01, 1.62680820e-03, 1.62808627e-01, 1.34147704e-01, 2.01822311e-01, 6.27691345e-03, 7.19762873e-03, 3.74700986e-02, 3.44181597e-01, 9.95386466e-02, 1.60152465e-01, 9.15851742e-02, 1.63604781e-01, 1.17830604e-01, -1.66135898e-03, 2.23148689e-01, 1.00941835e-02, 2.60033756e-01, 1.75621884e-03, 2.65612965e-03, 4.38557938e-04, 3.29755515e-01, 2.43666410e-01, 3.20717841e-01, 1.77067131e-01, 2.68771589e-01, 3.02107055e-02, 2.95764953e-02, 1.08823786e-02, -1.60506088e-02, 2.95251936e-01, -1.78114280e-01, 8.72005969e-02, -8.84137396e-03, 2.99656868e-01, 8.63162801e-02, 1.33525252e-01, 3.03551525e-01, 2.60979742e-01, -5.50631993e-03, 2.83051997e-01, 2.56603450e-01, 2.00551033e-01, 5.14968205e-03, 2.78209685e-04, 2.23627791e-01, 2.34901443e-01, -8.34921002e-03, 2.24900410e-01, -6.23680511e-03, -2.86224540e-02, 2.08953783e-01, 3.78563493e-01, 2.40491629e-01, 1.92855686e-01, 2.62306392e-01, 5.38155213e-02, 2.10460991e-01, 1.67684779e-02, 5.48215546e-02, 1.18601710e-01, 3.43158811e-01, 2.41343841e-01, 2.74241388e-01, -6.84597194e-02, 2.30040535e-01, 2.19160318e-01, 1.71067789e-01, 2.74585098e-01, 1.48560002e-01, 2.04053581e-01, -1.04316017e-02, 3.06529433e-01, 1.51589945e-01, -2.79646995e-03, 2.60486454e-01, 9.33720767e-02, 2.20337927e-01, 1.95606187e-01, -4.61113850e-05, 7.93705583e-02, 2.31670171e-01, 1.81787178e-01, 6.02688640e-02, 6.66163536e-03, 5.13362838e-03, 8.27364624e-02, 3.05262417e-01, 1.02641329e-01, 1.57344952e-01, 1.41675845e-01, 4.87802178e-03, 1.00217178e-01, 1.87011242e-01, -1.03036012e-03, 2.17317730e-01, 1.16062611e-01, 3.36587638e-01, 3.04190874e-01, 1.32016614e-01, 1.66113153e-01, 3.42909336e-01, 2.46614054e-01, 2.17617258e-01, 1.86562017e-01, 1.04256548e-01, 3.53437752e-01, 5.86070295e-04, 2.16139719e-01, 2.78897047e-01, 2.12332204e-01, 3.80921900e-01, 1.19425990e-01, 3.23329829e-02, -1.03264116e-02, 1.89511199e-02, 1.55454144e-01, -3.36376615e-02, 2.82643199e-01, 3.45727682e-01, 2.41249815e-01], dtype=float32), }, moe: { shared_experts: { intermediate: { bias: Array([-2.87919283e-01, -1.59886956e-01, -3.92205477e-01, -4.32191879e-01, -3.32138002e-01, -5.40607274e-01, -3.54391366e-01, -1.31854281e-01, -5.39756715e-01, -4.69244629e-01, -3.44658494e-01, -4.88029271e-01, -3.98942500e-01, -4.15098637e-01, -3.66716921e-01, -5.07466793e-01, -2.69494671e-02, -3.29423159e-01, -3.49476814e-01, -1.71700358e-01, -3.80648106e-01, -4.23726380e-01, -4.98886943e-01, -4.94191587e-01, -2.60305732e-01, -3.48132432e-01, -4.27568406e-01, -3.35825920e-01, -3.69937718e-01, -3.21786374e-01, -2.29871064e-01, -2.88744688e-01, -3.85481507e-01, -4.31343943e-01, -1.41554534e-01, -1.79118499e-01, -4.28618819e-01, -2.48103037e-01, -9.70930979e-02, -4.53777105e-01, -1.97259694e-01, -3.89665425e-01, -4.33900356e-01, -2.01605186e-01, -3.11053962e-01, -3.33365947e-01, -4.72573489e-01, -4.81180519e-01, -4.59191829e-01, -3.63978118e-01, -5.11594951e-01, -3.21153313e-01, -5.05568504e-01, -4.40596819e-01, -2.05569610e-01, -2.04613134e-01, -2.30359152e-01, -4.58120763e-01, -1.35851100e-01, -2.78527498e-01, -5.32928765e-01, -4.59086925e-01, -1.91062093e-01, -2.50361383e-01, -3.23409230e-01, -5.09010673e-01, -2.47201502e-01, -3.10001701e-01, -3.00568342e-01, -3.73322159e-01, -5.35207331e-01, -5.31656861e-01, -3.18275809e-01, -4.40275580e-01, -3.43852162e-01, -4.12837327e-01, -3.85347039e-01, -4.52745438e-01, -2.02980176e-01, -4.54181701e-01, -5.00234187e-01, -3.34358454e-01, -3.39169592e-01, -2.89554566e-01, -3.29827011e-01, -5.80846727e-01, -3.28373313e-01, -4.26805854e-01, -3.15485120e-01, -3.92600238e-01, -3.12504858e-01, -2.19271436e-01, -1.64767921e-01, -5.02310932e-01, -2.58719236e-01, -4.32749659e-01, -9.22016874e-02, -4.70086277e-01, -3.97373617e-01, -4.89154935e-01, -4.44214731e-01, -2.03317106e-01, -4.75122511e-01, -2.69819319e-01, -2.67087281e-01, -4.90763247e-01, -4.22214806e-01, -2.64238060e-01, -2.23214269e-01, -3.43690515e-01, -2.78915852e-01, -4.24799353e-01, -5.69192111e-01, -2.07584277e-01, -1.64182380e-01, -1.91950992e-01, -5.61932206e-01, -5.16008019e-01, -3.33474815e-01, -4.25828815e-01, -1.61096185e-01, -3.98274213e-01, -5.68151116e-01, -2.51645744e-01, -2.21852303e-01, -4.22558784e-01, -3.82831007e-01, -3.19712460e-01, -1.66578472e-01, -2.49394879e-01, -4.19118315e-01, -2.93457031e-01, -2.79008389e-01, -5.07268071e-01, -1.48156717e-01, -4.60653484e-01, -2.05382407e-01, -4.70562130e-01, -2.83998400e-01, -4.29632753e-01, -2.36523062e-01, -3.75578731e-01, -3.47366124e-01, -2.43162408e-01, -4.38231677e-01, -5.36709070e-01, -1.51090562e-01, -4.71059680e-01, -4.89868373e-01, -3.91524017e-01, -3.24053317e-01, -3.04224849e-01, -4.36558515e-01, -2.35701188e-01, -4.07339513e-01, -2.95855463e-01, -2.55902320e-01, -4.83437032e-01, -3.88129205e-01, -1.03971243e-01, -2.60267794e-01, -3.92015398e-01, -4.49646682e-01, -2.48017311e-01, -2.56850272e-01, -1.90488473e-01, -4.68013018e-01, -4.13734376e-01, -5.12630165e-01, -4.26820993e-01, -3.74209106e-01, -3.59761298e-01, -4.02847528e-01, -2.97854811e-01, -3.58279765e-01, -3.24418694e-01, -3.66349548e-01, -4.33402061e-01, -4.91001993e-01, -5.03202379e-01, -5.00787556e-01, -3.92910451e-01, -2.84466565e-01, -3.34319711e-01, -4.25348043e-01, -2.26335734e-01, -2.37801537e-01, -4.87084776e-01, -3.79190952e-01, -3.08594137e-01, -3.53578776e-01, -2.29536459e-01, -4.85832751e-01, -3.31020027e-01, -3.49860758e-01, -4.98145282e-01, -2.17634872e-01, -3.08456481e-01, -4.34388995e-01, -2.14153975e-01, -2.20951349e-01, -4.75258082e-01, -2.87501752e-01, -9.52811018e-02, -5.12147367e-01, -2.70634055e-01, -2.17771903e-01, -3.38476300e-01, -3.06095690e-01, -3.33468020e-01, -1.24255188e-01, -5.16220808e-01, -2.23946184e-01, -5.59429586e-01, -2.83430904e-01, -4.03296888e-01, -3.68934721e-01, -3.25362712e-01, -7.39004537e-02, -3.09295535e-01, -4.49250668e-01, -5.65693378e-01, -2.64678180e-01, -3.42923045e-01, -3.37939441e-01, -3.71786445e-01, -4.06780869e-01, -5.13805926e-01, -1.10545248e-01, -4.07462388e-01, -4.62072462e-01, -3.73685181e-01, -4.82776046e-01, -1.83113769e-01, -1.37951612e-01, -4.32110876e-01, -4.39051688e-01, -4.20039445e-01, -4.49939400e-01, -5.46136618e-01, -4.43518162e-01, -3.36255163e-01, -2.60323614e-01, -1.48123205e-01, -3.08105409e-01, -5.92286289e-01, -3.61411870e-01, -5.49996793e-01, -4.72203493e-01, -1.12566963e-01, -3.46157908e-01, -2.21687108e-01, -1.49518579e-01, -4.22669083e-01, -5.09057462e-01, -3.78608346e-01, -2.69701719e-01, -2.57610738e-01, -3.43267351e-01, -3.99610460e-01, -5.19208014e-01, -3.55502129e-01, -3.64218682e-01, -4.58064228e-01, -1.66265354e-01, -4.56624329e-01, -4.21405643e-01, -3.38668913e-01, -4.08919096e-01, -2.49855325e-01, -2.68893301e-01, 5.37731161e-04, -4.11918223e-01, -2.53745079e-01, -2.37736523e-01, -3.77250522e-01, -3.40394109e-01, -4.99163866e-01, -3.26041222e-01, -3.24631035e-02, -4.25087065e-01, -3.99343103e-01, -4.45631206e-01, -2.19975799e-01, -4.91138816e-01, -3.16317588e-01, -1.97857752e-01, -4.38183516e-01, -2.38349304e-01, -3.12527388e-01, -4.46206242e-01, -1.97425947e-01, -3.54068547e-01, -5.62409818e-01, -4.79922563e-01, -4.07248676e-01, -5.08961380e-01, -3.35608751e-01, -4.69772518e-01, -4.64478105e-01, -2.38058373e-01, -3.87834966e-01, -4.91571218e-01, -1.16468966e-01, -1.88293502e-01, -2.19764918e-01, -2.41493329e-01, -3.08782160e-01, -3.80097032e-01, -2.52258271e-01, -3.71472836e-01, -3.24205458e-01, -4.05035138e-01, -5.40303230e-01, -1.73389211e-01, -3.20962131e-01, -2.16032237e-01, -1.91938236e-01, -1.65382430e-01, -4.07554179e-01, -3.40726435e-01, -2.95837790e-01, -9.07381624e-02, -1.67001665e-01, -3.45026761e-01, -4.43528742e-01, -4.61021096e-01, -2.25696385e-01, -2.09777251e-01, -5.53110957e-01, -3.23776424e-01, -3.86852056e-01, -5.13776362e-01, -1.70138955e-01, -4.78795141e-01, -2.65858293e-01, -1.22696631e-01, -5.30045271e-01, -3.90655428e-01, -2.64281482e-01, -4.28456664e-01, -1.40250340e-01, -2.27036089e-01, -3.66672546e-01, -4.42133546e-01, -4.25859839e-01, -2.67484695e-01, -4.62734908e-01, -3.37051451e-01, -4.50581521e-01, -7.01922104e-02, -3.47592622e-01, -3.91412824e-01, -4.24011290e-01, -3.32346976e-01, -3.72510850e-01, -3.09896857e-01, -1.88529536e-01, -3.71884912e-01, -2.54814714e-01, -3.63058448e-01, -4.75363225e-01, -3.06162030e-01, -4.65719730e-01, -1.81163147e-01, -4.85125124e-01, -3.95848036e-01, -4.33863193e-01, -3.04795027e-01, -4.90597248e-01, -2.44484976e-01, -3.44704390e-01, -3.45088750e-01, -3.29458416e-01, -2.94500917e-01, -2.62573928e-01, -1.69639900e-01, -4.71560776e-01, -2.42599428e-01, -3.69580209e-01, -3.42656642e-01, -3.11950177e-01, -1.34305000e-01, -1.57368377e-01, -3.91964465e-01, -3.12808394e-01, -2.37698436e-01, -3.69058728e-01, -1.38768077e-01, -2.69422363e-02, -4.55560267e-01, -3.79797220e-01, -4.36610907e-01, -2.92106032e-01, -3.98713797e-01, -4.69899833e-01, -1.97569534e-01, -3.96044046e-01, -1.28238350e-01, -3.10444951e-01, -4.51549977e-01, -2.95563221e-01, -4.12408620e-01, -4.34255183e-01, -2.34395146e-01, -4.32429850e-01, -2.51709074e-01, -3.16113442e-01, -2.24395722e-01, -4.12795454e-01, -3.96394074e-01, -3.76580715e-01, -2.59395570e-01, -5.12954056e-01, -3.32104653e-01, -4.36192185e-01, -5.13575554e-01, -1.39632359e-01, -4.51251060e-01, -4.95827556e-01, -2.47558579e-01, -3.28758836e-01, -4.00682449e-01, -1.53779313e-01, -2.39558816e-01, -3.46421301e-01, -2.07403153e-01, -4.59828466e-01, -4.13055509e-01, -5.19209445e-01, -3.49062324e-01, -3.61741424e-01, -5.46588659e-01, -3.25105667e-01, -2.64346927e-01, -3.62703383e-01, -3.54973465e-01, -3.57288867e-01, -3.40226203e-01, -1.87480614e-01, -3.84213597e-01, -3.67905647e-01, -2.21579894e-01, -4.32286561e-01, -3.49667877e-01, -2.67124563e-01, -2.31143355e-01, -2.09680378e-01, -4.44781452e-01, -1.10374503e-01, -1.90934509e-01, -4.75128025e-01, -4.90421832e-01, -3.84051919e-01, -3.26278806e-01, -3.82725775e-01, -2.96725929e-01, -4.35144246e-01, -3.04508001e-01, -4.23557281e-01, -3.32650095e-01, -2.34854251e-01, -2.51995057e-01, -3.14376324e-01, -4.71209139e-01, -5.88116467e-01, -3.59949678e-01, -3.27794611e-01, -5.37792504e-01, -2.25097418e-01, -5.30355632e-01, -5.33735216e-01, -2.61757433e-01, -2.64203936e-01, -3.20991725e-01, -3.19024026e-01, -4.41864610e-01, -1.51468694e-01, -6.96782321e-02, -4.49556112e-01, -3.41252834e-01, -6.03692472e-01, -4.20485705e-01, -4.03241307e-01, -4.20258135e-01, -4.89987731e-01, -2.22824812e-01, -1.52097628e-01, -3.01070571e-01, -3.31862330e-01, -2.91463912e-01, -2.47648209e-01, -4.18553919e-01, 8.17964971e-02, -4.42326665e-01, -4.14036870e-01, -4.23043162e-01, -4.24916655e-01, -3.22944313e-01, -3.46353531e-01, -3.03149909e-01, -4.78416532e-01, -3.12508672e-01, -3.28645647e-01, -3.10102880e-01, 6.61208481e-02, -3.10083032e-01, -4.34432387e-01, -2.68977493e-01, -3.87750000e-01, -3.06754440e-01, -2.68470585e-01], dtype=float32), kernel: Array([[-8.63173604e-02, 5.54475226e-02, -7.44883567e-02, ..., -1.60854563e-01, -1.92300782e-01, 1.49715483e-01], [-2.55667627e-01, -5.76032937e-01, -2.72577375e-01, ..., -1.66639566e-01, -9.29499939e-02, -8.04038703e-01], [ 1.05862938e-01, -7.12434500e-02, -3.51943791e-01, ..., -4.56090540e-01, 6.42790049e-02, 2.83889264e-01], ..., [ 3.32350917e-02, -2.16388881e-01, -4.15698439e-01, ..., -2.22950518e-01, 1.03157982e-01, 4.05906420e-03], [ 1.34180099e-01, 2.93415725e-01, -3.85918528e-01, ..., 5.76932847e-01, -1.18126400e-01, 1.22173645e-01], [-3.73824616e-04, 1.94809213e-02, 2.37895355e-01, ..., -2.55678475e-01, -4.13769521e-02, 8.79469365e-02]], dtype=float32), }, output: { bias: Array([-0.14310208, 0.30402932, -0.04476445, 0.05614336, -0.00636329, -0.10514647, 0.06905492, 0.06197605, 0.02608861, 0.15183868, -0.0424929 , -0.01462859, -0.1350344 , 0.02212298, -0.05124652, 0.06123491, 0.03247906, -0.09240599, -0.01013029, -0.03223318, -0.0887438 , -0.06346248, 0.02817548, -0.05015382, -0.05139951, 0.01631817, -0.04831094, -0.02007524, 0.04097073, 0.1089523 , 0.14585266, -0.05194474, 0.04915197, -0.09083902, 0.01939188, -0.06052509, 0.05472679, 0.04450262, 0.03008708, -0.1406127 , -0.0286117 , 0.04946226, -0.8201933 , 0.05301183, -0.06421784, 0.03291068, 0.08303672, 0.10530804, -0.05531342, -0.09611814, -0.02686962, 0.10026816, -0.01932072, 0.17867358, -0.03787487, 0.01405966, -0.20247376, -0.07591344, 0.15851738, 0.16269532, 0.17163317, -0.16734251, -0.05690756, 0.02852942, 0.06089685, -0.04197973, -0.06455492, 0.07540473, -0.06004319, 0.08624789, -0.1040978 , 0.00247931, -0.02840352, -0.04520692, 0.8720378 , -0.06833719, -0.07094937, 0.05479927, -0.12080298, -0.0121475 , -0.5797822 , -0.02564085, 0.03635592, 0.03610236, -0.00385189, 0.1565094 , 0.14343542, 0.05017952, 0.00714095, 0.09029737, 0.0294838 , 0.01819415, -0.04793838, -0.1418107 , -0.00120118, -0.06547695, -0.03586074, -0.01915462, -0.03162472, 0.25057518, 0.21378383, -0.03219284, -0.03374225, -0.07630365, 0.13850403, -0.09364668, 0.02092841, -0.02086668, -0.11372983, -0.06087056, -0.14589557, 0.03926003, 0.0446431 , 0.00222743, 0.03577771, 0.14117669, 0.01693195, -0.06432896, 0.02443499, 0.04788267, -0.13781525, 0.06813591, 0.10359713, -0.10075218, 0.02409258, 0.05323493, 0.00397177, 0.03774352], dtype=float32), kernel: Array([[ 0.16250458, -0.15818997, 0.3543968 , ..., -0.06277009, -0.458476 , 0.21236774], [ 0.17157121, -0.33716652, 0.17193943, ..., -0.07994235, 0.03173731, -0.09947567], [ 0.20364675, 0.04469377, -0.3887421 , ..., -0.24748199, 0.29170632, -0.02271716], ..., [ 0.29266545, -0.02244417, 0.41813877, ..., -0.37108174, -0.26156428, -0.10771853], [-0.07057951, -0.23378706, 0.05101302, ..., 0.08562624, 0.45771068, 0.08117881], [-0.21127233, -0.6031122 , -0.4248507 , ..., 0.16734137, 0.09727449, -0.12857328]], dtype=float32), }, }, }, }, 3: { attention: { attention: { key: { bias: Array([-0.22031489, 0.02730584, 0.23429762, -0.3125938 , 0.22592317, -0.6771136 , -0.21247995, -0.341136 , 0.07302368, 0.44729188, 0.22906882, -0.524169 , -0.04314182, -0.22182061, -0.67601067, -0.01997758, -0.00890862, 0.31220517, 0.3243842 , -0.08772013, 0.18925034, -0.43556792, -0.30325386, 0.13643748, 0.27908477, 0.26076847, 0.19125256, 0.6231867 , 0.34878573, 0.21458477, 0.12557186, 0.25881395, 0.11842591, 0.14228185, -0.14626004, 0.02273282, 0.04946363, 0.3702873 , 0.4587399 , 0.46696767, 0.2649066 , -0.02744056, 0.21641424, 0.4786166 , -0.39902607, -0.25150967, 0.20756108, -0.9501807 , 0.2439707 , 0.07372095, 0.1429905 , 0.5078466 , -0.47601166, 0.01509763, -0.63272023, 0.5627352 , 0.5814683 , 0.3356227 , -0.07830408, -0.29685998, 0.09222841, -0.49363494, -0.09527437, -0.01283222, 0.10691621, 0.44450942, 0.2378995 , 0.14754085, 0.07311877, -0.4130656 , 0.12185058, 0.1367209 , 0.18846467, -0.11077244, -0.10472129, 0.13243866, -0.6801274 , 0.18685853, -0.40455377, -0.17242797, -0.3273368 , 0.21752419, 0.5587714 , 0.33085388, -0.13658075, 0.11016981, -0.14520068, -0.0551539 , -0.0988646 , 0.19754803, 0.28224358, 0.27123928, 0.01996558, 0.25261068, 0.3291632 , 0.73044914, -0.7135282 , -0.2712479 , 0.17239682, 0.4813283 , 0.350012 , 0.76145405, 0.01346729, 0.28428316, 0.17260526, -0.40829295, 0.66129416, -0.35163417, -0.15398747, -0.87850165, 0.16401769, -0.36521718, 0.08581091, 0.12348412, -0.731215 , 0.24665482, 0.77237046, 0.39620268, -0.4511514 , 0.34254473, -0.669006 , -0.50109476, -0.18661116, 0.2477596 , 0.65949994, 0.01988118, -0.04483053, -0.7621275 ], dtype=float32), kernel: Array([[ 0.14074115, 0.11149215, 0.02833093, ..., 0.24423462, -0.3480132 , -0.06901791], [ 0.13614887, -0.2725882 , -0.31767422, ..., -0.30451745, -0.0344476 , 0.2648898 ], [ 0.19196813, 0.24347357, 0.09369045, ..., 0.3346163 , -0.10084 , -0.19047955], ..., [ 0.02364591, 0.3210728 , 0.15941176, ..., -0.5476011 , 0.30842453, 0.16627334], [-0.3327589 , 0.2611727 , 0.04791668, ..., 0.5718048 , -0.00553066, -0.12860055], [ 0.13814813, -0.2723826 , -0.10484312, ..., 0.10856424, -0.46137026, 0.3135762 ]], dtype=float32), }, query: { bias: Array([ 0.01636963, 0.00303602, -0.2237812 , 0.17135 , -0.5504063 , 0.71538967, 0.34468257, 0.6879721 , -0.32814807, 0.12084308, -0.23467594, 0.24040897, -0.03144267, 0.41499555, 0.18407044, -0.19885245, 0.29428306, -0.17422631, -0.72670996, 0.19728792, -0.12558015, -0.04110402, 0.7530146 , -0.32672936, -0.34328216, 0.39724907, -0.19968817, -0.0735909 , -0.11611477, 0.2887974 , 0.27589712, -0.19556114, 0.00310108, 0.05990869, -0.06629719, 0.11465285, 0.30233756, -0.4840145 , 0.36792257, 0.01198539, 0.17746153, 0.00102492, -0.73527193, -0.04222427, 0.1278375 , -0.04002868, -0.04392892, 0.07372226, 0.00746391, 0.01089494, -0.06465401, -0.01894091, -0.04520243, -0.05135101, 0.10821903, -0.36141765, 0.01786461, -0.00758973, 0.20370121, 0.07571532, -0.04988701, -0.50066465, -0.29137757, 0.0468887 , -0.0369802 , -0.28495568, -0.15369293, -0.04906723, -0.05060489, 0.23548573, 0.02296117, 0.29195184, -0.4791245 , -0.32274112, 0.00163329, 0.6024667 , -0.02721317, 0.4371868 , -0.45318347, 0.15298416, 0.53022355, 0.23577152, -0.11320106, -0.05042947, 0.00792542, 0.34630367, 0.45112005, -0.17176089, -0.30412972, -0.00977058, 0.3797044 , -0.55427474, -0.14912651, -0.31753495, -0.14334434, -0.12312431, 0.29336822, 0.08129798, -0.09826545, -0.37803188, -0.03996738, -0.565557 , -0.15023306, -0.17836387, -0.00129342, 0.31617662, -0.40511948, -0.0246349 , 0.8049514 , 0.50346047, -0.08232877, 0.7958381 , 0.11525708, -0.87054974, 0.02044921, 0.10254359, -0.41025582, -0.11836894, 0.3934775 , -0.2755682 , 0.28480217, 0.28756127, -0.06242 , -0.36147273, -0.31789255, -0.1169754 , 0.13862565, 0.5221684 ], dtype=float32), kernel: Array([[-0.03366861, 0.1951855 , 0.13829306, ..., 0.20226802, 0.14245163, 0.0851718 ], [-0.12275442, 0.09568106, 0.1026165 , ..., -0.11756799, -0.03399425, 0.1083973 ], [-0.11897923, 0.44871688, 0.22821409, ..., -0.19063285, 0.14112021, 0.05127604], ..., [ 0.42162004, -0.30832654, -0.31275368, ..., -0.48211238, 0.221526 , 0.17857978], [ 0.23314169, 0.38464716, 0.37320703, ..., 0.7464507 , 0.65437895, 0.09728524], [-0.02876819, -0.11770961, -0.18619719, ..., -0.4305184 , -0.24435337, -0.0622173 ]], dtype=float32), }, value: { bias: Array([ 3.80855761e-02, 1.19626271e-02, 9.54822153e-02, 3.39327864e-02, 6.47817999e-02, 1.12312162e-04, 2.36410853e-02, 1.12216197e-01, 1.71483278e-01, -4.48498651e-02, -6.95201615e-03, -5.79331703e-02, 1.67439714e-01, -1.31781459e-01, 1.07828841e-01, -1.07748993e-01, -2.16874084e-03, -2.65070498e-02, -1.59811676e-02, 7.09029958e-02, 5.01474105e-02, -4.89775911e-02, 2.54220992e-01, -4.80323769e-02, -1.79449767e-02, -5.27789593e-02, -4.24154028e-02, -1.70985550e-01, -1.36494562e-01, 1.54837176e-01, -1.15875848e-01, -1.71242371e-01, 5.86349219e-02, 1.91222817e-01, -2.35057212e-02, -8.43578875e-02, -4.65269759e-02, -8.38292018e-03, -4.65860590e-02, -5.02779474e-03, 4.50352840e-02, -2.31239274e-02, 5.93480729e-02, 5.71514890e-02, 5.99062406e-02, 2.83261593e-02, -4.82074209e-02, -4.80451956e-02, -6.26677051e-02, 5.32909296e-02, 6.67422032e-03, 4.72835749e-02, 4.04058099e-02, -4.40718755e-02, -4.50864471e-02, 1.62461493e-02, -5.04599810e-02, -3.60567644e-02, 4.91038598e-02, -7.88058713e-03, 1.09715499e-01, 5.14943898e-02, 3.80876176e-02, 5.21656685e-02, 1.56386979e-02, 6.83846846e-02, 7.60920197e-02, -8.33767503e-02, -1.77296321e-03, -9.03388578e-03, -5.15193716e-02, 2.69258358e-02, 4.77294379e-04, -2.96406243e-02, -8.59358441e-03, -2.96034720e-02, -2.67447811e-02, -6.73482614e-03, 1.27622047e-02, -2.17594709e-02, 8.59617069e-02, 1.03174895e-01, 2.63699442e-02, -1.10600488e-02, -7.31578097e-02, -3.59972566e-02, 4.40209918e-02, -4.44998406e-02, -1.85169447e-02, -4.11919132e-02, -3.58521268e-02, 1.47900637e-03, 4.30759192e-02, 2.79228836e-02, 7.82085881e-02, 3.90527584e-02, 3.11766323e-02, 2.85879765e-02, -8.20098892e-02, -7.59413168e-02, -6.41580895e-02, 2.49105208e-02, -1.47475805e-02, -9.27800685e-02, -1.73971094e-02, 8.51288959e-02, -2.54925918e-02, 4.36546579e-02, -9.24658105e-02, -1.01948939e-02, -4.81284559e-02, -3.62936854e-02, 2.90373899e-02, -4.27892022e-02, 3.08775119e-02, 5.98094938e-03, 2.10134909e-02, 4.55508381e-02, 3.65342051e-02, 1.74732562e-02, 1.06893666e-02, -1.42213493e-03, -2.61030179e-02, 2.22432707e-02, 1.47231504e-01, 6.33382201e-02, 2.16640271e-02, 4.13991958e-02], dtype=float32), kernel: Array([[ 0.00939619, 0.02482712, 0.01164271, ..., 0.0164959 , 0.00769011, -0.03742562], [ 0.18655218, -0.36816835, -0.08651444, ..., 0.2512268 , 0.02853749, 0.00129432], [ 0.09724977, -0.17311321, -0.15452595, ..., -0.4560767 , -0.48346895, -0.4558498 ], ..., [-0.06453069, 0.4239059 , 0.45162615, ..., -0.27700382, 0.13836603, -0.33547392], [-0.02759907, -0.9605103 , -0.82787424, ..., -0.51210946, 0.33198425, 1.0528204 ], [-0.12098552, -0.10120461, -0.10430597, ..., 0.78033483, -0.08541391, 0.05855659]], dtype=float32), }, }, output: { dense: { bias: Array([-1.72631651e-01, 2.81819075e-01, -1.31835248e-02, -6.50164206e-03, -9.19895153e-03, -6.00125082e-02, 4.56261076e-02, -1.17319543e-03, 3.18587162e-02, 2.30925724e-01, -2.78387107e-02, -7.64354365e-03, -7.98858032e-02, 2.95766089e-02, -4.09963764e-02, 1.03628486e-01, 5.56401908e-02, -8.70198682e-02, -1.29737146e-02, 1.11193627e-01, -5.96015491e-02, -5.83102107e-02, -2.50512566e-02, -3.02468054e-02, -3.29878554e-02, 4.26750369e-02, -3.15967947e-02, 1.79770100e-03, 6.93126023e-02, 8.52573961e-02, 4.88307029e-02, -6.38186038e-02, 1.12157442e-01, -9.84508917e-02, 5.63154779e-02, -7.03682825e-02, 2.64801327e-02, -1.89797301e-02, 6.20345166e-03, -8.26654360e-02, 2.39220913e-02, -9.58621129e-03, -8.39129508e-01, 4.56922092e-02, -1.53480023e-01, 6.83549270e-02, 7.20679983e-02, 2.08752692e-01, 4.47404385e-02, -5.50162159e-02, -5.08189276e-02, 5.22453226e-02, -2.34786086e-02, 1.55734494e-01, -4.19281684e-02, -1.19545199e-02, -8.40202495e-02, -8.96746367e-02, 6.41859993e-02, 7.71264508e-02, 8.69383588e-02, -1.31521776e-01, -2.94522438e-02, 4.01933566e-02, -1.60081685e-03, -3.63952629e-02, -5.23505732e-02, 8.07748511e-02, 3.52987312e-02, 8.45032781e-02, -1.18315943e-01, 3.48625183e-02, 5.90237789e-02, -7.91934412e-03, 8.07135582e-01, -6.57365983e-03, -6.89911172e-02, 1.08262442e-01, -1.27490476e-01, 1.97968110e-02, -6.29805326e-01, 5.13264053e-02, 3.17039192e-02, 1.63930492e-03, 2.53316425e-02, 1.45042747e-01, 1.39836982e-01, 1.34169623e-01, 4.38121334e-02, -1.00197628e-01, 3.51289548e-02, 3.51637453e-02, -1.21273242e-01, -1.04602434e-01, -9.42274835e-03, -2.06037387e-02, -4.44292687e-02, -2.39612963e-02, -1.44756138e-02, 9.55058485e-02, 3.27826925e-02, 5.12430328e-04, 1.59999877e-02, -6.98894709e-02, 1.01640105e-01, -5.48832715e-02, 6.11420302e-03, -7.91232064e-02, -1.24512471e-01, -3.10322326e-02, -1.39527261e-01, 3.97435911e-02, 1.18619666e-01, -1.12082511e-01, 1.25646042e-02, 1.72376752e-01, 4.17521298e-02, -2.35856101e-02, -3.09479032e-02, 6.41909391e-02, -2.64365948e-03, 2.64008511e-02, 1.21359736e-01, -4.78715599e-02, 4.74941581e-02, 5.94909228e-02, -6.63625225e-02, -2.96208411e-02], dtype=float32), kernel: Array([[ 0.30733457, 0.11394037, -0.11875194, ..., -0.02600583, 0.26162902, -0.24450323], [ 0.32579228, -0.01019196, 0.08372336, ..., -0.23464818, 0.77629703, -0.14200607], [ 0.40005246, -0.09073471, 0.01217228, ..., 0.12469898, 0.4625953 , -0.2634586 ], ..., [-0.14610979, 0.26803997, -0.26479524, ..., -0.581134 , 0.39575958, -1.1754091 ], [-0.10739683, -0.4411798 , 0.68852794, ..., -0.01546267, 0.06261325, -0.04542963], [ 0.20287202, 0.13014646, 1.1704878 , ..., -0.26887265, -0.26802245, 0.629474 ]], dtype=float32), }, }, }, layernorm_after: { bias: Array([ 1.71948429e-02, -1.09049575e-02, 6.19933335e-03, -3.97290401e-02, -1.75763867e-04, 5.07760718e-02, 1.12305544e-02, -2.87154932e-02, -2.46254611e-03, -1.75729260e-01, 9.95321851e-03, -1.72805071e-01, -1.28897009e-02, 2.74409372e-02, -1.33280344e-02, 1.57673303e-02, -2.70904624e-03, 1.06892176e-03, 2.89133284e-03, 1.36673786e-02, 4.77137081e-02, -9.59468889e-06, 6.27165893e-03, -6.68704743e-03, 1.20838732e-02, 1.39770750e-02, -3.87260579e-02, -5.88312745e-03, 4.62619169e-03, 6.23012260e-02, 8.07429329e-02, 7.94423558e-03, -2.50669499e-03, 1.25099311e-03, -6.49371138e-03, -9.59886145e-03, 3.83810289e-02, 7.55729433e-03, 4.41730954e-03, 1.75167769e-02, 3.34830210e-02, 9.28634033e-03, 1.21751567e-02, -1.52964927e-02, -2.70186807e-04, 7.17550702e-03, 8.44905898e-02, -9.36958939e-02, -1.57131581e-06, 4.74082539e-03, -8.86013955e-02, 7.37810042e-03, -4.76047629e-03, 6.89025401e-05, -9.31238383e-03, 6.64415536e-03, 7.76583795e-03, 2.29338557e-02, -1.50072891e-02, -3.97083163e-03, 5.64871766e-02, 1.86805464e-02, 5.85068855e-03, 3.33538577e-02, 3.15117603e-03, 3.10075725e-03, -3.19483913e-02, 3.20728831e-02, 2.38816179e-02, 1.34205222e-02, -1.76607758e-01, -5.26308222e-03, 2.82549043e-03, -1.20671205e-01, -1.07251152e-01, 3.50134517e-03, 2.33329542e-04, -1.22928575e-01, -5.50287850e-02, -2.40997149e-04, 4.36784141e-03, -3.80961671e-02, -1.24475360e-01, -1.74865853e-02, 9.07492265e-02, -1.61794608e-03, 4.57073562e-03, -6.20833039e-03, -5.41980378e-03, 2.80724547e-04, -4.64204960e-02, -4.63194735e-02, 1.02222068e-02, -5.72193079e-02, -3.21766338e-03, 6.88658096e-03, -4.31186426e-03, -3.94533984e-02, -1.81648252e-03, 5.71832731e-02, -6.70271181e-03, 5.93083352e-03, -8.59189108e-02, -5.01573570e-02, -3.59455287e-03, -8.11670348e-02, -3.07016950e-02, 2.01600092e-03, 4.20039892e-02, -4.99186059e-03, -1.39956048e-03, -5.72945038e-03, 1.23080192e-03, 1.21427193e-01, 1.27224373e-02, 5.32536209e-02, -1.02717979e-02, -3.96808702e-03, 3.51909577e-04, 1.07945967e-02, -4.73669693e-02, -5.23439702e-03, -3.12553300e-03, 6.25749454e-02, -9.56553966e-03, -9.94754490e-03, 2.23868527e-02, -2.48279348e-02], dtype=float32), scale: Array([ 4.24516723e-02, 1.21176187e-02, -4.84997500e-03, 5.57146311e-01, 2.28432156e-02, 6.31359816e-01, -4.46775509e-03, 6.18052892e-02, -4.33097221e-03, 2.61805564e-01, -1.88892316e-02, 6.95380867e-01, -6.15348248e-03, 5.73870949e-02, 1.55512363e-01, -5.18870428e-02, 1.60957292e-01, -1.62363220e-02, 4.54635024e-01, -1.21937275e-01, 4.06006753e-01, 4.14683282e-01, 3.15475985e-02, 4.67635952e-02, 1.88465670e-01, -1.32318744e-02, 4.83835369e-01, 8.42766091e-02, -7.77654797e-02, 5.02031922e-01, 4.80292380e-01, 3.83823998e-02, -4.09559421e-02, 1.13746740e-01, -1.26264850e-02, 7.38768047e-03, 3.53148162e-01, -1.56045742e-02, 2.96747178e-01, 1.84800133e-01, 2.40674987e-01, -5.23572266e-02, 1.17433036e-03, 6.38127849e-02, 2.33978732e-03, -3.27690654e-02, 4.07037109e-01, 1.53259337e-01, 3.08140907e-02, -5.92192961e-03, 3.31769854e-01, 3.03894043e-01, -1.66736450e-02, -4.60295379e-02, 3.50147933e-01, 4.64291312e-02, 7.68471509e-03, 9.69270691e-02, 1.85923763e-02, -9.66442749e-03, 4.18877095e-01, 1.00185797e-01, 1.32984370e-01, 4.85280842e-01, -7.05129327e-03, 4.97093098e-03, 3.28959554e-01, 9.48822647e-02, 2.17689544e-01, 2.05316558e-01, 5.99317133e-01, 3.90748940e-02, 5.26148966e-03, 3.09464663e-01, 3.66784409e-02, 4.52369526e-02, -4.48510870e-02, 2.84545600e-01, 3.67504448e-01, 4.94225929e-03, -5.27279684e-03, 8.71935904e-01, 4.51588362e-01, 2.58467104e-02, 3.65057826e-01, 6.95686322e-03, -1.69470385e-02, 1.21857105e-02, -1.65910535e-02, -8.54126003e-04, 5.98546147e-01, 3.60920489e-01, 2.29799710e-02, 3.24622720e-01, -8.44317749e-02, -2.27606995e-03, 3.23808223e-01, 2.70338178e-01, 5.69886751e-02, 1.97269484e-01, 3.14833894e-02, -3.49436514e-02, 3.14999282e-01, 5.10905981e-01, 2.21107565e-02, 3.05217803e-01, 2.38849521e-01, -1.36028053e-02, 4.52475280e-01, 2.09674463e-02, -2.62735710e-02, -6.97454140e-02, -8.80253036e-03, 4.10522908e-01, -2.87996326e-03, 2.52809376e-01, -1.01799294e-02, 1.55074904e-02, 4.01195139e-02, 1.15418859e-01, 1.29438698e-01, -9.92912054e-03, 3.01613122e-01, 9.44039226e-02, -3.23632099e-02, 1.09247090e-02, 5.09600282e-01, 6.76498190e-02], dtype=float32), }, layernorm_before: { bias: Array([ 2.0053135e-02, -2.0385702e-01, -5.2851941e-02, -1.5826782e-02, -3.9745602e-03, -5.1497813e-02, 4.4322563e-03, -2.0497143e-02, 3.5912704e-02, -1.5449695e-01, -6.0098185e-03, 4.4156699e-03, -5.7948790e-03, 1.7218473e-03, 8.6186066e-02, -8.2310978e-03, 8.4572081e-03, 3.2470144e-02, -7.2876806e-04, -7.4659437e-02, -1.4426826e-02, -6.7664213e-03, 1.5026542e-02, 4.0727290e-03, 2.3569600e-04, -9.6417527e-04, -1.1503830e-03, -1.6481975e-02, -7.0209317e-03, 5.3436416e-03, 6.8159692e-02, -2.5559873e-03, -8.0474456e-05, -3.0456469e-03, -3.3534174e-03, 6.1106024e-04, -6.6513027e-04, -1.2051529e-01, -4.2777561e-02, 6.5030590e-02, 2.1362038e-02, 3.2946058e-02, 2.9181463e-01, 1.6087948e-03, 8.0183737e-02, 6.0054217e-04, -1.5031694e-03, -1.3727798e-01, 2.0673402e-02, 1.6634881e-02, -6.1021522e-02, -9.7963065e-03, -4.4847801e-03, -1.6810349e-03, -6.0316608e-03, -1.3532469e-04, 8.6372629e-02, -1.0683674e-02, -1.5961612e-02, 4.4840686e-02, 5.4682657e-02, 1.3794379e-02, 2.7690900e-03, 3.4621611e-02, 3.8512382e-03, 1.3362155e-03, 2.2860983e-02, 1.7252501e-02, 7.1021970e-03, -1.8353600e-02, -9.1281412e-03, -2.5052501e-02, 6.4535923e-02, -1.1188607e-01, -7.4342960e-01, 9.8237805e-03, -4.8992522e-02, -3.0361801e-02, 4.7444657e-05, -1.2857079e-02, -3.5183441e-03, -2.8601723e-02, -4.8774560e-03, -4.9416926e-02, 2.1382656e-03, -1.7278522e-02, 4.1791960e-03, -6.8051869e-04, 5.9474696e-04, 1.3638437e-01, -9.9423109e-03, 1.4076917e-03, -2.9004067e-03, 1.2549493e-02, 1.8330464e-02, 9.2371933e-02, -8.4709952e-04, -1.5642725e-02, 1.8227950e-02, 4.0236771e-02, -4.6799608e-02, -6.4027351e-03, -1.0166033e-01, 2.3036268e-02, -2.3710703e-02, -2.6576657e-02, 4.9998323e-03, 2.3283776e-02, 6.9294289e-02, -2.0691980e-02, 4.1049831e-02, 1.8761930e-04, -1.0511643e-01, 8.2109414e-02, -1.6860457e-02, -3.8362190e-02, 4.8856564e-02, -2.3737226e-02, -1.6952256e-02, 1.5742332e-02, -1.9643296e-02, 7.6947906e-03, 1.0548260e-03, 5.7553727e-02, 1.7045023e-02, -3.2199997e-02, 7.6176226e-02, -7.1644776e-02], dtype=float32), scale: Array([ 1.36227664e-02, 1.40898958e-01, 1.92645520e-01, 4.70757820e-02, -2.37463228e-03, 3.57880324e-01, 1.77441984e-01, 2.04336628e-01, 5.04735708e-02, 2.16793969e-01, 1.84319422e-01, 1.62005424e-01, -2.82178313e-04, 3.39963939e-03, 3.45121980e-01, 3.06478173e-01, 4.18081768e-02, 7.82519281e-02, -1.85935143e-02, 2.79815465e-01, 2.39701569e-01, 2.40149871e-02, 9.43208337e-02, 1.05237709e-02, 9.96645987e-02, 5.39937872e-04, -1.31208654e-02, 1.39623404e-01, -3.63838188e-02, 3.99398878e-02, 3.38648826e-01, 9.40667465e-02, 1.93201587e-01, -4.52386551e-02, 3.65994535e-02, 3.85606177e-02, -3.78566347e-02, 2.66506523e-01, 2.40695551e-01, 3.16709459e-01, 8.35959762e-02, 2.61536777e-01, 1.21971466e-01, 1.21512767e-02, 3.52942318e-01, 2.85213944e-02, -1.15962522e-02, 2.87642092e-01, 2.67458588e-01, 3.02993476e-01, 1.73773259e-01, 1.12724066e-01, -8.12414661e-03, 2.45114770e-02, 1.12504624e-01, 2.88635880e-01, 1.51033133e-01, -3.74849811e-02, 3.15434486e-01, 3.69797438e-01, 3.93722087e-01, 1.04381077e-01, 1.80823863e-01, 2.28892982e-01, 2.66926765e-01, -5.73799293e-03, 1.90365806e-01, 2.51159281e-01, 2.34797284e-01, 2.54733443e-01, 1.83505401e-01, 1.44614145e-01, 2.63359755e-01, 4.59246188e-01, 2.01807499e-01, 2.74832934e-01, 2.06239209e-01, 1.09179243e-01, 6.59588631e-03, 2.21557349e-01, -5.72106568e-03, 2.62187541e-01, 5.55602312e-02, 2.09347799e-01, 6.81416085e-03, 6.24816194e-02, -1.57044847e-02, 5.38076796e-02, -1.21786725e-03, 3.13683510e-01, 7.88503736e-02, 1.91961545e-02, -7.89848214e-04, 2.04859436e-01, 1.77144408e-01, 1.27074376e-01, -2.01367047e-02, 1.25420019e-01, 6.45905286e-02, 2.66942471e-01, 3.49235058e-01, 2.52543539e-01, 3.24409813e-01, 2.59198427e-01, 3.92820060e-01, 1.41132906e-01, 9.48882196e-03, 1.74179897e-01, 5.39034963e-01, 1.58920869e-01, 2.73100972e-01, 1.81179568e-02, 1.68145299e-01, 3.43630970e-01, 1.51725076e-02, 2.98200041e-01, 2.57570058e-01, 2.81464636e-01, 2.47943982e-01, 1.76195994e-01, 5.23593724e-02, 1.48531556e-01, 2.94840173e-03, 2.51852840e-01, -7.62507394e-02, 1.64415836e-01, 4.18539464e-01, 1.66102067e-01], dtype=float32), }, moe: { shared_experts: { intermediate: { bias: Array([-8.55890103e-03, 3.67313088e-03, -4.36119083e-03, -8.68369080e-03, -1.34754460e-03, -6.52316585e-03, -5.01648569e-03, -8.97654798e-03, 3.13179940e-03, -7.83655513e-03, 2.37924187e-03, -5.66111412e-03, -1.27145583e-02, -8.22409242e-03, 2.74701975e-03, -4.37194854e-03, -4.04660078e-03, -3.13023920e-03, -1.81288610e-03, -9.03141592e-03, -1.66141614e-03, -5.17393708e-01, -1.02338977e-02, 3.56379431e-04, -7.41085410e-03, 9.37872624e-04, -2.64340243e-03, 5.81945898e-03, -1.19625553e-02, -4.65264637e-03, -6.52810233e-03, -7.84709677e-03, 1.05619885e-03, -9.87111218e-03, 4.54840425e-04, -5.49622951e-03, 7.56876124e-03, -5.51949022e-03, -3.83866858e-03, -1.84899215e-02, 3.52073257e-04, -2.78544938e-03, -5.13466494e-03, -4.08213615e-01, -5.55376470e-01, -1.75705249e-03, 4.89682052e-03, 1.53476396e-03, 1.07086054e-03, -3.85978585e-03, 9.40655766e-04, 6.59122784e-03, -3.99606815e-03, -6.94369990e-03, -2.59699044e-03, 7.18095253e-05, -8.83713830e-03, -1.96340843e-03, -2.48700310e-03, -9.87609662e-03, -6.28701295e-04, -7.88388960e-03, 7.09942728e-03, 9.48800612e-03, -1.95277121e-03, -1.22706648e-02, 2.85065360e-03, 9.00720805e-03, -7.02000320e-01, 4.25555557e-03, -1.28851901e-03, 2.98675569e-03, -3.15172016e-03, -1.08107440e-02, -8.73899739e-03, -1.31636495e-02, -9.91051551e-04, -1.34902084e-02, 3.62220523e-03, 3.06941615e-03, -1.32400328e-02, -2.95568615e-01, 7.01408414e-03, -5.55180886e-04, 3.73574020e-03, -7.83216953e-03, 7.60261260e-04, -9.49840934e-04, -1.34194596e-02, -1.67394914e-02, -8.75537097e-03, -1.25364307e-03, -6.53209072e-03, -1.00438660e-02, -4.49657347e-03, -1.30805969e-02, 7.16269715e-03, -1.18865566e-02, 1.15038871e-04, -7.90204573e-03, -1.13492664e-02, -1.28111634e-02, -2.23936560e-03, -5.98883303e-03, -1.25457710e-02, -1.07937502e-02, -3.34314862e-03, -7.55358534e-03, -6.86253095e-03, -2.35536560e-01, -9.40762553e-03, 1.64968718e-03, -2.74949777e-03, -3.80960264e-05, 1.37860198e-02, 1.36729311e-02, -1.17542176e-03, -4.75597620e-01, 1.06082670e-02, -4.49829688e-03, 1.80422806e-03, -3.01245158e-03, -5.77643573e-01, -8.06397665e-03, 1.37929013e-03, -9.44454595e-03, -2.07981328e-03, 9.04347980e-05, -5.42783411e-03, 1.13049364e-02, -6.55044988e-03, -9.25507303e-03, -1.70839131e-02, -3.13228741e-03, -1.04950890e-02, -9.20292269e-03, -4.24578832e-03, -8.48777406e-03, -1.27782347e-02, -7.17586139e-03, -5.02940093e-04, -6.85630506e-03, -5.49637014e-03, 5.60814887e-03, -7.42090074e-03, -1.33384299e-02, -4.22688900e-03, -3.86846159e-03, -1.31040469e-01, -5.72234035e-01, 1.44373742e-03, -2.45060981e-03, -5.27091743e-03, -1.61155630e-02, -8.04609526e-03, -3.71574331e-03, 4.77314461e-03, -2.98696361e-03, -1.80231046e-03, -3.76312784e-03, 2.70129368e-03, -5.70825338e-01, -1.65502690e-02, -6.61239505e-01, -9.16241575e-03, -1.41646145e-02, -5.00553148e-03, -4.18055616e-03, -4.17894637e-03, -1.43527528e-02, 1.20390498e-03, 1.38255325e-03, -2.05959729e-03, -1.65631052e-03, 8.46141134e-04, -4.89455275e-03, -3.43874632e-03, -4.29662468e-04, -9.45413113e-03, -4.12517926e-04, -2.43844930e-03, -8.26839264e-03, -6.06988370e-03, -1.55913727e-02, -6.00419287e-03, 7.41206226e-04, -4.03943751e-03, -8.32396373e-03, -1.77192141e-03, -8.27328023e-03, -4.80760727e-03, -1.19567281e-02, -3.16868187e-04, -1.15141738e-02, 7.74921337e-03, -2.20119534e-03, 1.03418762e-03, -5.75479446e-03, -4.55250964e-03, -1.04654999e-02, -6.14567241e-03, -1.54978642e-02, -7.39947148e-03, -1.07835466e-02, -1.21487593e-02, -6.76209899e-03, 1.25275925e-03, -3.19723099e-01, -3.41119803e-03, 2.97757820e-03, -5.52376360e-03, -9.89249256e-03, -4.31803286e-01, -1.36245508e-02, -7.62469275e-03, -1.74430560e-03, -1.96244684e-03, 5.65344701e-04, -6.50570095e-01, -1.45257404e-03, -2.76738009e-03, -8.20099376e-03, 2.03965130e-04, -1.13646872e-02, -2.37802067e-03, -5.39474888e-03, 2.26747943e-04, -2.44784285e-03, -1.09350355e-02, 8.08892888e-04, -8.32007360e-03, -1.91815558e-03, -9.69145447e-03, 1.22545648e-03, 4.08804649e-03, -5.84384939e-03, 3.66018945e-03, 4.44652513e-03, -9.09210462e-03, -4.17519035e-03, -3.98625523e-01, -3.37073533e-03, -8.35657492e-03, -4.79207002e-03, -1.39846019e-02, -8.26936215e-03, -8.90700985e-03, -2.52558112e-01, 3.46473581e-03, -1.71433843e-03, -1.91845442e-03, -3.26462858e-03, -8.05919431e-03, -5.96231222e-03, -2.60497467e-03, -5.12533821e-03, -9.94778052e-03, -1.35790939e-02, -7.90751539e-03, -5.66528225e-03, 2.63936934e-03, -4.14405525e-01, 2.22919116e-04, 8.12191633e-04, -1.82921547e-04, -7.25358841e-04, -1.08812936e-02, 3.49700474e-03, -1.17458524e-02, -1.99729693e-03, -5.74411824e-03, -1.04112234e-02, -5.87427756e-03, 5.42524364e-03, -4.66417568e-03, -3.61002120e-03, -4.92561430e-01, -5.91339357e-03, -8.21670890e-03, 6.48486370e-04, -1.16507420e-02, -7.28300912e-03, 9.69227776e-03, -1.40586449e-03, 1.34252605e-03, -4.55454225e-03, -1.05377687e-02, 8.61465523e-04, -1.29449442e-02, 3.23367422e-03, -1.80069392e-03, 2.35577201e-04, -8.91069509e-03, -1.41903665e-03, -8.80321302e-03, -4.76717623e-03, -2.86917109e-03, -3.21599003e-03, 3.61907203e-03, -7.99222197e-03, -6.45599188e-03, -5.55047020e-03, -9.93613340e-03, -8.86950456e-03, 1.25965700e-02, 3.33399116e-03, -1.01383878e-02, 2.34254939e-03, 5.69945900e-03, -5.75918052e-03, 7.65805482e-04, -1.72046013e-02, -8.67277035e-04, -1.03675043e-02, -7.15132989e-03, -1.15488358e-02, -2.48059118e-03, -1.51636070e-02, -1.34787019e-02, -8.78769148e-04, -6.11743808e-01, -6.02744520e-04, 7.56409660e-04, -3.29326536e-03, 5.58212679e-03, -9.33979824e-03, -2.45822361e-03, -5.48888650e-03, -4.64835018e-03, -4.38324921e-03, -3.24640493e-03, -6.00662351e-01, 1.93342322e-03, -1.04327723e-02, 4.34050598e-04, 3.05471686e-03, -7.74156442e-03, -1.68287978e-02, -1.33079069e-04, -3.91604519e-03, -1.49135012e-02, -8.73885211e-03, -5.66676259e-03, 3.75113130e-04, -5.36525948e-03, -1.39129013e-02, 1.88402925e-03, -6.80929542e-01, 2.44663190e-03, -1.16351806e-02, -5.38230175e-03, -3.69915483e-03, -4.62271646e-03, 4.26167302e-04, 1.99855946e-04, -1.86087582e-02, 7.48794526e-03, -1.61632244e-02, -4.98278052e-01, -8.40996485e-03, -6.01133797e-03, -8.28853925e-04, -5.63414209e-03, 5.07016899e-03, -9.90651455e-03, -4.54475218e-03, 1.22021348e-03, 5.08222263e-03, -5.74093172e-03, 2.58398801e-03, -5.80827938e-03, 2.76929396e-03, -8.52370542e-03, -8.13993160e-03, -3.10428627e-03, 9.73783247e-03, -2.26997398e-03, -7.61909829e-03, 4.68078069e-03, -3.60694993e-03, -8.70567188e-03, -1.44884512e-02, -2.72786571e-03, 4.09194687e-03, -9.40709934e-03, 4.71121492e-03, -5.04164118e-03, -1.17910588e-02, 1.36497710e-03, -1.07826991e-02, -1.53774535e-03, -1.12592215e-02, -2.51332531e-03, -1.03800860e-03, 9.40567255e-03, -1.59978624e-02, -9.16326977e-03, 5.42400870e-04, -2.22451333e-03, -7.41019659e-03, -3.90202254e-01, -1.51460301e-02, -6.29314408e-03, -4.74618049e-03, 3.59613076e-03, -1.17005240e-02, -1.27715422e-02, -1.15625896e-02, -1.68063808e-02, -1.12238340e-02, -5.49799949e-03, 7.02496106e-03, -2.54716747e-03, -9.75850690e-03, -3.14950384e-03, -7.65251042e-03, -1.46619789e-03, 1.40559685e-03, -2.16309223e-02, 4.46918281e-03, -8.65491666e-03, 5.14270552e-03, 3.51362582e-03, -9.26864985e-03, -1.17338272e-02, -2.89736595e-03, -1.97741878e-03, -5.52454265e-03, -8.56413040e-03, -9.34949901e-04, 2.97652930e-03, -2.00355076e-03, -3.76361725e-03, 1.10566849e-03, -5.63339936e-03, -2.81627057e-03, -1.68547058e-03, -4.57938435e-03, 6.58686552e-03, -4.55372721e-01, -5.61221363e-03, -6.51222235e-03, 1.07192027e-05, -8.45061056e-03, 5.90987783e-03, -5.17501831e-01, -4.53207910e-01, 1.17717020e-03, -7.01933773e-03, 4.97156382e-03, 1.22358715e-02, -3.75395943e-03, -1.19871208e-02, 7.76410243e-03, -1.40578160e-02, 3.29062645e-03, 6.30691764e-04, -1.67252067e-02, 1.95841817e-03, -9.67236888e-03, -1.17397988e-02, -6.99722324e-04, -6.32376294e-04, -6.49616960e-03, -2.94766188e-01, 1.18942198e-03, -7.66102644e-03, 3.43931909e-03, 2.72910809e-03, -5.50179362e-01, -1.15189431e-02, -5.44976583e-03, -3.17149970e-05, 1.41578354e-03, -6.40626810e-03, -3.47646652e-03, 6.37558231e-04, -2.99780950e-04, 5.98926889e-03, -4.70549008e-03, -7.91283231e-03, -9.49787791e-04, -3.55603476e-03, -1.14685595e-02, -1.00854738e-03, 2.23665382e-03, -2.58625904e-03, -1.55551946e-02, -9.62937716e-03, -3.64469341e-03, -6.78142011e-01, -3.52118816e-03, -1.36016486e-02, -1.84007958e-01, -5.86909475e-04, -8.81857052e-03, -1.06971972e-02, -7.59707938e-04, -5.68926990e-01, -5.22532880e-01, -2.09414326e-02, -1.72124021e-02, -6.04376057e-03, -3.09983082e-03, -4.42491425e-03, -8.42403620e-03, 7.68742245e-03, -3.21401260e-03, 6.66661141e-03, 2.13651289e-03, -4.34801867e-03, -7.12090218e-03], dtype=float32), kernel: Array([[-0.06747823, 0.12502806, 0.01688226, ..., 0.04402845, 0.0489017 , 0.02632801], [-0.01445617, 0.03798477, -0.00418379, ..., 0.01224544, 0.02048267, 0.01027028], [-0.01829829, 0.01891246, 0.01610937, ..., 0.03225997, -0.01639963, -0.03890606], ..., [ 0.00887221, -0.00487363, 0.02399558, ..., 0.01346108, 0.01049829, -0.03240183], [-0.01674049, -0.00387911, -0.03074075, ..., 0.00146568, -0.00876305, -0.02836148], [ 0.01625663, 0.03327096, -0.04843638, ..., 0.08134065, 0.03410971, 0.01957767]], dtype=float32), }, output: { bias: Array([-0.16341606, 0.2722003 , -0.01811248, 0.01928756, -0.02578784, -0.10320964, 0.0457683 , 0.00972074, 0.05574008, 0.24268645, -0.02999464, 0.043408 , -0.0737564 , 0.03339861, -0.04492559, 0.09947823, 0.05016089, -0.08309133, -0.05127812, 0.11153366, -0.08190622, -0.06840179, -0.02421227, -0.02895692, -0.04004822, 0.03941108, -0.0170032 , -0.00097349, 0.07023378, 0.02564083, 0.03406862, -0.06778747, 0.11675587, -0.08771975, 0.04982556, -0.07175723, -0.04058652, -0.01989682, -0.02999596, -0.07330333, 0.03119163, -0.0147863 , -0.83884764, 0.04124971, -0.147959 , 0.06891607, 0.05300924, 0.2076162 , 0.04646805, -0.05764642, -0.03448724, 0.05170846, -0.02983713, 0.14581041, -0.04477751, -0.0028539 , -0.08908532, -0.09743858, 0.05685575, 0.08213867, 0.0757231 , -0.13601153, -0.02382083, -0.01865633, -0.00175037, -0.03086867, -0.03948168, 0.06656119, 0.03003574, 0.07914743, -0.02571616, 0.03044318, 0.05824798, 0.02247159, 0.7948062 , -0.00407046, -0.07054033, 0.12808433, -0.08461454, 0.02311847, -0.65620077, 0.10964675, 0.10405939, -0.00352513, -0.02670519, 0.15050422, 0.13908558, 0.14308207, 0.03576402, -0.11146745, 0.07615335, 0.04982926, -0.11778215, -0.10201599, -0.01381564, -0.01693749, -0.03509882, 0.01073222, -0.01447865, 0.08480705, 0.02910439, 0.00359502, 0.01869445, -0.05797101, 0.09884419, 0.0161234 , 0.00668302, -0.08439763, -0.13141344, -0.03596077, -0.14647698, 0.03868678, 0.11478355, -0.14481665, 0.00641343, 0.16066928, 0.04811645, -0.02036389, -0.03384693, 0.06242058, -0.00735435, 0.02676932, 0.12175596, -0.06479023, 0.04808705, 0.05643554, -0.07896087, -0.02733074], dtype=float32), kernel: Array([[ 0.04431782, -0.07884661, -0.02030264, ..., 0.03209 , 0.02117256, 0.00271614], [-0.05527279, 0.03204606, -0.00293706, ..., -0.03798404, -0.02230972, 0.00913886], [-0.01904109, -0.02136857, 0.00946268, ..., 0.00186177, -0.02516458, 0.05516352], ..., [-0.03168841, 0.07299381, -0.00359565, ..., -0.01263119, -0.04168412, 0.01372963], [-0.02394411, 0.08588913, 0.00119561, ..., 0.0126469 , -0.04365737, 0.0029881 ], [-0.01780676, 0.05708533, 0.01049639, ..., 0.03207007, -0.03454534, 0.0507676 ]], dtype=float32), }, }, }, }, 4: { attention: { attention: { key: { bias: Array([ 0.25172058, 0.68627775, 0.5949247 , 0.1658349 , -0.3173467 , 0.1774752 , 0.25412992, 0.08010545, 0.56002 , 0.17958656, -0.17482378, 0.07066374, 0.53656256, -0.5385957 , 0.03753721, 0.3764516 , -0.05485516, 0.7278333 , -0.7301842 , 0.7662722 , 0.8557523 , 0.19676152, -0.7459649 , 0.11304809, -0.27465415, 0.41445032, 0.31016302, -0.9027352 , 0.15300739, -0.12067419, 0.11333649, 0.36544785, 0.10615282, -0.19266807, 0.00742444, -0.3133698 , 0.13811715, 0.17462967, -0.04043375, 0.01431966, 0.02120961, -0.31457424, 0.24998754, -0.30905357, -0.33957723, 0.44098425, 0.31071022, 0.5162943 , 0.1519571 , 0.20767058, 0.11702721, 0.30976802, -0.26687843, 0.02610935, -0.38046166, 0.15602328, 0.1049099 , -0.3816162 , 0.34945863, -0.13488789, 0.25807512, -0.14170453, -0.2737117 , 0.6794868 , -0.5637404 , -0.7506761 , -0.59640557, -0.35966128, -0.36779413, 0.6445572 , -0.15863825, 0.28188404, 0.6817374 , -0.30411932, 0.15068318, 0.21245846, 0.47254717, -0.3180801 , -0.72213024, -0.00796549, 0.52852696, -0.74865776, 0.72052324, -0.6231846 , -0.35524404, -0.6322673 , 0.5927172 , 0.7579426 , 0.22011517, 0.37161058, -0.58662343, -0.5653432 , 0.4952815 , 0.34009102, 0.4199334 , -0.04050065, 0.967862 , 0.62645787, -0.24915175, 0.6059676 , -0.12386217, -0.97235304, 0.98436683, -0.71207863, -0.2554404 , 0.29857978, -0.6535443 , -0.00581208, 0.54410416, -0.64588374, -1.2313039 , -0.7612568 , -0.5072008 , -0.15740068, 0.5934992 , 0.29966694, -0.78861797, 0.95648044, 0.90275365, 0.3240316 , -0.7827806 , -0.90508664, 0.24878727, -0.92607623, 0.45041922, -0.7728655 , 0.2784089 , 0.75287944], dtype=float32), kernel: Array([[ 1.6211687e-01, -2.6181787e-02, -5.1868878e-02, ..., -2.0115136e-01, -1.6275570e-01, 4.0626042e-02], [ 2.0827402e-01, -1.1491413e-01, -1.4099246e-01, ..., 1.2704584e-01, 2.0065814e-01, 3.5636088e-01], [ 1.4854251e-01, -1.1401314e-01, -4.7746781e-02, ..., -8.4292240e-02, 1.9939394e-01, 9.9089958e-02], ..., [-5.0814073e-03, 7.1649961e-02, 8.1174605e-02, ..., 1.8439499e-01, -1.8450679e-01, 3.2407336e-04], [ 5.2984446e-02, -4.0257105e-01, -3.7533784e-01, ..., 2.7334958e-01, -7.9834215e-02, -2.6604572e-01], [-1.3808161e-01, -2.5356847e-01, -2.2354984e-01, ..., -3.9307380e-01, 3.9180964e-01, 4.5249498e-01]], dtype=float32), }, query: { bias: Array([ 3.67379338e-01, -3.86799455e-01, -3.21791142e-01, -8.42057094e-02, 4.18459654e-01, 5.85961580e-01, -4.60834682e-01, -6.04065180e-01, 1.94889054e-01, 5.26792884e-01, -4.81182262e-02, -8.55236873e-02, -5.21454439e-02, 4.12250131e-01, -1.91803545e-01, -5.08220792e-01, -6.88099623e-01, 1.73875868e-01, -6.42500934e-04, 3.13504860e-02, 7.57871941e-02, -1.18366331e-01, 7.69717107e-03, 6.50874972e-01, -1.46498501e-01, -3.85033339e-02, 1.68164968e-01, 2.94075936e-01, 5.43904722e-01, 1.82771876e-01, 4.28779982e-02, 3.94661844e-01, 2.40447834e-01, 7.63046503e-01, 1.41106710e-01, 1.50562599e-01, 7.20623806e-02, -3.31430919e-02, 9.66051295e-02, -4.25018109e-02, 2.59480290e-02, 5.50236627e-02, 6.59183860e-02, 7.35617638e-01, 9.20522138e-02, -1.12395249e-01, -2.63706028e-01, -4.15704325e-02, 9.31539387e-02, -6.19517326e-01, 3.92871583e-03, -2.88900491e-02, 8.03363383e-01, -1.52911708e-01, 7.79530257e-02, -1.06074944e-01, -5.33656120e-01, 1.22914307e-01, 1.09775752e-01, 3.54613900e-01, -4.29570854e-01, 1.18217006e-01, 5.40162444e-01, -1.17120035e-01, -6.53074026e-01, -4.32521045e-01, -7.02763855e-01, -4.33477342e-01, -4.24702972e-01, 4.39991415e-01, -2.19614744e-01, 4.09397632e-01, 5.78722954e-01, 1.95280928e-02, 2.52549767e-01, 5.14830410e-01, 5.46005607e-01, -6.23379529e-01, -4.24228251e-01, -2.12890625e-01, 2.95019180e-01, -6.27236724e-01, 5.73115885e-01, -3.63081634e-01, -3.95359695e-01, -4.50571030e-01, 5.81661999e-01, 5.84261417e-01, 2.53536344e-01, 5.42065144e-01, -3.84602755e-01, -4.97589141e-01, 4.21692282e-01, 4.60142285e-01, 3.82059246e-01, -3.72222006e-01, 5.13080180e-01, 2.00476050e-01, -6.01216495e-01, 8.54436874e-01, 2.28271738e-01, -3.90953124e-01, -9.38059986e-02, -8.37827742e-01, 5.72855584e-02, -1.37482524e-01, 1.96730390e-01, 1.36508822e-01, 1.39595941e-01, -4.52210784e-01, 1.51813626e-01, -7.74807990e-01, 2.37015381e-01, -3.03821355e-01, -3.49135190e-01, -1.64326906e-01, -8.14965189e-01, -1.61662132e-01, 8.50251555e-01, 2.36134544e-01, 1.86818361e-01, -8.37599099e-01, 9.94282067e-02, 8.31496865e-02, 1.03579015e-01, -4.60613877e-01, 8.01612064e-02, 8.32802534e-01], dtype=float32), kernel: Array([[ 0.00486563, 0.01157592, 0.0173437 , ..., 0.06944456, -0.07013498, 0.01022998], [-0.07428966, -0.05709548, -0.03850973, ..., -0.09689807, 0.22320649, 0.2770357 ], [-0.10973258, 0.19551165, 0.12421291, ..., -0.02782791, -0.36778155, -0.19087008], ..., [ 0.30281523, 0.3621789 , 0.32462892, ..., 0.0546729 , -0.04859495, -0.10934605], [ 0.5838381 , 0.22978936, 0.17813566, ..., -0.16678724, 0.70342004, 0.18783332], [-0.48419687, 0.04412645, -0.01096392, ..., -0.05038855, 0.37432894, 0.02556431]], dtype=float32), }, value: { bias: Array([ 0.00544928, 0.17689343, 0.04735808, -0.18264541, -0.04237033, -0.17941093, 0.00340467, -0.00149406, -0.23728922, -0.07449124, -0.2651422 , -0.0984367 , -0.12018497, -0.08068941, -0.01734875, -0.06606279, 0.1477764 , -0.02022878, -0.1402983 , -0.02072977, 0.12049822, 0.06401265, 0.00406305, 0.15095161, -0.20597391, 0.21405579, -0.08197237, -0.00951357, -0.13306661, -0.08558191, 0.08653639, -0.10818963, -0.1644184 , -0.15369424, 0.3490528 , 0.01967676, -0.16458155, 0.10356332, -0.2320539 , -0.0433312 , 0.294788 , 0.08036903, -0.12358049, 0.16056572, 0.25535643, 0.16914676, -0.06201077, 0.28501964, -0.13111252, 0.11861207, 0.26627177, 0.16901444, -0.19237208, -0.08137509, 0.2016543 , 0.15213989, -0.14299503, -0.24307302, -0.29300013, -0.0722383 , 0.05268372, -0.14988767, -0.19426815, -0.24243473, -0.06109034, -0.13096058, 0.155086 , 0.15769799, 0.10678913, 0.00772086, 0.12339002, -0.02504941, -0.13470462, 0.02844545, 0.09698638, 0.11937644, -0.01387584, -0.13328613, -0.05046262, -0.02981377, -0.11246655, -0.06740061, -0.00562667, 0.15140204, 0.08057123, -0.03175431, 0.05622792, 0.11395832, -0.09259764, -0.02921034, -0.10965273, 0.09367548, 0.07572406, 0.17622064, -0.12645517, -0.06927612, -0.13641733, 0.11861874, -0.01573502, -0.10696157, -0.14588912, -0.16159363, 0.1815869 , 0.08687254, 0.11738776, -0.0497781 , -0.12963611, 0.01230228, -0.10121961, -0.06638794, 0.11086646, -0.05197389, 0.24918154, -0.16594617, 0.20172946, 0.06618202, 0.1124498 , 0.11753738, -0.19368784, 0.04725859, -0.13396324, -0.10240424, -0.11015446, 0.0015793 , 0.05593257, -0.12091528, 0.08938907, -0.03860855], dtype=float32), kernel: Array([[ 0.06369832, 0.04487664, -0.0572863 , ..., -0.00566542, 0.00958204, -0.0823305 ], [-0.48305708, -0.05723378, 0.09843583, ..., 0.01919795, 0.04487755, 0.36410812], [-0.39578405, 0.17672975, -0.04119537, ..., 0.06564505, -0.01046039, -0.30802208], ..., [-0.04638171, -0.03762753, -0.15475406, ..., 0.11968867, -0.1331625 , 0.01799979], [ 0.34840006, 0.0131262 , -0.17438433, ..., 0.21620421, -0.14971787, -0.73045504], [ 0.14833876, 0.18964465, 0.05431836, ..., 0.10322906, 0.10638059, -0.31155896]], dtype=float32), }, }, output: { dense: { bias: Array([-1.46490365e-01, 2.10801542e-01, -1.72206163e-02, -3.77512947e-02, -7.96984136e-03, -7.52876028e-02, 1.01055026e-01, 2.95142401e-02, 4.51585725e-02, 2.89253950e-01, -3.78346108e-02, 3.64244655e-02, -6.14614636e-02, 4.95899320e-02, -3.73293981e-02, 5.79853542e-02, 8.40439461e-03, -1.02269538e-01, -7.46445209e-02, 1.14303716e-01, -2.32893787e-02, -3.25526744e-02, -1.71621908e-02, -2.69102436e-02, -4.24926169e-02, 6.34983480e-02, -9.30675417e-02, -2.39975707e-04, 8.57859775e-02, 5.52252270e-02, 8.55132714e-02, -9.58903059e-02, 1.34693608e-01, -1.06799692e-01, 8.42991620e-02, -7.29722008e-02, -3.16112302e-02, -1.33186892e-01, -3.01891286e-02, -1.23371415e-01, 2.47585457e-02, 9.32596158e-03, -8.83968234e-01, 3.25804874e-02, -1.36455193e-01, 5.22013418e-02, 7.06611201e-02, 2.37159967e-01, 8.91016126e-02, -2.99384054e-02, -1.87199526e-02, 2.26440327e-03, -2.86240019e-02, 1.45185545e-01, 1.16965972e-01, -3.56947407e-02, 1.23156169e-02, -7.89485201e-02, 2.42000669e-01, 5.18658943e-02, 1.23368412e-01, -1.33664429e-01, -5.28409593e-02, -1.49152186e-02, -7.04273768e-03, 4.40860493e-03, -9.83215496e-02, 4.80295867e-02, 1.94351804e-02, 9.06426013e-02, -1.53706968e-02, 5.25671393e-02, 7.91645125e-02, 2.07034983e-02, 7.01766312e-01, 1.80371366e-02, -1.23936214e-01, 3.77195626e-02, -9.12794545e-02, -5.07131638e-03, -5.29812515e-01, 2.14143340e-02, 7.39834160e-02, -7.41400057e-03, 6.01231493e-02, 7.33223036e-02, 1.53090119e-01, 1.70849040e-01, -4.07319590e-02, -4.82572988e-03, 2.90474202e-02, 4.86663431e-02, -1.04625016e-01, -8.78558010e-02, -4.20586728e-02, 3.25415730e-02, -7.98427388e-02, -1.87286697e-02, -2.41589937e-02, 8.27142298e-02, 4.98151742e-02, 5.18750064e-02, 3.29844393e-02, -3.68855074e-02, 8.92899334e-02, -4.56710160e-03, 2.97619849e-02, -7.04986304e-02, -1.08520284e-01, -1.11746334e-01, -7.71660432e-02, 8.49972386e-03, 2.37586319e-01, -1.24874555e-01, -1.93465855e-02, 8.61749053e-02, 8.92835855e-02, -8.50701034e-02, -3.94290835e-02, -2.06257198e-02, -1.92599650e-02, 2.76334919e-02, 6.91349506e-02, -2.97043975e-02, 8.56228359e-03, 5.08836359e-02, -1.17148422e-02, -2.16257907e-02], dtype=float32), kernel: Array([[ 0.16482222, -0.22908561, 0.5409443 , ..., 0.52288747, 0.45133135, -0.3148359 ], [ 0.04791335, -0.19366321, 0.02344365, ..., 0.25754645, 0.15889917, -0.26091245], [ 0.21047738, 0.00189075, -0.053845 , ..., -0.10111952, -0.42335704, -0.05204519], ..., [-0.07512332, 0.07116441, -0.20204972, ..., -0.3296345 , 0.2802311 , -0.00193282], [-0.01015196, 0.04124757, 0.08690923, ..., 0.38333163, -0.18434097, -0.01834725], [ 0.0501605 , 0.7881087 , 0.21703115, ..., -1.006583 , -0.5440753 , 0.06839034]], dtype=float32), }, }, }, layernorm_after: { bias: Array([ 7.00707082e-03, -2.13932544e-01, -4.38091122e-02, -1.80086847e-02, 1.56905562e-01, 3.25518586e-02, -6.51721493e-04, -3.70227383e-04, -9.39783454e-03, -1.21371552e-01, 6.61288295e-03, -3.71838771e-02, 1.69692468e-02, -1.13847349e-02, -1.44199710e-02, -1.44837052e-03, 1.02954609e-02, 7.14356266e-03, -2.28633024e-02, -9.65952054e-02, -7.08255393e-05, 4.81518777e-03, -1.34146630e-05, 3.52103100e-03, -1.46182910e-01, -2.66583692e-02, -2.81199813e-02, 2.20761336e-02, 8.75673592e-02, 1.68776400e-02, 7.70330988e-03, -7.63349526e-04, 1.69382174e-03, -5.38112130e-03, -2.32931431e-02, 1.33482378e-03, 5.63706718e-02, -1.53671801e-01, -5.60674816e-04, 1.17010390e-02, 4.67808433e-02, -1.00002938e-03, 7.54884481e-02, 9.78838280e-03, -2.51426380e-02, -1.93388745e-01, -1.30395563e-02, -3.22579712e-01, -1.23171741e-02, -2.32061837e-02, -5.90802915e-02, 6.99069537e-03, 1.86737273e-02, -1.55908689e-01, 5.42053096e-02, -1.68696530e-02, -3.76862218e-03, 1.75712314e-02, -2.17723008e-02, -1.39841557e-01, -1.11271013e-02, -8.20189528e-03, -1.91661697e-02, -2.04071011e-02, 5.92404092e-03, 2.63817403e-02, 2.42956262e-02, 6.90133497e-02, -7.82949443e-04, 6.91150129e-03, -6.76295608e-02, -8.51555727e-03, -1.79697247e-03, 4.60966006e-02, -4.87765044e-01, 3.24066654e-02, -7.08101019e-02, -2.59833187e-02, -4.40584077e-03, 3.93147208e-03, 2.80292213e-01, -3.62146534e-02, 1.14013357e-02, 1.44051407e-02, 8.79329368e-02, -3.40585620e-03, 1.61548674e-01, -4.32909727e-02, 5.22663780e-02, -8.88153631e-03, -5.72965667e-03, 1.60463457e-03, -1.44764967e-03, -4.41680253e-02, -6.15086630e-02, 2.85431091e-03, -5.11140227e-02, 3.58232483e-02, 3.14552360e-03, -8.34190473e-02, -6.30146563e-02, 3.52296643e-02, 1.75646115e-02, -2.81428844e-01, 1.00661494e-01, -1.63044967e-02, -1.77673250e-03, -2.50048609e-03, 9.31597501e-02, -5.21637946e-02, -1.21001536e-02, -3.70397139e-03, -1.89869609e-02, 4.85324338e-02, -1.78271660e-03, -1.17816813e-02, 6.07670583e-02, -6.83511868e-02, -7.59183466e-02, 2.76686642e-02, -2.92036515e-02, 7.23615754e-03, -1.44946417e-02, 6.89900964e-02, -1.19854533e-03, -3.26084495e-02, -3.72714689e-03, 8.10674950e-02], dtype=float32), scale: Array([ 0.01037673, 0.20990598, 0.17576548, 0.21721113, 0.29847282, 0.69343257, 0.00376189, -0.01851019, 0.03572948, 0.19570692, -0.02536977, 0.42987674, 0.01018215, 0.15321532, 0.48395535, -0.00318469, 0.16068086, 0.01112385, 0.36739224, 0.4355385 , -0.01194211, -0.00359833, -0.086844 , 0.00597237, 0.5273556 , 0.03060911, 0.6654633 , -0.05266754, 0.36045882, 0.10801558, 0.29989985, -0.04426623, 0.00284036, 0.02850281, 0.08200129, 0.00828265, 0.4786764 , 0.32044822, 0.6007385 , 0.39719838, 0.38223657, -0.02341203, 0.02303379, 0.25033328, -0.08627673, 0.5241369 , 0.544966 , 0.3724897 , -0.32547915, -0.24456456, 0.32741693, 0.11249542, 0.07124001, 0.5731743 , 0.22054233, 0.68054676, -0.04105046, 0.07573894, 0.05682403, 0.4511571 , 0.51980895, 0.05435016, 0.06856246, 0.16396722, 0.00960637, 0.06122901, 0.23258007, 0.3978171 , 0.23067153, -0.0107969 , 0.517839 , 0.03140327, 0.02615155, 0.70969087, 0.1419986 , 0.37057772, 0.54277253, 0.18082201, 0.0020515 , -0.11159092, 0.43598834, 0.37816525, 0.586952 , 0.35433614, 0.50891733, 0.00721514, 0.27681717, 0.38934416, 0.4676223 , 0.18506213, 0.18672895, 0.01464603, -0.00548072, 0.72455907, 0.26698393, -0.00541573, 0.8330279 , 0.64272374, 0.00286223, 0.52491736, 0.5533957 , 0.3439355 , 0.32210883, 0.56023586, 0.5354366 , 0.3412869 , 0.05528033, -0.15407614, 0.28940812, 0.39378768, 0.5161475 , -0.03409952, 0.05442618, 0.390517 , 0.00593842, 0.06129869, 0.42658743, 0.4008259 , 0.46517026, 0.30656123, 0.2991743 , 0.03482211, 0.5670987 , 0.35471496, 0.08178211, 0.0831662 , 0.00438205, 0.29678687], dtype=float32), }, layernorm_before: { bias: Array([-1.00732700e-03, -3.65120143e-01, -3.34881805e-02, 4.26957064e-04, -7.94777123e-04, -6.86906278e-03, -1.02201905e-02, -4.80189286e-02, 1.55887365e-01, -1.59472153e-01, 5.88005059e-04, -4.74225767e-02, 1.99526129e-03, -6.20241184e-03, 2.79833134e-02, 7.60109648e-02, -4.03508246e-02, -5.76728676e-03, -8.25315190e-04, -5.45392111e-02, -9.78699885e-03, -2.21987873e-01, 8.89718998e-03, 1.56212691e-02, 2.51562544e-03, -2.31638588e-02, 2.57532690e-02, -3.37633025e-03, -4.75819921e-03, 2.06635101e-03, 4.15294990e-02, -1.22549701e-02, 9.92366020e-03, 1.33297034e-02, -6.88615516e-02, -2.13969056e-03, -9.33400914e-03, -1.47152513e-01, 1.47844050e-02, 2.68603060e-02, 1.06496513e-01, -1.40187237e-03, 2.46544197e-01, -1.18307155e-02, 1.22905318e-02, 7.02085486e-03, 3.16104256e-02, 1.29607348e-02, -1.19067011e-02, 5.89615945e-03, 1.45399803e-02, 7.92596638e-02, 8.44230223e-03, -5.16842827e-02, 2.47099735e-02, 7.68707599e-03, -7.68995145e-03, 3.25880721e-02, -3.79534140e-02, -1.32843507e-02, 1.00192919e-01, 8.90694335e-02, 1.43536124e-02, -1.85336508e-02, 4.20249486e-03, 4.04581055e-02, 1.56428572e-02, 5.54322731e-03, 7.59444525e-03, -9.50715330e-04, -5.35057345e-03, 2.70144927e-04, 1.31974250e-01, 2.08387990e-02, -5.05078256e-01, -4.19664755e-02, 2.06986954e-03, 2.65133400e-02, 1.08654350e-02, -2.48142332e-02, -2.32796840e-04, -3.82554834e-03, -3.06907180e-03, -3.17831822e-02, -2.74789296e-02, 1.76229049e-04, -5.93862869e-03, -2.54624914e-02, 5.30235134e-02, 3.77030671e-02, -3.30569521e-02, 6.81126863e-02, -3.34922522e-02, -5.41602597e-02, -4.98229451e-03, -1.29764492e-03, 1.27451606e-02, 5.30401291e-03, 8.72196928e-02, 5.26771834e-03, -5.91456294e-02, 2.01584827e-02, -1.72205925e-01, -2.79393774e-02, -3.19484659e-02, -3.31652700e-03, 3.22748870e-02, 1.10277152e-02, 7.98177347e-02, -2.52419740e-01, 6.41727597e-02, 2.78249923e-02, -1.18425563e-01, 5.46144024e-02, -1.38351889e-02, 1.97037379e-03, 3.38066697e-01, -1.05525143e-02, -5.22397608e-02, -8.39580037e-03, 1.55923087e-02, 2.35547218e-02, -4.73677404e-02, 7.04186484e-02, -5.95991965e-03, 5.85895148e-04, 4.35512178e-02, -9.74338278e-02], dtype=float32), scale: Array([-0.00892128, 0.22834587, 0.07191961, -0.03907548, -0.00996873, 0.35814652, 0.27738327, 0.11775097, 0.07957964, 0.2968416 , 0.01412186, 0.27139878, -0.00244969, -0.01754642, 0.31717873, 0.24255775, 0.34514305, -0.0094078 , -0.01727619, 0.2070495 , 0.23165925, 0.48358163, 0.08980764, 0.16053861, 0.07536255, 0.02379382, 0.3457654 , 0.22846563, -0.1335053 , 0.01941462, 0.26780826, 0.09502309, 0.14652625, 0.05376663, 0.22817616, 0.20199119, 0.2713218 , 0.26872477, -0.0762281 , 0.22205614, 0.31939635, 0.24618064, 0.17259929, 0.21590437, 0.02615085, 0.17664434, 0.36018652, 0.12768349, 0.20409435, 0.14295201, 0.18369466, 0.3224867 , 0.01613115, 0.11641789, 0.19955999, 0.08757862, -0.01346728, 0.198473 , 0.29986396, 0.36929435, 0.5931218 , 0.19321899, 0.11543731, 0.16761477, -0.00143133, 0.07413565, 0.2747692 , 0.02911899, 0.12630646, 0.02624349, 0.3049502 , 0.17227921, 0.35921425, 0.295701 , 0.13059482, 0.19860806, 0.01733009, 0.24637464, 0.00986639, 0.19075045, -0.00124613, 0.27278668, 0.01111688, 0.16424142, 0.18801676, 0.01323086, 0.00368139, 0.03970122, 0.26844364, 0.10092615, 0.09264332, 0.16377948, 0.22653754, 0.35064098, -0.05561429, -0.00500794, 0.07151927, 0.0050664 , 0.2468497 , 0.10480628, 0.17903912, 0.08759426, 0.41523352, 0.24769782, 0.11629754, 0.03568345, 0.20900775, 0.05634574, 0.4340352 , 0.3333243 , 0.41587278, 0.20580357, 0.3584831 , 0.2544934 , 0.01131326, 0.34364027, 0.22543208, 0.13177043, 0.19698657, 0.21796568, -0.03295131, 0.298913 , 0.47042173, 0.26397368, 0.02212258, 0.01400256, 0.3371711 , 0.22812258], dtype=float32), }, moe: { shared_experts: { intermediate: { bias: Array([-5.27199693e-02, -1.22556537e-02, -4.80708107e-02, -2.20634907e-01, -1.40253054e-02, -4.87366840e-02, -8.70805699e-03, -1.54133514e-02, -2.52253171e-02, -3.50418165e-02, -2.73905657e-02, -3.95800360e-02, -3.71573754e-02, -2.81724222e-02, -1.33444788e-02, -1.28413305e-01, -4.53637205e-02, -3.11292056e-02, -3.32505777e-02, -3.26416194e-02, -3.45684476e-02, -1.40969567e-02, -2.91586872e-02, -2.76421569e-02, -3.73606943e-02, -3.91720347e-02, -3.77491489e-02, -1.35208545e-02, -2.09600553e-02, -4.55045812e-02, -5.19248992e-02, -2.19188482e-02, -1.82508994e-02, -6.17397487e-01, -3.09374705e-02, -5.18258326e-02, -2.61735450e-02, -3.89497764e-02, -1.46954479e-02, -1.18027776e-02, -2.21383367e-02, -3.07989083e-02, -6.69780612e-01, -1.84117928e-02, -4.12670113e-02, -8.98960326e-03, -3.24395895e-02, -1.61407497e-02, -4.12646681e-02, -5.37053635e-03, -3.38159055e-02, -3.29089276e-02, -1.68235525e-02, -3.26590054e-02, -2.52911523e-02, -4.61294129e-02, -5.67609370e-01, -2.84224618e-02, -4.13801000e-02, -3.34288217e-02, -3.53450403e-02, -3.29895802e-02, -4.02423888e-02, -2.35914946e-01, -4.60832685e-01, -3.70765813e-02, -2.73666903e-02, -2.54514460e-02, -1.44379437e-02, -4.47799824e-02, -4.39045168e-02, -4.60769348e-02, -1.07184583e-02, -2.94511896e-02, -3.48258428e-02, -4.94903252e-02, -4.71179605e-01, -3.04534864e-02, -2.14969758e-02, -2.71185916e-02, -2.13122219e-02, -2.35180482e-02, -3.83645929e-02, -2.21622270e-02, -2.68899292e-01, -3.47404443e-02, -3.07555981e-02, -2.66567357e-02, -2.49998886e-02, -4.09555882e-01, -3.07542346e-02, -1.73584837e-02, -3.32305208e-02, -3.31670046e-02, -3.28688100e-02, -2.02093106e-02, -3.04487552e-02, -4.77275476e-02, -4.22687948e-01, -4.02031802e-02, -2.18092110e-02, -3.58968377e-02, -2.14504618e-02, -3.14717107e-02, -6.16603434e-01, -3.43027376e-02, -4.16996516e-02, -2.88160015e-02, -3.17607224e-02, -4.91070986e-01, -4.11182418e-02, -3.82331535e-02, -1.99981369e-02, -3.31944197e-01, -1.49236154e-02, -4.61634010e-01, -4.73840237e-02, -1.24957161e-02, -3.81090641e-01, -1.33936629e-01, -4.61227804e-01, -3.94086614e-02, -5.02103120e-02, -1.19854463e-02, -2.26863697e-02, -3.57204154e-02, -2.89868433e-02, -3.16134878e-02, -2.95881126e-02, -3.33401635e-02, -2.84984969e-02, -3.39026898e-02, -4.84661758e-01, -3.00496016e-02, -2.10581459e-02, -3.03259753e-02, -2.73086485e-02, -2.92658973e-02, -3.28680202e-02, -4.29259539e-02, -1.15742823e-02, -3.06628253e-02, -3.49430703e-02, -4.13970686e-02, -4.64555889e-01, -2.44929623e-02, -2.32402366e-02, -4.51661311e-02, -3.42191793e-02, -1.04636485e-02, -2.76410598e-02, -2.72743534e-02, -3.71849462e-02, -2.61768941e-02, -2.64907163e-02, -4.13466580e-02, -2.31192280e-02, -2.66561229e-02, -2.44059507e-02, -2.19917297e-02, -3.10962517e-02, -3.92389782e-02, -2.89020780e-02, -4.72296998e-02, -1.47718657e-02, -4.71781231e-02, -3.74658257e-02, -3.74986231e-02, -3.42181958e-02, -3.34076025e-02, -1.90843388e-01, -3.28308083e-02, -1.35448873e-02, -3.84903140e-02, -3.14540192e-02, 9.85380495e-04, -2.98315007e-02, -4.16581221e-02, -2.44194381e-02, -2.18599793e-02, -4.54461351e-02, -4.33417000e-02, -2.37555113e-02, -2.65070703e-02, -3.74873281e-02, -2.00963188e-02, -5.27748942e-01, -1.71016864e-02, -2.98697986e-02, -3.35949808e-02, -4.21402045e-02, -4.34817523e-02, -3.61697711e-02, -3.83502319e-02, -2.63728369e-02, -9.91544034e-03, -4.13909107e-02, -3.27453129e-02, -1.61904916e-02, -2.92313341e-02, -1.82820782e-02, -3.27837802e-02, -4.96267676e-02, -2.13364977e-02, -3.07123903e-02, -1.29112750e-02, -3.05922143e-02, -1.34589160e-02, -1.97699457e-01, -1.26856491e-02, -3.33952643e-02, 3.23457294e-03, -2.03445368e-02, -2.09414233e-02, -1.26234703e-02, -3.02408133e-02, -2.34196670e-02, -1.23696076e-02, -2.40369998e-02, -2.81809513e-02, -2.13961061e-02, -5.47036016e-03, -4.53693084e-02, -3.31229195e-02, -1.50107536e-02, -2.91419178e-02, -2.07277276e-02, -5.49518093e-02, -2.95787137e-02, -2.38886494e-02, -2.65640840e-02, -2.46801712e-02, -3.84779572e-02, -1.79381035e-02, -2.02893633e-02, -3.30808014e-02, -9.46734101e-03, -3.95300575e-02, -3.69772725e-02, -3.11741419e-02, -4.37073298e-02, -2.90284753e-02, -3.63519609e-01, -1.85389761e-02, -1.82310361e-02, -1.51738618e-02, -1.89224500e-02, -2.03151274e-02, -3.81886885e-02, -2.58216038e-02, -3.81475925e-01, -2.79287975e-02, -1.52051644e-02, -2.38528401e-02, -4.04975682e-01, -3.95358130e-02, -4.14295375e-01, -4.38631624e-02, -4.29423749e-02, -3.17317769e-02, -3.27443145e-02, -2.63639372e-02, -3.31052691e-02, 4.43929108e-03, -3.27373035e-02, -2.39749160e-02, -3.15703414e-02, -5.37155151e-01, -8.51269998e-03, -3.57643776e-02, -3.98143642e-02, -3.45447585e-02, -2.49514170e-02, -2.87994146e-01, -5.97163796e-01, -3.36527936e-02, -3.88488770e-02, -3.22511718e-02, -4.89461757e-02, -3.97258699e-02, -2.26812717e-02, -7.74491072e-01, -5.23366332e-01, -4.26757559e-02, -3.48140113e-02, -2.94348393e-02, -3.54919247e-02, -2.96318009e-02, -1.08983302e-02, -3.76659594e-02, -1.47500113e-02, -2.93985605e-02, -2.53436007e-02, 5.79031697e-03, -2.69875340e-02, -2.84397658e-02, -3.33693698e-02, -4.03800532e-02, -2.46412121e-02, -2.71790083e-02, -6.01157732e-02, -3.36032584e-02, -3.37207466e-02, -1.18618924e-02, -4.47471261e-01, -4.70100343e-02, -1.94270499e-02, -1.41834235e-02, -4.12988141e-02, -4.31584977e-02, -2.58529224e-02, -3.11732627e-02, -3.12253926e-02, -4.00089920e-02, -2.82256324e-02, -1.63722355e-02, -2.23478712e-02, -3.37487720e-02, -1.17873568e-02, -3.71523947e-02, -3.02173067e-02, -2.47102734e-02, -1.75078567e-02, -8.67900550e-01, -4.19426337e-02, -2.89364811e-02, -3.62409726e-02, -1.40926614e-02, -3.56511444e-01, -3.45515162e-02, -5.06763607e-02, -2.30538756e-01, -3.77509855e-02, -3.81864123e-02, -3.19356658e-02, -2.84604561e-02, -4.72619906e-02, -1.65800638e-02, -4.46446165e-02, -1.26604997e-02, -3.79219428e-02, -3.04708660e-01, -3.42885070e-02, -4.41885963e-02, -3.81699838e-02, -3.03119402e-02, -2.27376912e-02, -1.89236663e-02, -4.71340455e-02, -1.56659111e-02, -3.82196195e-02, -3.61527875e-02, -2.05292553e-02, -3.56830619e-02, 3.07818269e-03, -7.64196873e-01, -2.73544770e-02, -5.15536249e-01, -1.76008549e-02, -6.18065059e-01, -3.16055007e-02, -4.54795063e-02, -2.52228677e-02, -1.78863723e-02, -3.71731035e-02, -3.15613933e-02, -3.33131254e-02, -1.16747636e-02, -1.52025633e-02, -3.18961823e-03, -2.85964645e-02, -3.42308469e-02, -6.01763930e-03, -1.44698163e-02, -3.33204381e-02, -1.46625296e-03, -1.37048895e-02, -2.66424436e-02, -4.91908789e-02, -3.08587961e-02, -2.29315758e-02, -3.17254931e-01, -1.58655606e-02, -8.84249713e-03, -1.90528110e-02, -4.16713282e-02, -3.05165984e-02, -3.23779024e-02, 1.71483532e-02, -1.71599649e-02, -1.27734728e-02, -3.09068114e-02, -3.11801564e-02, -3.09810303e-02, -3.29174399e-02, -4.35743593e-02, -3.38067338e-02, -2.26877723e-02, -3.14960629e-02, -2.18670722e-02, -2.10534912e-02, -2.37145834e-02, -3.18468064e-02, -3.17289941e-02, -3.52053195e-02, -3.62415873e-02, -3.78505550e-02, -8.54883492e-01, -6.25598431e-01, -2.91614741e-01, -4.12686355e-02, -3.40597779e-02, -4.49921228e-02, -1.83722675e-01, -6.20228611e-03, -9.09710955e-03, -4.55179624e-02, -4.24469262e-02, -4.34849598e-02, -1.18993660e-02, -2.96284892e-02, -1.82864573e-02, -1.58976540e-02, -4.17148732e-02, -1.04806479e-02, -2.51498409e-02, -3.05920038e-02, -2.39100829e-02, -2.05667838e-02, -4.11339812e-02, -2.58969050e-02, -2.54813936e-02, -3.46504189e-02, -3.05222347e-02, -3.66321839e-02, -1.69235300e-02, -3.10963802e-02, -2.00775322e-02, -3.57228443e-02, -4.01586331e-02, -2.64529884e-02, -4.16483805e-02, -4.06530350e-02, -2.00617034e-02, -4.20974828e-02, -3.35381776e-02, -4.97709587e-02, -1.51588218e-02, -4.78560105e-02, -1.61089674e-02, -2.51727998e-02, -4.55499999e-02, -3.32853608e-02, -9.99737252e-03, -9.59883165e-03, -3.51412706e-02, -3.77242006e-02, -1.95016433e-02, -8.43567073e-01, -1.19444542e-02, -1.49743442e-04, -3.27145346e-02, -3.28630805e-02, -1.98399983e-02, -1.64248962e-02, -3.85820344e-02, -1.64614365e-01, -1.02842608e-02, -1.50990197e-02, -3.22756879e-02, -1.80973206e-02, -2.27656811e-02, -4.72140424e-02, -3.06061078e-02, -5.52861243e-02, -3.98167334e-02, -3.09152249e-02, -3.28242555e-02, -3.62482332e-02, -2.84483526e-02, -7.59550452e-01, -4.58953902e-02, -2.13591140e-02, -2.78457664e-02, -1.19789401e-02, -2.98056193e-02, -3.15071791e-02, 5.63531043e-03, -2.39895564e-02, -3.00265819e-01, -4.11561355e-02, -2.46453267e-02, -4.19642702e-02, -5.44620812e-01, -3.15308571e-02, -3.95948440e-02, -2.19081920e-02, -2.35356446e-02, -1.36993527e-02, -1.34584429e-02, -5.07402495e-02, -3.16087119e-02, -2.87754349e-02, -5.01885235e-01, -4.85699102e-02, -4.50458750e-02, -2.57593673e-02, -2.32719798e-02, -3.24436370e-03, -2.84067634e-02, -2.62353010e-02, -3.73448022e-02], dtype=float32), kernel: Array([[ 0.03366351, -0.01376447, 0.00334523, ..., -0.01204627, 0.0351628 , 0.0058507 ], [ 0.01479752, -0.00059022, 0.02888848, ..., 0.01372632, 0.05084606, -0.00852082], [-0.08748913, 0.06752201, -0.02948284, ..., -0.04957195, -0.01602156, -0.0564839 ], ..., [ 0.03158367, -0.0024418 , -0.01248567, ..., 0.00120247, -0.03067757, 0.05438066], [ 0.00710372, -0.07475934, -0.03510208, ..., -0.00362264, -0.01655151, 0.0695975 ], [ 0.03609428, -0.04006155, -0.0374678 , ..., -0.02961577, -0.03186451, 0.05374904]], dtype=float32), }, output: { bias: Array([-1.57430485e-01, 2.07714707e-01, -1.93849746e-02, -2.16451492e-02, -2.94376239e-02, -1.40355468e-01, 1.01594821e-01, 9.77267139e-03, 3.38900611e-02, 3.08969438e-01, -4.50306423e-02, 4.16773744e-02, -5.64103350e-02, 4.65862826e-02, -1.98888164e-02, 5.25933765e-02, 5.03250724e-03, -9.93339792e-02, -9.45335254e-02, 1.34056509e-01, -2.08574124e-02, -2.76607983e-02, -2.45273374e-02, -2.30302606e-02, 4.72312383e-02, 6.59246966e-02, -6.18701801e-02, -2.54601543e-03, 8.12924057e-02, 4.04679216e-02, 7.19875395e-02, -8.80277827e-02, 1.21765286e-01, -1.06022269e-01, 8.69699791e-02, -7.91017562e-02, -7.66793415e-02, -9.08406228e-02, -1.50119308e-02, -1.08254775e-01, 1.43383909e-02, 1.25184851e-02, -8.86707306e-01, 3.01385261e-02, -1.39143646e-01, 1.25063658e-01, 5.04919551e-02, 2.49843255e-01, 5.21366931e-02, -3.97394188e-02, -1.14855161e-02, -1.06901973e-02, -2.97776181e-02, 1.87453896e-01, 8.37227628e-02, -1.05806980e-02, -2.71216761e-02, -8.99265856e-02, 2.36086398e-01, 1.00231051e-01, 1.18218437e-01, -1.38503894e-01, -4.76376042e-02, -2.26873197e-02, -9.81818512e-03, -6.29501976e-03, -1.04944162e-01, -4.35994677e-02, 1.57965776e-02, 8.73260945e-02, 8.73774476e-03, 5.19699082e-02, 7.45332241e-02, -8.56231526e-03, 7.00146556e-01, 9.85437538e-03, -9.72239524e-02, 3.94646786e-02, -9.50414538e-02, -3.36449943e-04, -4.73328263e-01, 4.33811136e-02, 1.06056370e-01, -4.22528051e-02, 2.30107084e-02, 1.08815312e-01, 1.21224724e-01, 1.74949020e-01, -2.51618456e-02, -2.49235518e-02, 2.24952735e-02, 4.77656983e-02, -1.08213760e-01, -1.28404409e-01, -3.41621116e-02, 3.01731993e-02, -7.27362260e-02, -1.10188257e-02, -2.82108001e-02, 1.34670347e-01, 5.28050587e-02, 5.32444753e-02, 3.20014991e-02, 5.04108816e-02, -3.19805578e-04, -1.66307837e-02, 3.41970064e-02, -6.60975128e-02, -1.23339623e-01, -9.24760997e-02, -8.67600888e-02, 8.86741187e-03, 2.36912012e-01, -1.58255234e-01, -1.41610643e-02, 8.07640478e-02, 2.81964112e-02, -8.16835240e-02, -1.64333303e-02, -2.98798494e-02, -7.30156377e-02, 2.87766512e-02, 1.21695735e-01, -5.90394363e-02, 2.60551348e-02, 5.38477339e-02, -2.07343251e-02, -6.99748248e-02], dtype=float32), kernel: Array([[ 0.02169053, 0.02691332, -0.00372989, ..., -0.00424074, 0.03661725, 0.08953198], [-0.01738822, 0.01985764, 0.03393567, ..., -0.00211225, 0.01224655, 0.03731351], [ 0.00689401, 0.02467112, 0.01469113, ..., -0.01584834, -0.03345798, 0.01720266], ..., [ 0.01760239, 0.00760174, 0.01594303, ..., 0.00213138, 0.00272581, -0.03686259], [ 0.01555281, -0.08100978, 0.04178749, ..., -0.0257984 , -0.04736535, -0.03686064], [-0.01932741, 0.05431129, -0.04024049, ..., -0.00516084, -0.01021444, 0.02485615]], dtype=float32), }, }, }, }, 5: { attention: { attention: { key: { bias: Array([ 0.15380459, -0.23325492, 0.5298237 , -0.25394794, -0.1792779 , 0.2635233 , -0.2518766 , 0.51324993, 0.44011664, -0.15511836, 0.1065289 , -0.18103923, -0.24789087, 0.01313723, -0.00924131, 0.2846446 , -0.02708824, 0.01480816, 0.12186062, -0.18671922, 0.47530514, -0.44773427, -0.42629737, -0.25819948, 0.15633082, -0.48324105, -0.2818154 , -0.61912835, -0.29742548, 0.28758916, 0.07666907, 0.27997455, -0.5159455 , -0.00791557, 0.40676025, -0.13460469, 0.5236954 , -0.5017536 , 0.5868691 , 0.01062866, -0.10968728, -0.03771151, -0.33758804, -0.53288084, -0.6708519 , 0.60964614, 0.14677066, -0.52277976, -0.00805334, -0.3969676 , 0.5209981 , 0.34777972, 0.5903865 , -0.7917442 , -0.64988923, -0.63839513, -0.74101704, 0.11602865, 0.8335415 , 0.2829763 , -0.41182905, -0.3442916 , -0.47213373, 0.5067961 , 0.05412646, 0.11681292, 0.26332203, -0.25662488, 0.4626192 , -0.2888915 , 0.3559262 , 0.43773538, -0.4578352 , 0.32878822, -0.2354647 , -0.39989737, -0.21824938, 0.23775657, 0.28870043, -0.35698473, 0.2748483 , 0.05920567, -0.09002119, -0.35282713, 0.20067027, -0.54461175, -0.23671876, -0.26704383, -0.06359763, 0.01624733, -0.20986806, -0.0469152 , -0.21808156, 0.10429718, 0.28616387, -0.0075318 , 0.41809165, 0.21852025, -0.68503106, 0.15126543, -0.5385752 , -0.6387902 , -0.09934053, 1.1725092 , 0.9802906 , -0.24173768, -0.3697774 , 0.30808112, 1.2169337 , -0.4494893 , 0.80163956, 0.62732154, -0.03264387, 0.67577094, -0.39557546, -0.40416336, 0.0219003 , 0.2829875 , -0.28239524, 0.14352222, -0.8248839 , 0.02846357, 0.13515349, -0.19850218, -0.22375287, 0.2135571 , -0.27206707, 0.76276755], dtype=float32), kernel: Array([[ 0.26200238, 0.13167961, -0.11595184, ..., 0.0414915 , 0.11238165, -0.30494452], [ 0.22360708, 0.08118725, 0.02069189, ..., 0.00547806, -0.10924851, 0.26473263], [-0.16265303, 0.17587142, 0.12479865, ..., -0.07595218, 0.1396125 , -0.19043295], ..., [-0.26119086, -0.38923773, 0.35491627, ..., -0.7368683 , -0.49823397, -0.6397334 ], [ 0.13432133, -0.16010363, 0.22101043, ..., -0.06990113, 0.25519368, 0.09977636], [-0.12494502, -0.38168752, 0.16611753, ..., 0.36234066, -0.0669388 , 0.3607124 ]], dtype=float32), }, query: { bias: Array([-0.18479703, -0.35146004, 0.7537319 , -0.17177609, 0.01369478, 0.18720719, -0.17976424, 0.15347035, 0.08994126, 0.22886601, 0.18279251, -0.35825828, -0.06468849, -0.12611765, -0.028998 , 0.9971776 , -0.18668026, -0.04904906, -0.14856422, -0.24965602, 0.05141901, -1.1571982 , 0.05819758, 0.14917302, 0.15365446, -0.85294926, -0.08199202, -0.9097335 , 0.15537313, -0.26021054, 0.12598538, -0.06024593, -0.11726313, -0.20979708, 0.20502815, 0.22255987, 0.30414093, -0.11271469, 0.4672999 , 0.00509386, -0.13042028, 0.22407965, -0.10005572, -0.14989175, -0.4365904 , 0.2107629 , 0.18609835, -0.72566986, 0.18099388, -0.30312923, 0.5998623 , -0.15350953, 0.36957732, -0.70576656, -0.68291605, -0.420833 , -0.34906906, 0.06481054, 0.58362794, 0.31369215, -0.82122093, -0.25893688, -0.24382648, 0.19694354, 0.16259049, -0.11133604, -0.0934663 , -0.02774331, 0.21941422, -0.01251643, 0.12317675, 0.6149345 , 0.14704953, 0.37044725, -0.28755513, 0.1573624 , -0.44036826, 0.1894717 , 0.12188441, -0.49551368, 0.14024805, -0.03632037, -0.11464448, -0.6433361 , 0.15439317, -0.32633895, 0.00914915, -0.05114478, -0.1278209 , 0.33733654, -0.07761955, -0.3304572 , -0.59981114, 0.14654088, 0.06847122, -0.259691 , 0.25001425, 0.05990116, -0.645737 , 0.02534364, -0.6641346 , -0.17594564, -0.01261867, 0.5100829 , 0.06452097, -0.5573339 , 0.13375577, 0.25303102, 0.43233076, 0.15803403, 0.254334 , 0.03359455, -0.05677273, -0.11994728, -0.2779966 , -0.23615459, -0.04638248, -0.03050251, 0.3847538 , 0.0142238 , -0.26559195, -0.03787645, 0.02767896, -0.08072381, -0.01213382, 0.15982705, -0.07372479, 0.6117539 ], dtype=float32), kernel: Array([[-0.21102534, 0.13572834, 0.08487151, ..., -0.37551054, -0.08974288, -0.14673388], [-0.42533755, 0.19477351, -0.05581117, ..., -0.19027376, 0.11326433, 0.280313 ], [ 0.20709127, -0.02206374, 0.00242551, ..., -0.21323436, -0.1782914 , -0.49215955], ..., [-0.35879493, -0.13843112, 0.20357771, ..., -0.3228759 , -0.48138356, -0.60722345], [ 0.409117 , 0.09512895, -0.07367483, ..., -0.79642665, -0.3589279 , 0.36061507], [-0.1480074 , 0.24665533, -0.10206775, ..., 0.03880555, -0.12229352, 0.42183685]], dtype=float32), }, value: { bias: Array([-0.06382484, -0.10177656, -0.01783326, 0.07238453, 0.0269923 , 0.03472366, 0.00803764, 0.09564783, -0.02505345, 0.16271482, 0.01969427, -0.08568954, -0.02066216, -0.03939519, -0.09977195, -0.01792192, 0.02864774, 0.03915608, 0.02681843, 0.07789411, -0.12741898, 0.03238543, 0.01603028, -0.11089758, -0.08953978, 0.11356429, -0.03502819, -0.0710902 , -0.07928857, -0.07034726, -0.00949693, -0.11724807, 0.03667774, -0.0831453 , -0.08759701, -0.11416655, 0.05479722, 0.02118164, 0.0808093 , -0.04038942, -0.05000678, -0.06270798, -0.00748655, -0.0456011 , 0.09870113, 0.03644619, 0.06150641, 0.15579787, 0.05361883, 0.07626446, -0.02815951, 0.042865 , 0.06312688, 0.07170736, 0.185382 , 0.06517194, 0.0584194 , -0.07202493, -0.09540485, -0.08599403, -0.03540141, -0.10775873, 0.03609048, -0.06903887, -0.10958587, -0.09116477, -0.10746475, -0.05029682, 0.07927579, 0.10405283, 0.12020227, 0.03288461, -0.1196027 , 0.03575619, 0.0972458 , 0.05244093, 0.08934712, 0.12486867, 0.08823898, 0.0983156 , 0.10495333, -0.10362058, 0.09903006, 0.09088268, -0.12305731, 0.10125972, -0.08239742, -0.00916524, -0.02156379, -0.09864786, 0.10193784, -0.09551246, -0.05185878, -0.09967213, -0.10428255, 0.10507633, -0.14955677, -0.12777895, -0.07334802, 0.16633663, -0.04327126, 0.11767196, 0.08965011, 0.17071536, -0.0584787 , -0.09368757, -0.16054668, -0.16359754, 0.0472418 , 0.1223535 , -0.07023636, 0.24043013, 0.15963615, -0.01664593, 0.15456486, 0.11985014, 0.25587445, 0.17214614, -0.15644138, 0.02051393, -0.15587153, 0.03782795, 0.0117235 , 0.04015777, 0.17542166, -0.1388207 , 0.09689201, -0.11741312], dtype=float32), kernel: Array([[-0.06625638, -0.10676572, -0.11858384, ..., 0.12592293, 0.19599795, -0.10273615], [ 0.19458613, -0.08272579, -0.01620658, ..., 0.07205058, 0.00837682, 0.03907323], [ 0.17456186, -0.00289036, -0.24084125, ..., -0.03393131, -0.21872245, 0.21209157], ..., [-0.10240208, -0.01977782, -0.14529926, ..., 0.48570195, 0.29290506, -0.28927922], [ 0.5828709 , 0.14644368, 0.23230499, ..., -0.51522744, -0.00893875, 0.11504804], [-0.18426676, -0.12852795, -0.02424787, ..., -0.17730054, -0.39859936, 0.3791855 ]], dtype=float32), }, }, output: { dense: { bias: Array([-1.47393912e-01, 1.51925683e-01, -1.37644792e-02, -5.88501468e-02, -2.74660420e-02, 2.99488921e-02, 6.36566132e-02, -3.54291759e-02, -7.69458618e-03, 3.24805588e-01, -5.66644669e-02, 2.47724857e-02, -1.12583041e-02, -9.56679955e-02, 1.36414379e-01, 3.80160026e-02, 4.16892134e-02, -1.21877357e-01, -9.89358947e-02, 8.71492922e-02, -5.38453227e-04, 6.59572333e-02, -4.53343103e-03, -4.99130785e-02, 4.46245968e-02, 1.36791214e-01, -3.86863551e-03, 1.96640324e-02, 6.48279041e-02, 6.96821213e-02, -2.48288596e-03, -8.72585997e-02, 2.33484972e-02, -5.56293987e-02, 6.42355978e-02, -3.93086448e-02, -5.08882850e-03, -8.52516666e-02, 1.37223052e-02, -1.85753062e-01, 6.52511492e-02, -4.16962616e-03, -6.68387711e-01, -2.01383736e-02, -1.15884997e-01, 1.01936243e-01, 4.85421829e-02, 2.81445622e-01, 9.36387926e-02, -4.22131829e-02, -2.58780066e-02, -9.31250453e-02, -8.50194786e-03, 1.86161455e-02, 1.08278200e-01, -1.33628231e-02, -2.95319203e-02, -7.03121051e-02, 1.65721059e-01, 1.23157077e-01, 1.26239538e-01, 8.69110152e-02, -6.91822842e-02, -6.16661599e-03, -4.39871997e-02, 6.22018203e-02, -9.05643180e-02, 7.91532993e-02, -6.81676865e-02, -5.36848418e-02, 2.38579530e-02, 2.05037221e-02, 1.06868640e-01, -3.20243724e-02, 4.63792652e-01, 2.49073096e-02, -1.17389895e-01, 3.09424363e-02, -7.55871981e-02, -5.58793545e-02, -3.00298661e-01, 3.98461558e-02, -5.11545166e-02, -3.12989019e-02, 6.69162199e-02, 7.48850405e-03, 8.32765177e-02, 1.84600934e-01, 3.32334414e-02, 4.34078313e-02, 3.89995314e-02, 1.41597819e-02, -9.58840549e-02, 1.53720230e-02, -5.27216345e-02, 6.26715794e-02, -1.02486372e-01, -5.28356284e-02, -1.23486230e-02, 9.72601771e-02, -4.58076522e-02, 1.89271364e-02, 6.57725483e-02, 7.59906247e-02, -7.66134709e-02, -4.67620529e-02, 6.40749931e-02, 2.11690012e-02, -8.14582258e-02, -7.82966167e-02, 3.28965578e-03, -1.84342973e-02, 1.21189028e-01, -2.12921515e-01, -4.28700894e-02, -5.90704009e-02, 1.14371911e-01, -1.21928066e-01, -3.76502164e-02, -3.38824801e-02, -6.17372356e-02, -1.77653898e-02, 1.73333704e-01, 8.95729363e-02, -6.20543491e-03, 2.95204688e-02, 2.87007019e-02, -4.14825082e-02], dtype=float32), kernel: Array([[ 0.02049902, -0.1422533 , -0.4065625 , ..., 0.36780554, 0.4333429 , 0.19315735], [-0.03959692, 0.25549796, -0.06924722, ..., -0.55895936, -0.27878842, 0.00849131], [ 0.14230408, 0.490068 , -0.18883815, ..., -0.30614755, -0.30285606, 0.07531235], ..., [ 0.0692085 , 0.07044961, 0.61252284, ..., -0.23293641, -0.15725587, -0.13308391], [ 0.19255584, -0.04722506, 0.23870187, ..., 0.26104337, -0.1339734 , -0.28957063], [-0.11599459, 0.07947025, -0.35309428, ..., -0.22296615, 0.19523388, 0.4148677 ]], dtype=float32), }, }, }, layernorm_after: { bias: Array([-1.27752021e-01, -1.48293898e-01, -1.61956966e-01, 2.83768959e-03, -1.43960312e-01, 4.62501980e-02, 1.55017406e-01, -1.05546750e-02, 1.24534264e-01, 3.37903023e-01, -4.59039863e-03, -1.85061380e-01, 1.73028678e-01, 2.01043487e-02, 2.81281001e-03, 9.39656422e-02, -1.36595475e-03, -2.31539598e-03, -1.25028998e-01, 2.32474544e-04, -2.05569100e-02, -1.70720860e-01, 9.77594405e-03, 1.18461205e-02, 2.47885808e-01, -1.36246130e-01, -1.46172762e-01, -6.74006566e-02, 1.35634257e-03, -6.88059255e-02, -1.38106868e-01, -3.34430300e-02, 8.12808890e-03, 9.31725744e-03, 2.75329649e-02, -1.16437320e-02, -1.76476855e-02, -1.26601487e-01, -3.02024838e-02, -5.67115694e-02, 9.60008055e-02, 8.26168135e-02, -5.39703900e-03, 3.32480557e-02, -1.42446086e-01, 2.15325981e-01, 3.59231502e-01, 6.70243725e-02, 5.95594849e-03, 2.55793538e-02, 1.20438235e-02, -1.38192281e-01, -9.54014882e-02, -6.51674941e-02, 1.10238351e-01, -9.56654698e-02, 9.43305194e-02, -7.60812163e-02, -7.58638158e-02, 1.66560858e-01, 1.86092202e-02, 1.56081989e-01, 5.11778006e-03, -4.85944413e-02, 8.28391612e-02, 5.49042262e-02, -3.91565077e-02, -2.07471265e-03, 3.06699127e-02, 3.01516578e-02, -3.39646004e-02, -2.84999385e-02, 7.43485242e-02, 1.12809241e-02, -1.99580923e-01, -5.58756031e-02, 2.26514554e-03, -5.92570044e-02, -5.33170663e-02, 4.89857085e-02, 9.51478072e-03, -2.19953805e-02, -3.24405394e-02, -1.16561711e-01, -7.09283957e-03, -4.68143495e-03, 5.30386604e-02, -1.35984283e-03, 7.81681612e-02, -5.89879602e-03, -1.73093099e-03, -6.07467350e-03, -3.00015986e-01, -3.05419937e-02, -1.54022276e-02, -6.91811442e-02, -5.47736287e-02, 2.07600649e-02, 1.68445095e-01, -1.03444420e-02, -1.04541652e-01, 5.39848953e-02, -2.24056616e-01, 1.65632993e-01, -1.40168756e-01, -4.03944461e-04, -4.72264551e-02, 1.25392643e-03, 6.72694370e-02, -2.77178794e-01, -4.07134034e-02, 8.86196047e-02, 4.30512354e-02, -1.80928916e-01, -2.78730467e-02, 5.36768697e-02, 2.03331217e-01, -2.78739282e-03, 1.01259798e-01, 6.90696239e-02, -9.43698585e-02, 7.54763791e-03, 1.15448259e-01, 2.77357996e-01, 1.10673435e-01, 1.47797644e-01, -9.77296499e-04, -1.15336455e-01], dtype=float32), scale: Array([ 1.0143379 , 0.56084377, 0.61511797, 0.875825 , 0.7977206 , 0.86066914, 0.4466555 , 0.5817147 , 0.8288987 , 1.2778934 , 0.14663759, 0.92707175, 0.29109854, -0.5407728 , -0.00583301, 0.80144596, 0.39370012, -0.01780795, 0.55510044, 0.8787668 , 0.8118457 , 0.7212926 , 0.22898938, 0.00633279, 1.0344126 , 0.4334817 , 1.2401807 , 0.45261577, 0.04603433, 0.5680823 , 0.6459878 , 0.6229751 , 0.7747696 , 0.87726367, 0.4118688 , 0.02244997, 0.7460163 , 0.53923315, 0.5421808 , 0.90724677, 0.7915582 , 0.3890947 , 0.7099728 , 0.94361395, 0.44176197, 0.708961 , 0.51787186, 1.1014241 , 0.3872233 , 0.6767877 , 0.7677383 , 0.61901575, 0.73735505, 0.28867054, 0.81986 , 0.68635523, 0.40406612, 0.7550665 , 0.3736026 , 0.42338955, 0.51958174, 0.75038564, 0.10604075, 0.47208443, 0.53189516, 0.43309155, 0.37580943, 0.48088777, 0.4441973 , 0.5321363 , 0.5055118 , 0.98682684, 0.7596939 , 0.835741 , 0.10439195, 0.57524043, 0.43323192, 0.71470016, 0.38151345, 0.8561705 , 0.6086962 , 0.8476424 , 0.68529296, 0.81576556, 1.0155932 , 0.2647536 , 0.66374 , 0.8123001 , 0.43478364, 0.7506543 , 0.70008427, 0.01624053, 0.5848578 , 0.672363 , 0.5528674 , 0.17722914, 0.7533496 , 0.50877273, 0.87119836, 0.50191104, 0.70362985, 0.7272638 , 0.79176766, 1.0370609 , 0.6759593 , -0.02001736, 0.8079056 , 0.76534927, 0.6989189 , 0.55686975, 0.8860217 , 0.43745935, 0.6700682 , 0.90106094, 0.05214773, 0.5131126 , 0.7131446 , 0.76999676, 0.6244328 , 0.87544256, 0.54827476, 0.00163172, 0.72723454, 0.770018 , 0.62415475, 0.49478632, 0.32370028, 0.52392954], dtype=float32), }, layernorm_before: { bias: Array([ 5.46789803e-02, -1.76678717e-01, -3.22135426e-02, -1.92919653e-02, -2.71175429e-03, -5.60209937e-02, 3.75961289e-02, 2.42887228e-03, 2.20702612e-04, -1.47421882e-01, 1.25962566e-03, -9.49192513e-03, 3.36418934e-02, 1.43754203e-02, 2.79071257e-02, 3.28558236e-02, 1.25313839e-02, 5.57482056e-03, 2.42552720e-03, -4.17415686e-02, -6.24468969e-03, -7.01998994e-02, 6.89587668e-02, -7.22786319e-03, 1.22491186e-02, -1.73403710e-01, 4.65809219e-02, -2.40579285e-02, 4.88756225e-04, 1.74035388e-03, 6.00601882e-02, -2.83498946e-03, -4.87266015e-03, -6.68269256e-03, 2.96081766e-03, -4.06322069e-02, -5.13824681e-03, -5.18148541e-02, -6.03306864e-04, 3.28535922e-02, 5.79561479e-03, -5.60164126e-03, 2.76499480e-01, -5.52190188e-03, 1.08755060e-01, 2.78750118e-02, -1.74138229e-03, -4.13134582e-02, -1.12804314e-02, 1.25732794e-02, -2.00521722e-02, 3.98424082e-02, 1.83249754e-03, -1.12617195e-01, 8.08609067e-04, 6.40430721e-03, 4.37817909e-03, 2.31746268e-02, -2.95173544e-02, 5.21452923e-04, 5.27834482e-02, 4.03346457e-02, -6.75705774e-03, -1.17142848e-03, 2.27539521e-02, 8.85420591e-02, -6.18154998e-04, 2.34325184e-03, 2.68001352e-02, 3.51031497e-03, -3.19339558e-02, 1.70021728e-02, 5.05615212e-02, -3.64386290e-02, -5.41854799e-01, -3.14815044e-02, -8.37918895e-04, -5.96139133e-02, -5.71792945e-03, 2.48509813e-02, 5.07591176e-04, -9.80351865e-03, -4.27319445e-02, -4.39409167e-02, -2.25920565e-02, 5.03404811e-03, 5.27535053e-03, -2.32479889e-02, 8.92616529e-03, 2.46615335e-02, -1.04511445e-02, 4.72732633e-02, 2.06987560e-03, -4.62427586e-02, 2.54220869e-02, -2.62153684e-03, -1.93837471e-02, 3.46671464e-03, 9.97190922e-02, 5.05104400e-02, -1.72052216e-02, 5.25105570e-04, -1.76775366e-01, 3.32803018e-02, -2.44557224e-02, -1.54652279e-02, -1.59627572e-02, 3.77600156e-02, -2.54226532e-02, -1.87684055e-02, -8.18012469e-03, 6.22503310e-02, -9.38802143e-04, 4.75377999e-02, -9.10606887e-03, 2.95884646e-02, 1.39154634e-02, -4.97860415e-03, 5.38583286e-03, 2.14410182e-02, -7.47090429e-02, 2.04133317e-02, 1.69800371e-02, 5.70605770e-02, -5.42162033e-03, -4.40403894e-02, 3.08437683e-02, -1.20276608e-01], dtype=float32), scale: Array([ 3.09920963e-02, 1.43276036e-01, 1.49911180e-01, 2.15380549e-01, 8.57652703e-05, 2.60982364e-01, 3.85197014e-01, -2.69828960e-02, -3.57246934e-03, 1.77718669e-01, 1.78545862e-02, 2.83004701e-01, 4.92948219e-02, 2.25244924e-01, 1.80071101e-01, 3.60851258e-01, 2.13657439e-01, 6.50591101e-04, -1.27726868e-02, 8.39237347e-02, 1.44251525e-01, 3.37201595e-01, 2.32133791e-01, 3.55153927e-03, 1.30111605e-01, 3.19995940e-01, 1.58201158e-01, 1.93055466e-01, 1.13361664e-02, 1.06982812e-02, 2.68632799e-01, 1.73866183e-01, 2.49319479e-01, 2.33700909e-02, -3.93300736e-03, 1.54559046e-01, 7.23777190e-02, 2.99003601e-01, 2.09477082e-01, 1.43370494e-01, 2.27352500e-01, 2.16101669e-02, 1.23486273e-01, 3.68611127e-01, 2.57967263e-01, 3.01627338e-01, 2.64851768e-02, 3.02257270e-01, -8.79388452e-02, 1.15935393e-01, 3.20747286e-01, 3.84979308e-01, 2.91317515e-02, 3.35988641e-01, 1.00166025e-03, -2.34178808e-02, 1.65450841e-03, 9.69321802e-02, 1.97463453e-01, 2.78218836e-02, 3.64540905e-01, 4.77716655e-01, -1.46322930e-02, 3.61578852e-01, 2.67611772e-01, 1.56464770e-01, -1.12778105e-01, 3.55959013e-02, 2.19024077e-01, 5.52458549e-03, 2.59881735e-01, 2.20885038e-01, 2.75174618e-01, 2.36324608e-01, 1.31579578e-01, 2.25922078e-01, 2.05405220e-01, 2.81007886e-01, -3.42874392e-03, 1.95928365e-01, 1.01560242e-02, 1.41977131e-01, 2.52693713e-01, 2.46991277e-01, 1.85481176e-01, -2.18331348e-02, -1.25332624e-02, 1.17323473e-01, 6.74787387e-02, 1.07466944e-01, 1.33154258e-01, 1.68752119e-01, -5.74604236e-03, 2.33412385e-01, 1.67403102e-01, -5.18426392e-03, 1.57531217e-01, 1.38286993e-01, 3.68412942e-01, 2.31308982e-01, 1.48976266e-01, 7.62487238e-04, 3.20091337e-01, 1.83057398e-01, 2.31623769e-01, 2.69163251e-01, 1.83478698e-01, 3.07111442e-01, 3.13205153e-01, 2.52505481e-01, 3.30679387e-01, 3.39435637e-01, 9.45753418e-03, 4.02425140e-01, 6.86238613e-03, 2.50461936e-01, 3.17101419e-01, 4.53172848e-02, 6.39474615e-02, 2.83503503e-01, 1.75329089e-01, 1.88262612e-01, 1.82763785e-01, 3.37596089e-01, 3.82400723e-03, 2.11784661e-01, 2.38452464e-01, 3.09545726e-01], dtype=float32), }, moe: { shared_experts: { intermediate: { bias: Array([-3.9847344e-01, -3.6854276e-01, -4.5092517e-01, -4.3007284e-01, -4.8035741e-01, 1.8460776e-03, -4.3567291e-01, -4.2967001e-01, -3.3667952e-01, -4.1538629e-01, -3.6099383e-01, -4.3359479e-01, -4.4380239e-01, -5.5524331e-01, -3.6870196e-01, -2.4764657e-01, -4.7775912e-01, -4.1283187e-01, -3.4256911e-01, -2.0356853e-01, -4.6999124e-01, -5.6414276e-01, -3.3079070e-01, -3.8981372e-01, -4.5676839e-01, -8.1164964e-02, -2.3958890e-01, -4.1569802e-01, -3.3400738e-01, -3.1094922e-02, -4.0657026e-01, -3.2647237e-01, -5.5474406e-01, -5.1273751e-01, -4.6838763e-01, -4.5339519e-01, -3.2953310e-01, -4.2982268e-01, -4.1318712e-01, -4.8618275e-01, -3.3651689e-01, -3.8407576e-01, -3.7777162e-01, -4.4129571e-01, -2.5866541e-01, -4.4932291e-01, -4.2343855e-01, -3.6280292e-01, -3.8582063e-01, -3.7247223e-01, -7.4966386e-02, -3.5159770e-01, -4.0223166e-01, -1.5560038e-01, -3.4897429e-01, -3.5570183e-01, -2.3108204e-01, -3.0199406e-01, -5.0925249e-01, -4.2046854e-01, -5.7922322e-01, -4.1019771e-01, -4.2622930e-01, -4.4688463e-01, -4.6536162e-01, -3.6277786e-01, -2.4544826e-01, -3.0414537e-01, -4.6335497e-01, -3.0622986e-01, -3.6244193e-01, -1.3396668e-01, -4.9729797e-01, -4.0432706e-01, -1.7137310e-01, -1.5809147e-01, -4.3105653e-01, -2.7127942e-01, -3.0011219e-01, -2.2747953e-01, -3.7861067e-01, -2.8296992e-01, -4.9367398e-01, -1.4615911e-01, -3.4952867e-01, -2.4669057e-01, -4.6957275e-01, -3.5128793e-01, -3.9153334e-01, -4.2859459e-01, -4.0044680e-01, -4.3775627e-01, -3.7979439e-01, -5.2443427e-01, -1.5687215e-01, -4.4120738e-01, -4.7335675e-01, -3.9626312e-01, -3.6113608e-01, -8.2696810e-02, -2.6075220e-01, -1.9168627e-01, -4.8281315e-01, -1.7972828e-01, -5.2597886e-01, -2.7033496e-01, -5.4604769e-01, -3.4184673e-01, -4.3216091e-01, -4.4862813e-01, -2.6511189e-01, -4.5229965e-01, -3.8147661e-01, -3.2180479e-01, -3.1602433e-01, -4.3689847e-01, -5.1146489e-01, -3.5368246e-01, -3.7338638e-01, -3.0413544e-01, -2.4527813e-01, -4.7685048e-01, -2.5364643e-01, -3.0931181e-01, -3.2056129e-01, -3.0224264e-01, -3.6615273e-01, -3.6344633e-01, -4.7639742e-02, -4.2077422e-01, -4.8495576e-01, -3.2529086e-01, -3.9314100e-01, -3.9539710e-01, -4.4750559e-01, -2.1847221e-01, -3.4810659e-01, -3.5670453e-01, -4.8384792e-01, -4.6719027e-01, -2.9682928e-01, -4.2099711e-01, -4.7452945e-01, -4.3598908e-01, -4.5502350e-01, -4.7916898e-01, -5.1886410e-01, -2.9352316e-01, -4.6964645e-01, -4.7157702e-01, -2.2555082e-01, -3.9752248e-01, -7.7490084e-02, -1.6522288e-01, -4.9982330e-01, -4.2842680e-01, -3.2544607e-01, -3.0850667e-01, -4.0218577e-01, -4.1208234e-01, -4.0038344e-01, -3.7057555e-01, -2.2373217e-01, -3.4376973e-01, -3.1466734e-01, -5.2123147e-01, -5.3340238e-01, -2.8239700e-01, -3.3951777e-01, -4.2198521e-01, -4.4805121e-01, -2.5464025e-01, -1.5028036e-01, -4.1943482e-01, -4.2202288e-01, -3.9689219e-01, -3.7402752e-01, -1.6487882e-01, -3.0544364e-01, -4.0235436e-01, -4.1456652e-01, -3.5549703e-01, -4.5845604e-01, -3.7517163e-01, -4.1666454e-01, -2.5526059e-01, -4.9071726e-01, -3.1533489e-01, -3.4268734e-01, -4.6997485e-01, -3.2526866e-01, -4.7221246e-01, -3.2835072e-01, -5.4744834e-01, -4.1387323e-01, -2.6110759e-01, -4.3684092e-01, -4.7436625e-01, -3.4099048e-01, -3.7735420e-01, -4.5077652e-01, -5.1907909e-01, -3.7325633e-01, -2.2339475e-01, -2.8594923e-01, -4.6022233e-01, -3.7742871e-01, -3.5255402e-01, -1.6996916e-01, -3.3114195e-01, -2.1370201e-01, -4.9245229e-01, -4.8635155e-01, -4.3522072e-01, -3.8971159e-01, -4.3678322e-01, -4.6781665e-01, -2.9914606e-01, -4.2043447e-01, -2.9230228e-01, -2.6290324e-01, -2.3281787e-01, -5.2061099e-01, -3.1569782e-01, -5.0653201e-01, -4.7192627e-01, -3.8380232e-01, -4.1764271e-01, -3.3932805e-01, -4.6573785e-01, -4.0508986e-01, -1.6529684e-01, -3.9876980e-01, -4.5229152e-01, -2.8774762e-01, -3.5973960e-01, -3.5486862e-01, -2.4070273e-01, -2.9540777e-01, -2.9138458e-01, -3.0258924e-01, -3.0566293e-01, -3.6295635e-01, -2.6400667e-01, -3.0755830e-01, -2.9665276e-01, -3.2570139e-01, -3.8138995e-01, -2.5602007e-01, -3.7541685e-01, -4.3872544e-01, -4.1538125e-01, -4.2229000e-01, -3.4041786e-01, -3.0963835e-01, -4.1050071e-01, -5.6023502e-01, -5.6029701e-01, -3.5283986e-01, -2.6442611e-01, -3.5243306e-01, -3.4913927e-01, -2.1650518e-01, -4.0154454e-01, -3.7275749e-01, -4.7750309e-01, -3.6822703e-01, -4.5743442e-01, -4.8160750e-01, -4.7312003e-01, -3.9876688e-01, -5.0516546e-01, -5.4552037e-01, -3.4587166e-01, -1.6361688e-01, -3.0174541e-01, -3.3792850e-01, -4.7918764e-01, -4.8411125e-01, -4.3018821e-01, -4.6167120e-01, -4.1177577e-01, -4.5708910e-01, -4.5317286e-01, -2.9856035e-01, -3.5552701e-01, -3.3068407e-01, -4.0677980e-01, -4.2047119e-01, -2.2582465e-01, -5.4681146e-01, -3.3787546e-01, -4.4846484e-01, -3.3120909e-01, -3.4642521e-01, -3.9533862e-01, -3.4406123e-01, -4.5375788e-01, -5.0441438e-01, -3.3547118e-01, -2.8920132e-01, -1.0944715e-01, -3.5370487e-01, -3.3076125e-01, -2.8330564e-01, -3.6274710e-01, -2.7184573e-01, -3.5993487e-01, -4.0513000e-01, -3.0320236e-01, -2.5303522e-01, -4.7531182e-01, -4.6395004e-01, -4.2257047e-01, -4.8910224e-01, -3.6540136e-01, 1.3740684e-01, -3.0950975e-01, -4.7773069e-01, -3.0934384e-01, -4.1294369e-01, -4.5992157e-01, -4.8535463e-01, -4.4464013e-01, -3.5180721e-01, -3.6469987e-01, -4.2818370e-01, -5.0174421e-01, -3.8951018e-01, -3.7477994e-01, -3.4967715e-01, -4.3007034e-01, -4.4603419e-01, -3.4457707e-01, -4.6689948e-01, -3.0864301e-01, -4.9847844e-01, -3.7400711e-01, -4.7358903e-01, -3.5766226e-01, -2.8434107e-01, -2.7686146e-01, -3.4817812e-01, -3.6055216e-01, -4.3933406e-01, -5.2891338e-01, -2.7604723e-01, -5.1344776e-01, -2.9326063e-01, -5.0666833e-01, -3.3879602e-01, -4.9309245e-01, -3.0533355e-01, -5.3628761e-01, -4.3473902e-01, -4.1327205e-01, -1.9567779e-01, -4.3977755e-01, -4.7691789e-01, -3.4455177e-01, -4.1720390e-01, -3.4753558e-01, -4.0889883e-01, -4.2685688e-01, -3.7366074e-01, -1.6097990e-01, -3.0323794e-01, -4.8915049e-01, -3.5860199e-01, -3.2890522e-01, -3.6635777e-01, -3.0436921e-01, -4.3726847e-01, -1.7787141e-01, -3.3810359e-01, -4.7570062e-01, -5.1493889e-01, -4.9520645e-01, -4.8514831e-01, -4.0491089e-01, -5.2531552e-01, -4.0086076e-01, -5.0189817e-01, -4.7050786e-01, -2.1616998e-01, -3.5684329e-01, -4.2839581e-01, -3.8793033e-01, -2.3242010e-01, -3.5336223e-01, -3.3516747e-01, -3.7301928e-01, -4.4247723e-01, -4.8835176e-01, -2.0324853e-01, -3.9160007e-01, -3.3982515e-01, -2.5195253e-01, -5.4986537e-01, -3.1654030e-01, -4.6651766e-01, -4.7360784e-01, -2.8256953e-01, -3.3791304e-01, -3.5749507e-01, -3.0531624e-01, -4.9014235e-01, -4.1924736e-01, -2.0831045e-02, -3.5200366e-01, -2.2583617e-01, -4.5401311e-01, -3.8651881e-01, -4.5588291e-01, -4.9305251e-01, -1.7810422e-03, -2.6118660e-01, -3.1825066e-01, -2.4539576e-01, -2.5099796e-01, -5.3763103e-01, -3.6526870e-02, -2.3981811e-01, -3.0151102e-01, -4.7348249e-01, -2.2897437e-01, -4.1141531e-01, -4.8857701e-01, -3.2772517e-01, -5.7252842e-01, -4.5862877e-01, -3.6546475e-01, -3.6844012e-01, -2.0294371e-01, -4.8403230e-01, -4.8141652e-01, -1.4463230e-03, -4.9992949e-01, -3.6349079e-01, -2.1532026e-01, -4.1290241e-01, -4.8122567e-01, -2.8661153e-01, -4.4909769e-01, -3.6796397e-01, -2.6207677e-01, -3.9517501e-01, -4.4621187e-01, -3.6207435e-01, -4.8050162e-01, -3.7443826e-01, -2.3039897e-01, -3.5041386e-01, -3.9968762e-01, -3.6701423e-01, -3.8556710e-01, -3.4414786e-01, -4.8362854e-01, -3.8469809e-01, -4.8367801e-01, -3.7582186e-01, -4.8235339e-01, -3.0144668e-01, -3.7805808e-01, -3.9068311e-01, -4.6765688e-01, -4.5509905e-01, -4.1643125e-01, -4.3739045e-01, -5.5417305e-01, -3.9493406e-01, -4.5592216e-01, -3.2413527e-01, -4.4394735e-01, -2.5142363e-01, -2.9339787e-01, -4.6713012e-01, -2.5261605e-01, -2.6347214e-01, -3.7425596e-01, -2.1422139e-01, -4.4427580e-01, -3.5874227e-01, -4.1653034e-01, -3.3887890e-01, -3.0552921e-01, -2.4492532e-01, -4.0205300e-01, -3.2229105e-01, -4.8967350e-01, -2.1930854e-04, -3.7214684e-01, -3.0033603e-01, -4.7503376e-01, -3.7032735e-01, -3.4375238e-01, -3.8198787e-01, -3.3686790e-01, -5.5172253e-01, -3.0320832e-01, -1.3836212e-01, -3.9621928e-01, -3.8446391e-01, -4.3740526e-01, -3.8315192e-01, -4.0578344e-01, -3.2919899e-01, -4.7428545e-01, -4.4342765e-01, -4.8948532e-01, -2.3972549e-01, -3.4704226e-01], dtype=float32), kernel: Array([[ 0.10745826, 0.48113093, -0.04176594, ..., -0.36369812, 0.15021859, -0.05807315], [-0.58055264, -0.5796294 , -0.19744751, ..., -0.07468567, -0.51942426, -0.03569252], [ 0.01138834, 0.26800624, -0.01454838, ..., -0.8072465 , 0.5447465 , 0.36949006], ..., [ 0.36528957, 0.27883154, -0.3109063 , ..., -0.5552432 , 0.23742002, 0.05708292], [-0.09702465, 0.6521357 , 0.16876654, ..., -0.252267 , -0.09898252, -0.03783756], [-0.05449405, -0.4971733 , -0.02605094, ..., 0.03078811, 0.16231588, 0.4204838 ]], dtype=float32), }, output: { bias: Array([-1.16826370e-01, 1.19446300e-01, -3.04981880e-03, -1.48974463e-01, 5.59353642e-02, 7.24940300e-02, 3.09443232e-02, -3.97321768e-02, -1.61944836e-01, -1.69336453e-01, -5.43983765e-02, 1.14368267e-01, -1.82648357e-02, -5.84783889e-02, 2.02381879e-01, -1.69515777e-02, 4.80070449e-02, -1.26781985e-01, -6.17002137e-02, 3.90345929e-03, -1.18189584e-02, 1.06815845e-01, 2.18161605e-02, -5.02868928e-02, -1.54914372e-02, 2.92950004e-01, 3.89679447e-02, 4.18758281e-02, 6.82518259e-02, 8.17627087e-02, 2.73418054e-02, -7.99397752e-02, 2.45839432e-02, 1.14206569e-02, 3.90483811e-02, -3.62712778e-02, 2.89402343e-02, -7.93813691e-02, 1.43112317e-01, -1.52107775e-01, 1.80989935e-03, -1.87356304e-02, -5.44708908e-01, -9.66665521e-03, -5.63648678e-02, 1.56991724e-02, -2.02740412e-02, 5.39821014e-02, 1.57445461e-01, -2.57236585e-02, -4.68497947e-02, -2.43687704e-02, 9.51719135e-02, 7.29279891e-02, 1.93980023e-01, 1.36569049e-02, 6.77157715e-02, -6.52143881e-02, 2.89444447e-01, 1.13720141e-01, 1.87333062e-01, 1.36707956e-02, -7.90089220e-02, 1.85856335e-02, -5.92513643e-02, 7.04728812e-02, 8.90799426e-03, 1.34663433e-01, -6.09093085e-02, -2.86442898e-02, 3.39895934e-02, -6.57519400e-02, -6.73585013e-02, -5.29800951e-02, 5.72383285e-01, 7.09643438e-02, -1.11088164e-01, 3.76055352e-02, -4.64858226e-02, 1.60521493e-04, -2.10771397e-01, -1.34013280e-01, -9.27246362e-02, -2.77775893e-04, 1.43522322e-01, 4.07825224e-02, 5.58940060e-02, 1.22946128e-01, -1.75536349e-01, 2.35335454e-01, 8.74865577e-02, 1.05813751e-02, 1.50552183e-01, 9.60319340e-02, 6.48361864e-03, 6.70408085e-02, -1.03584118e-01, -6.56064600e-02, -5.26508018e-02, 1.28201723e-01, 3.76123935e-04, -2.51332112e-02, 8.39170814e-02, 3.41377631e-02, -2.44789012e-02, -2.70240922e-02, 5.39324842e-02, -2.40320619e-02, -9.49936584e-02, -1.08399363e-02, 8.49590078e-02, -4.56464440e-02, -1.49793148e-01, 6.02403209e-02, -2.90313717e-02, -4.08411808e-02, 2.29514278e-02, -1.53553560e-01, -5.11574633e-02, 3.26815434e-02, 1.73677616e-02, -2.31411960e-02, 1.70279935e-01, -1.79555595e-01, -7.74298981e-02, 9.48442426e-03, 4.52735312e-02, 9.07007884e-03], dtype=float32), kernel: Array([[ 0.49678656, -0.04703587, 0.14592949, ..., 0.14652656, 0.4102155 , 0.43552795], [ 0.12093473, 0.28034988, 0.05580731, ..., -1.0471447 , 0.00430787, -0.37116638], [ 0.4650048 , 0.25468072, 0.04566461, ..., -0.30497208, -0.02476864, 0.01837519], ..., [ 0.13562478, -0.04827686, 0.8059429 , ..., -0.31421518, 0.24673045, -0.4773905 ], [-0.73090583, -0.39090383, 0.1884505 , ..., 0.3404823 , 0.5766392 , 0.37859675], [-0.57174814, 0.14778472, -0.11844227, ..., -0.2646688 , -0.18360172, -0.04887927]], dtype=float32), }, }, }, }, }, }, layernorm: { bias: Array([ 4.99482080e-02, -4.29396838e-01, 1.81105509e-02, 4.06332692e-05, 1.38564035e-01, -2.67413799e-02, 4.30011488e-02, -3.71168442e-02, -5.17193019e-01, -9.74547490e-03, -6.02933764e-02, 8.66996311e-03, 3.20597798e-01, -1.68491941e-04, 1.20097917e-04, -8.34563673e-02, 9.34075788e-02, -6.27656728e-02, -3.47385721e-05, -4.00960334e-02, 1.16589768e-02, 2.07135342e-02, 5.25931716e-02, -5.63471504e-02, 1.14345705e-04, -2.30510355e-04, 2.13737637e-02, -1.45103643e-03, 1.10674717e-01, 3.27529460e-02, 1.30938485e-01, 2.17286926e-02, -6.83023154e-06, -9.22439562e-04, -5.39473398e-03, -4.33185026e-02, -9.40410607e-03, -1.07025966e-01, -1.36162853e-03, 1.07634941e-03, 2.64154427e-04, -1.61221977e-02, 1.59604579e-01, 6.28793612e-04, 2.51234043e-03, -1.61393564e-02, -3.47336605e-02, -2.62861013e-01, -1.80254143e-03, 1.19618746e-02, -1.74341142e-01, -2.35029664e-02, 1.11634769e-01, -7.43765607e-02, 2.52770900e-04, 5.80298016e-03, 1.65572765e-05, -1.99696115e-05, 8.98374594e-07, 2.72183418e-02, 5.92560282e-05, 2.25074757e-02, -7.86222667e-02, 3.12634259e-02, -5.19049093e-02, 7.74146914e-02, -1.94988661e-02, 3.86598520e-04, 3.62606696e-03, 1.13942362e-02, -5.74776307e-02, 1.42280420e-04, -7.51663325e-03, -2.08205208e-01, -3.37208897e-01, 6.53023422e-02, -9.40454900e-02, 2.59398657e-04, -1.21128708e-01, 6.60678595e-02, 5.90708017e-01, -3.01386812e-04, -4.50036081e-04, -4.51468586e-05, 1.67313181e-02, 1.51245928e-04, 1.40742473e-02, 5.68821088e-06, 1.60795447e-04, 1.45528553e-04, 6.64302963e-04, 4.23284993e-02, -2.40530949e-02, 2.40640657e-06, -1.34843877e-02, 4.14691959e-03, -1.12099377e-02, -5.02210781e-02, 2.03482993e-02, 2.34702706e-01, 7.58242968e-05, -5.44176670e-04, -2.74804551e-02, 2.38151588e-02, -4.01919484e-02, -3.65443416e-02, -7.20889375e-06, -2.77027546e-04, -8.81933421e-02, 1.69912830e-01, 2.06964463e-02, 1.79356318e-02, -1.09324465e-04, 6.39357895e-04, 3.84408049e-02, 6.90272776e-04, -3.20791632e-01, 6.30142495e-06, 4.83152457e-02, -2.25914060e-04, -3.39954719e-02, -4.70114350e-02, 2.85215719e-05, 1.06935140e-05, -1.23181306e-02, 1.01582222e-01, 3.10630426e-02, -4.47405800e-02], dtype=float32), scale: Array([ 6.8868792e-01, 3.1249523e-01, 9.1828078e-01, -2.3013241e-04, 3.7263566e-01, 2.3251632e-01, 6.5610683e-01, 1.1425257e-01, 3.4526226e-01, 1.4145033e-02, 7.2997391e-01, 8.0623746e-02, 2.9320392e-01, -4.1230087e-05, -6.6874339e-04, 5.7173258e-01, 6.9446439e-01, 1.2802645e+00, -6.2002582e-05, 1.1703534e-01, 4.1233909e-01, 8.1602544e-01, 5.0138348e-01, 5.7336593e-01, 7.1710587e-05, 1.3667290e-05, 1.9022560e-01, 6.5288413e-01, 4.8426083e-01, 4.8938426e-01, 6.1096561e-01, 4.0365598e-01, 2.9291890e-05, 1.8095745e-03, 4.2584249e-01, 1.0999548e+00, 2.1008846e-01, 3.8091195e-01, 4.8326021e-03, -1.2936177e-04, -3.4556305e-04, 6.0092252e-01, 3.3731986e-02, 2.1269962e-03, 7.6969045e-01, 3.1354058e-01, 5.6419522e-01, 3.5723236e-01, 1.6803181e-01, 1.7196296e-01, 6.4958000e-01, 5.5659479e-01, 2.6263377e-01, 2.4760671e-01, -5.8708456e-04, 3.1410387e-01, -1.7643184e-04, 2.0361553e-04, -2.0005717e-05, 6.7649913e-01, -1.2435010e-04, 3.2834163e-01, 6.4273733e-01, 4.8385906e-01, 8.3392429e-01, 2.8222600e-01, 1.5083265e-01, -5.6320438e-03, 3.3680016e-01, 2.1632074e-01, 4.0965131e-01, 4.5164741e-04, 1.3090445e-01, 7.1241659e-01, 8.9404747e-02, 4.2915529e-01, 6.3035327e-01, -3.9689415e-04, 6.1234730e-01, 2.9691941e-01, 1.6158453e-01, 1.7273755e-04, 1.4721676e-04, -1.1007604e-04, 1.3981619e-01, 1.9772084e-04, 4.9367583e-01, 2.4138608e-06, -8.6720851e-05, 3.0593758e-03, 2.4073297e-01, 1.1165603e+00, 1.5985191e-01, 3.0023075e-04, 2.5869986e-01, 9.9650544e-01, 2.4187727e-01, 4.2451209e-01, 4.7070310e-01, 4.9373725e-01, -3.1352194e-04, 1.4548376e-04, 1.2102582e+00, 2.8354323e-01, 4.2492944e-01, 3.4629729e-01, 6.0579823e-06, -1.6931001e-03, 7.3870343e-01, 6.7030573e-01, 1.5534639e-01, 5.8719957e-01, -1.9493036e-04, 1.6623877e-03, 7.6086062e-01, 3.6656085e-01, 3.9590457e-01, 7.1911036e-06, 7.4346775e-01, -2.0176137e-04, 2.7498129e-01, 6.6088724e-01, -2.2207441e-06, -7.9591937e-06, 2.0538671e-01, 1.2334855e+00, 2.2934711e-01, 1.6446222e-01], dtype=float32), }, }, })