Upload aleph_byte_trigram_mlp_hard_K64/final_report.json with huggingface_hub

aleph_byte_trigram_mlp_hard_K64/final_report.json

@@ -0,0 +1,1507 @@
+{
+  "run_name": "aleph_byte_trigram_mlp_hard_K64_tau0.1_lr0.001",
+  "config": {
+    "model_type": "aleph",
+    "V": 32,
+    "D": 4,
+    "ps": 4,
+    "hidden": 64,
+    "depth": 1,
+    "channels": 3,
+    "address": "hard",
+    "K": 64,
+    "address_tau": 0.1,
+    "address_chunk": null,
+    "decode_mode": "mlp",
+    "n_atoms": 64,
+    "code_tau": 1.0,
+    "readout": "linear",
+    "dec_hidden": 512,
+    "dec_depth": 8,
+    "row_norm": "sphere",
+    "n_cross": 0,
+    "n_heads": 2,
+    "smooth_mid": 8,
+    "boundary_smooth": true,
+    "activation": "gelu",
+    "init_scheme": "orthogonal"
+  },
+  "n_params": 4321963,
+  "dataset": "byte_trigram",
+  "loss_mode": "mse",
+  "baseline_mse": 3.8e-07,
+  "best_test_mse": 0.001025198120623827,
+  "best_test_cos": 0.9966991305351257,
+  "history": [
+    [
+      100,
+      0.028800761327147484,
+      0.028682528249919414,
+      0.9028955042362213,
+      0.07561498507857323,
+      117.78747634887695,
+      0.2042787879705429,
+      97.07473068237304
+    ],
+    [
+      200,
+      0.02017538622021675,
+      0.020240215584635735,
+      0.9326131284236908,
+      0.08998905569314956,
+      118.74396743774415,
+      0.2079954579472542,
+      99.22240753173828
+    ],
+    [
+      300,
+      0.014003166928887367,
+      0.014034537505358458,
+      0.9537552058696747,
+      0.09270011335611343,
+      118.72926406860351,
+      0.2125896170735359,
+      101.09286880493164
+    ],
+    [
+      400,
+      0.01112838089466095,
+      0.01114426488056779,
+      0.963476812839508,
+      0.09395118653774262,
+      119.11174087524414,
+      0.2175210028886795,
+      103.60477600097656
+    ],
+    [
+      500,
+      0.009193015284836292,
+      0.009193478617817163,
+      0.9699370801448822,
+      0.09412979111075401,
+      119.66599884033204,
+      0.22177157551050186,
+      105.79857788085937
+    ],
+    [
+      600,
+      0.008699442259967327,
+      0.008474219031631947,
+      0.9723290383815766,
+      0.09206515401601792,
+      120.8564338684082,
+      0.2248409166932106,
+      108.89261322021484
+    ],
+    [
+      700,
+      0.007352334912866354,
+      0.007374811312183737,
+      0.9759877920150757,
+      0.0908099390566349,
+      121.26124572753906,
+      0.22652733325958252,
+      110.590673828125
+    ],
+    [
+      800,
+      0.007039516232907772,
+      0.007290515536442399,
+      0.9763115227222443,
+      0.08916582837700844,
+      121.93007125854493,
+      0.2279304802417755,
+      112.4853889465332
+    ],
+    [
+      900,
+      0.006398181430995464,
+      0.006378237064927817,
+      0.9792672395706177,
+      0.08805952519178391,
+      122.38634719848633,
+      0.22908698320388793,
+      113.87698059082031
+    ],
+    [
+      1000,
+      0.006200946867465973,
+      0.0063998537138104435,
+      0.9791910290718079,
+      0.08705428764224052,
+      123.10055999755859,
+      0.2301049441099167,
+      115.44801864624023
+    ],
+    [
+      1100,
+      0.005933549255132675,
+      0.0061329701915383335,
+      0.9800885140895843,
+      0.08648089915513993,
+      123.09069213867187,
+      0.23061649352312089,
+      116.00185775756836
+    ],
+    [
+      1200,
+      0.005545405205339193,
+      0.005546820117160678,
+      0.981985205411911,
+      0.08600256368517875,
+      123.38833084106446,
+      0.2310832604765892,
+      116.84726486206054
+    ],
+    [
+      1300,
+      0.005401348229497671,
+      0.005434776330366731,
+      0.9823466956615448,
+      0.0856689155101776,
+      123.59698867797852,
+      0.23151482194662093,
+      117.48534927368163
+    ],
+    [
+      1400,
+      0.005146161653101444,
+      0.0051509617827832695,
+      0.9832847356796265,
+      0.0853622354567051,
+      123.82352447509766,
+      0.23177338689565657,
+      118.10209121704102
+    ],
+    [
+      1500,
+      0.004968306981027126,
+      0.005000282498076558,
+      0.9837918996810913,
+      0.0851564422249794,
+      124.10174713134765,
+      0.2318960353732109,
+      118.76488571166992
+    ],
+    [
+      1600,
+      0.004822506569325924,
+      0.004881884576752782,
+      0.9841634571552277,
+      0.08491984531283378,
+      124.23340225219727,
+      0.23201111406087876,
+      119.1922622680664
+    ],
+    [
+      1700,
+      0.005124540068209171,
+      0.005152856325730681,
+      0.9832762539386749,
+      0.08470780849456787,
+      124.38554229736329,
+      0.23209028393030168,
+      119.61577377319335
+    ],
+    [
+      1800,
+      0.004602013621479273,
+      0.0045878331176936625,
+      0.9851345360279083,
+      0.08455418646335602,
+      124.60688858032226,
+      0.23205490708351134,
+      120.11979217529297
+    ],
+    [
+      1900,
+      0.004422039724886417,
+      0.004351375997066498,
+      0.9859015464782714,
+      0.08436492681503296,
+      124.72407836914063,
+      0.2319388911128044,
+      120.48799285888671
+    ],
+    [
+      2000,
+      0.004177963361144066,
+      0.004194619227200746,
+      0.9864106059074402,
+      0.08423566594719886,
+      124.85453796386719,
+      0.23189564794301987,
+      120.8257339477539
+    ],
+    [
+      2100,
+      0.004128929227590561,
+      0.004106536926701665,
+      0.9867047071456909,
+      0.08405674248933792,
+      125.02396697998047,
+      0.23183878362178803,
+      121.1934799194336
+    ],
+    [
+      2200,
+      0.003912894055247307,
+      0.003942071599885822,
+      0.9872406899929047,
+      0.08383938893675805,
+      125.03693313598633,
+      0.23177088499069215,
+      121.34869842529297
+    ],
+    [
+      2300,
+      0.003938999027013779,
+      0.0037895050132647156,
+      0.9877329409122467,
+      0.08361706510186195,
+      125.27596740722656,
+      0.231649486720562,
+      121.79842071533203
+    ],
+    [
+      2400,
+      0.003672948107123375,
+      0.00364375866483897,
+      0.988222086429596,
+      0.0833701103925705,
+      125.35969009399415,
+      0.23154629170894622,
+      122.0184944152832
+    ],
+    [
+      2500,
+      0.0036077741533517838,
+      0.003584604850038886,
+      0.9884110271930695,
+      0.08322659358382226,
+      125.3956802368164,
+      0.23152234852313996,
+      122.15045623779297
+    ],
+    [
+      2600,
+      0.0034071761183440685,
+      0.0033928266493603587,
+      0.9890355825424194,
+      0.08301392421126366,
+      125.49665298461915,
+      0.23136224448680878,
+      122.39637298583985
+    ],
+    [
+      2700,
+      0.003511592512950301,
+      0.0035396843450143933,
+      0.9885582387447357,
+      0.0827761322259903,
+      125.66863250732422,
+      0.2312498152256012,
+      122.71056060791015
+    ],
+    [
+      2800,
+      0.0031105203088372946,
+      0.0031062768772244453,
+      0.9899601340293884,
+      0.08258820995688439,
+      125.65294036865234,
+      0.23126746118068695,
+      122.7443832397461
+    ],
+    [
+      2900,
+      0.0030415551736950874,
+      0.0030434978194534778,
+      0.9901583194732666,
+      0.08237338736653328,
+      125.72677230834961,
+      0.23131745755672456,
+      122.86639709472657
+    ],
+    [
+      3000,
+      0.002938929945230484,
+      0.0029579553753137588,
+      0.9904418110847473,
+      0.0821945458650589,
+      125.75474700927734,
+      0.2312873125076294,
+      122.95954208374023
+    ],
+    [
+      3100,
+      0.0029169085901230574,
+      0.0029050474986433983,
+      0.9906009018421174,
+      0.08203032612800598,
+      125.89385833740235,
+      0.23121549040079117,
+      123.19489898681641
+    ],
+    [
+      3200,
+      0.002838249085471034,
+      0.002848999109119177,
+      0.9907860875129699,
+      0.08190143257379531,
+      125.93639373779297,
+      0.23127535730600357,
+      123.2628044128418
+    ],
+    [
+      3300,
+      0.002770825056359172,
+      0.0028306677471846343,
+      0.9908604562282562,
+      0.0815457321703434,
+      125.98058624267578,
+      0.2312321126461029,
+      123.37019958496094
+    ],
+    [
+      3400,
+      0.002659772988408804,
+      0.0026211941847577693,
+      0.9915298283100128,
+      0.08144506886601448,
+      126.01874389648438,
+      0.2312561109662056,
+      123.43526992797851
+    ],
+    [
+      3500,
+      0.0026122829876840115,
+      0.00256750073749572,
+      0.9917085766792297,
+      0.08114034160971642,
+      126.04556808471679,
+      0.23129291981458663,
+      123.47251052856446
+    ],
+    [
+      3600,
+      0.0024830647744238377,
+      0.0025027392199262976,
+      0.9919166505336762,
+      0.08097699508070946,
+      126.08363494873046,
+      0.2312628895044327,
+      123.55306091308594
+    ],
+    [
+      3700,
+      0.0024610518012195826,
+      0.0024666435550898314,
+      0.9920329988002777,
+      0.08089833855628967,
+      126.15046539306641,
+      0.23128306567668916,
+      123.64582672119141
+    ],
+    [
+      3800,
+      0.002472104737535119,
+      0.0024253945099189877,
+      0.9922000765800476,
+      0.08074926882982254,
+      126.15773315429688,
+      0.23127617090940475,
+      123.67056884765626
+    ],
+    [
+      3900,
+      0.0023962524719536304,
+      0.002380920108407736,
+      0.9923263490200043,
+      0.08058213070034981,
+      126.19769973754883,
+      0.23133646547794343,
+      123.71744003295899
+    ],
+    [
+      4000,
+      0.0023477915674448013,
+      0.002312851487658918,
+      0.9925319731235505,
+      0.0804730400443077,
+      126.22331924438477,
+      0.23135345578193664,
+      123.75740661621094
+    ],
+    [
+      4100,
+      0.0022752273362129927,
+      0.002282513841055334,
+      0.9926348507404328,
+      0.08043553978204727,
+      126.24455032348632,
+      0.23135641366243362,
+      123.79819030761719
+    ],
+    [
+      4200,
+      0.002229586010798812,
+      0.002257625968195498,
+      0.992709618806839,
+      0.0803785502910614,
+      126.3028350830078,
+      0.23135948330163955,
+      123.88111877441406
+    ],
+    [
+      4300,
+      0.0022168715950101614,
+      0.002217195788398385,
+      0.9929048717021942,
+      0.08017252385616302,
+      126.32649536132813,
+      0.23125579953193665,
+      123.96189422607422
+    ],
+    [
+      4400,
+      0.0022612588945776224,
+      0.0022756216581910848,
+      0.9926675617694855,
+      0.08014647215604782,
+      126.36302337646484,
+      0.23135244399309157,
+      123.9635383605957
+    ],
+    [
+      4500,
+      0.0021021447610110044,
+      0.002095357491634786,
+      0.9932371079921722,
+      0.07997790500521659,
+      126.3650634765625,
+      0.23131420463323593,
+      123.98940200805664
+    ],
+    [
+      4600,
+      0.002111873123794794,
+      0.002094084769487381,
+      0.9932398796081543,
+      0.07988383322954178,
+      126.41786041259766,
+      0.2312866359949112,
+      124.0627944946289
+    ],
+    [
+      4700,
+      0.002090655267238617,
+      0.0020858654053881763,
+      0.9932752966880798,
+      0.07989657297730446,
+      126.43543167114258,
+      0.2313254103064537,
+      124.0644744873047
+    ],
+    [
+      4800,
+      0.0020510677713900805,
+      0.002055858424864709,
+      0.9933654487133026,
+      0.07964377105236053,
+      126.44262619018555,
+      0.23127870708703996,
+      124.08365859985352
+    ],
+    [
+      4900,
+      0.0020076793152838945,
+      0.0019908223999664187,
+      0.9935692191123963,
+      0.07961980402469634,
+      126.46912155151367,
+      0.23128263056278228,
+      124.10295104980469
+    ],
+    [
+      5000,
+      0.002018183469772339,
+      0.002020675386302173,
+      0.993477201461792,
+      0.07957172691822052,
+      126.5152198791504,
+      0.231326225399971,
+      124.14352111816406
+    ],
+    [
+      5100,
+      0.0020221974700689316,
+      0.0020551204215735197,
+      0.9934046745300293,
+      0.07948955446481705,
+      126.50787734985352,
+      0.23128571063280107,
+      124.1221939086914
+    ],
+    [
+      5200,
+      0.001920930459164083,
+      0.0019266690942458808,
+      0.9937751114368438,
+      0.07938562631607056,
+      126.5387809753418,
+      0.23128836303949357,
+      124.14249267578126
+    ],
+    [
+      5300,
+      0.0019223339622840285,
+      0.0019179008551873267,
+      0.9938127040863037,
+      0.07943232804536819,
+      126.54245452880859,
+      0.23133704513311387,
+      124.11039962768555
+    ],
+    [
+      5400,
+      0.0019116991898044944,
+      0.0019365958869457245,
+      0.9938064157962799,
+      0.07936607673764229,
+      126.5306266784668,
+      0.23132063895463945,
+      124.06238403320313
+    ],
+    [
+      5500,
+      0.0018943792674690485,
+      0.0018895602086558938,
+      0.9939144670963287,
+      0.07937306314706802,
+      126.60442123413085,
+      0.2313313752412796,
+      124.11153564453124
+    ],
+    [
+      5600,
+      0.0019141410011798143,
+      0.0018895510002039373,
+      0.9939114034175873,
+      0.07941649258136749,
+      126.61778717041015,
+      0.23137812912464142,
+      124.06573486328125
+    ],
+    [
+      5700,
+      0.001864536083303392,
+      0.0018314062734134495,
+      0.9940945386886597,
+      0.07938547953963279,
+      126.60712585449218,
+      0.23137085139751434,
+      124.02060317993164
+    ],
+    [
+      5800,
+      0.0017995152156800032,
+      0.0018191961804404854,
+      0.9941280066967011,
+      0.07939281836152076,
+      126.65283966064453,
+      0.23140767812728882,
+      124.0166847229004
+    ],
+    [
+      5900,
+      0.0018244159873574972,
+      0.0018172520794905722,
+      0.994140237569809,
+      0.0794448621571064,
+      126.70045852661133,
+      0.2313748389482498,
+      124.04528427124023
+    ],
+    [
+      6000,
+      0.001827523112297058,
+      0.0018227588036097587,
+      0.9941176354885102,
+      0.07949492558836938,
+      126.6602165222168,
+      0.23139528036117554,
+      123.91989440917969
+    ],
+    [
+      6100,
+      0.0017575330566614866,
+      0.0017589682014659048,
+      0.9943238854408264,
+      0.07955802530050278,
+      126.69050827026368,
+      0.23142008930444719,
+      123.8784164428711
+    ],
+    [
+      6200,
+      0.0017420900985598564,
+      0.0017410170985385776,
+      0.994382506608963,
+      0.07962236329913139,
+      126.72306060791016,
+      0.2314225897192955,
+      123.84532852172852
+    ],
+    [
+      6300,
+      0.0017956674564629793,
+      0.0018137503298930823,
+      0.9942233741283417,
+      0.07955478280782699,
+      126.72207489013672,
+      0.23144118785858153,
+      123.78192443847657
+    ],
+    [
+      6400,
+      0.0017568928888067603,
+      0.0017328771529719234,
+      0.9944137573242188,
+      0.07983577400445938,
+      126.70204162597656,
+      0.23148181438446044,
+      123.65705871582031
+    ],
+    [
+      6500,
+      0.0016806230414658785,
+      0.00167753811692819,
+      0.9945945680141449,
+      0.07987815588712692,
+      126.7296974182129,
+      0.23145822584629058,
+      123.60242309570313
+    ],
+    [
+      6600,
+      0.0016736730467528105,
+      0.0016603583004325627,
+      0.9946491241455078,
+      0.07995172888040543,
+      126.75533981323242,
+      0.23146243244409562,
+      123.5505157470703
+    ],
+    [
+      6700,
+      0.0017215724801644683,
+      0.0017224322305992245,
+      0.9944741070270539,
+      0.0800711989402771,
+      126.72297668457031,
+      0.23145310282707215,
+      123.38360290527343
+    ],
+    [
+      6800,
+      0.0016299323178827763,
+      0.0016327994759194552,
+      0.9947307348251343,
+      0.08014403879642487,
+      126.7591049194336,
+      0.2314585864543915,
+      123.34561920166016
+    ],
+    [
+      6900,
+      0.001644105650484562,
+      0.001646440476179123,
+      0.9947325825691223,
+      0.08021370992064476,
+      126.7607177734375,
+      0.23147274553775787,
+      123.23168563842773
+    ],
+    [
+      7000,
+      0.0016557917697355151,
+      0.0016696124454028904,
+      0.9946186780929566,
+      0.08038085997104645,
+      126.72517700195313,
+      0.23148207515478134,
+      123.09529342651368
+    ],
+    [
+      7100,
+      0.001590296975336969,
+      0.0015924957464449109,
+      0.9948704421520234,
+      0.08041918948292733,
+      126.74960784912109,
+      0.2314906045794487,
+      122.98674240112305
+    ],
+    [
+      7200,
+      0.0015747345751151443,
+      0.0015631102374754845,
+      0.9949571013450622,
+      0.0804242491722107,
+      126.76013641357422,
+      0.23150597810745238,
+      122.9097671508789
+    ],
+    [
+      7300,
+      0.0015604895306751132,
+      0.0015573139768093824,
+      0.9949760556221008,
+      0.08059232532978058,
+      126.74603958129883,
+      0.23152956515550613,
+      122.74346923828125
+    ],
+    [
+      7400,
+      0.0015519072767347097,
+      0.0015415136818774044,
+      0.9950366795063019,
+      0.08067063540220261,
+      126.75478973388672,
+      0.23154350072145463,
+      122.62043075561523
+    ],
+    [
+      7500,
+      0.001512161223217845,
+      0.0015096124494448303,
+      0.9951374530792236,
+      0.08070659562945366,
+      126.75213623046875,
+      0.23155840784311293,
+      122.5160888671875
+    ],
+    [
+      7600,
+      0.0015041797887533903,
+      0.0015133681939914823,
+      0.9951196014881134,
+      0.08087778463959694,
+      126.73544235229492,
+      0.23161667734384536,
+      122.34964218139649
+    ],
+    [
+      7700,
+      0.0014959126710891724,
+      0.0014837358612567186,
+      0.9952118456363678,
+      0.081002726405859,
+      126.7448501586914,
+      0.23164913952350616,
+      122.1952392578125
+    ],
+    [
+      7800,
+      0.0014869370497763157,
+      0.0014933192054741085,
+      0.9951830565929413,
+      0.08110854253172875,
+      126.71752548217773,
+      0.2317025125026703,
+      122.02957000732422
+    ],
+    [
+      7900,
+      0.0014731890987604856,
+      0.001475029403809458,
+      0.9952970683574677,
+      0.0811948984861374,
+      126.73487548828125,
+      0.23174750804901123,
+      121.89350128173828
+    ],
+    [
+      8000,
+      0.0014732003910467029,
+      0.001453221566043794,
+      0.9953251719474793,
+      0.08130787163972855,
+      126.73810806274415,
+      0.2317811891436577,
+      121.77336730957032
+    ],
+    [
+      8100,
+      0.0014371012803167105,
+      0.0014391553471796215,
+      0.995365422964096,
+      0.08138529360294341,
+      126.71938781738281,
+      0.23185634166002272,
+      121.60609970092773
+    ],
+    [
+      8200,
+      0.001407342730090022,
+      0.0014078694279305638,
+      0.9954617619514465,
+      0.08147962540388107,
+      126.6977325439453,
+      0.23190893083810807,
+      121.40859146118164
+    ],
+    [
+      8300,
+      0.0013974992325529456,
+      0.0014177319128066302,
+      0.995443707704544,
+      0.08154578506946564,
+      126.67058639526367,
+      0.23196188658475875,
+      121.23370742797852
+    ],
+    [
+      8400,
+      0.0014017665525898337,
+      0.0014129838673397899,
+      0.9954666674137116,
+      0.08164956495165825,
+      126.6683219909668,
+      0.23200830221176147,
+      121.08546295166016
+    ],
+    [
+      8500,
+      0.001376533298753202,
+      0.0013697363203391433,
+      0.9955820858478546,
+      0.08175490200519561,
+      126.66518630981446,
+      0.23204002529382706,
+      120.92351913452148
+    ],
+    [
+      8600,
+      0.001375622465275228,
+      0.0013682058895938098,
+      0.9955901980400086,
+      0.08190884441137314,
+      126.6588134765625,
+      0.2320862293243408,
+      120.78725509643554
+    ],
+    [
+      8700,
+      0.0013631064211949706,
+      0.0013623884064145387,
+      0.9956271111965179,
+      0.08194247037172317,
+      126.6337287902832,
+      0.2321307122707367,
+      120.60103378295898
+    ],
+    [
+      8800,
+      0.0013350084191188216,
+      0.001334067981224507,
+      0.995696222782135,
+      0.08203213214874268,
+      126.60412521362305,
+      0.23216927498579026,
+      120.40522232055665
+    ],
+    [
+      8900,
+      0.0013341724406927824,
+      0.0013357195421122015,
+      0.9957010865211486,
+      0.08211298361420631,
+      126.59007034301757,
+      0.23220048248767852,
+      120.25910568237305
+    ],
+    [
+      9000,
+      0.0013326950138434768,
+      0.001316152629442513,
+      0.9957575857639313,
+      0.08218028545379638,
+      126.57774505615234,
+      0.23220487982034682,
+      120.08484191894532
+    ],
+    [
+      9100,
+      0.0013234707294031978,
+      0.0013190383440814912,
+      0.9957852125167846,
+      0.08228823989629745,
+      126.56712417602539,
+      0.2322417363524437,
+      119.93122482299805
+    ],
+    [
+      9200,
+      0.0012811244232580066,
+      0.0012922008521854878,
+      0.9958364307880402,
+      0.08231571391224861,
+      126.56075286865234,
+      0.23226410001516343,
+      119.77842864990234
+    ],
+    [
+      9300,
+      0.001271858112886548,
+      0.0012748652487061917,
+      0.9958885312080383,
+      0.08244126662611961,
+      126.53187103271485,
+      0.23227482587099074,
+      119.60641937255859
+    ],
+    [
+      9400,
+      0.0012733794283121824,
+      0.001268042076844722,
+      0.995913952589035,
+      0.08253940716385841,
+      126.51984100341797,
+      0.2322801172733307,
+      119.4392074584961
+    ],
+    [
+      9500,
+      0.0012597206514328718,
+      0.0012545489356853069,
+      0.9959543287754059,
+      0.0825526811182499,
+      126.49425735473633,
+      0.232297582924366,
+      119.2741912841797
+    ],
+    [
+      9600,
+      0.001262378878891468,
+      0.001259789662435651,
+      0.9959520161151886,
+      0.0826128825545311,
+      126.48273162841797,
+      0.2322937935590744,
+      119.13689880371093
+    ],
+    [
+      9700,
+      0.0012339565437287092,
+      0.0012401902582496404,
+      0.9960058748722076,
+      0.08267315030097962,
+      126.47219772338867,
+      0.23229686319828033,
+      118.98978424072266
+    ],
+    [
+      9800,
+      0.0012451952788978815,
+      0.0012396169709973036,
+      0.996020519733429,
+      0.08271930664777756,
+      126.45364761352539,
+      0.23228815346956252,
+      118.82071533203126
+    ],
+    [
+      9900,
+      0.0012206927640363574,
+      0.001223292073700577,
+      0.9960624694824218,
+      0.08276181891560555,
+      126.43686752319336,
+      0.23228850960731506,
+      118.66319046020507
+    ],
+    [
+      10000,
+      0.0012179886689409614,
+      0.001215439662337303,
+      0.9960873842239379,
+      0.08289652615785599,
+      126.41727676391602,
+      0.23229897171258926,
+      118.5048713684082
+    ],
+    [
+      10100,
+      0.0011959292460232973,
+      0.0012036134488880635,
+      0.9961264848709106,
+      0.08297973200678825,
+      126.418505859375,
+      0.23229006826877593,
+      118.35326385498047
+    ],
+    [
+      10200,
+      0.0011971313506364822,
+      0.0011960234493017196,
+      0.9961579620838166,
+      0.08307387605309487,
+      126.38851165771484,
+      0.23228520154953003,
+      118.15753173828125
+    ],
+    [
+      10300,
+      0.001186882145702839,
+      0.0011831406853161752,
+      0.9961854875087738,
+      0.0831611268222332,
+      126.38624114990235,
+      0.2322772413492203,
+      118.01736297607422
+    ],
+    [
+      10400,
+      0.001169860828667879,
+      0.0011758520035073162,
+      0.996211439371109,
+      0.08317943662405014,
+      126.35736999511718,
+      0.23228410482406617,
+      117.85008316040039
+    ],
+    [
+      10500,
+      0.0011716445442289114,
+      0.001166188740171492,
+      0.9962461233139038,
+      0.08332401514053345,
+      126.36005859375,
+      0.23228048980236055,
+      117.7102264404297
+    ],
+    [
+      10600,
+      0.0011603375896811485,
+      0.0011644201702438294,
+      0.9962540566921234,
+      0.0833615705370903,
+      126.3455696105957,
+      0.23228487074375154,
+      117.57540512084961
+    ],
+    [
+      10700,
+      0.0011775912716984749,
+      0.0011514237034134568,
+      0.996292769908905,
+      0.0835562601685524,
+      126.32433700561523,
+      0.23228495866060256,
+      117.38793487548828
+    ],
+    [
+      10800,
+      0.0011502448469400406,
+      0.0011473078979179264,
+      0.9963088452816009,
+      0.08359328210353852,
+      126.31597137451172,
+      0.23228182047605514,
+      117.26414947509765
+    ],
+    [
+      10900,
+      0.001145348884165287,
+      0.0011428493075072766,
+      0.996321576833725,
+      0.0837028332054615,
+      126.30367965698242,
+      0.23229680806398392,
+      117.13600082397461
+    ],
+    [
+      11000,
+      0.0011396566405892372,
+      0.0011314728995785116,
+      0.996358048915863,
+      0.08382222428917885,
+      126.29342269897461,
+      0.23229234367609025,
+      116.99420547485352
+    ],
+    [
+      11100,
+      0.0011218092404305935,
+      0.0011276265140622854,
+      0.9963764607906341,
+      0.0839310958981514,
+      126.27030639648437,
+      0.23228280395269393,
+      116.85280914306641
+    ],
+    [
+      11200,
+      0.0011113043874502182,
+      0.0011210313299670816,
+      0.9963958740234375,
+      0.08398709371685982,
+      126.26123809814453,
+      0.23229481875896454,
+      116.72589874267578
+    ],
+    [
+      11300,
+      0.0011123770382255316,
+      0.001112472708337009,
+      0.9964176416397095,
+      0.08410351797938347,
+      126.24457244873047,
+      0.23230913579463958,
+      116.59096221923828
+    ],
+    [
+      11400,
+      0.0011123956646770239,
+      0.001106641162186861,
+      0.9964319169521332,
+      0.08425490036606789,
+      126.22441253662109,
+      0.23231162428855895,
+      116.46762313842774
+    ],
+    [
+      11500,
+      0.0010997107019647956,
+      0.0011017954093404114,
+      0.9964551985263824,
+      0.08433346152305603,
+      126.21752395629883,
+      0.23231641352176666,
+      116.369677734375
+    ],
+    [
+      11600,
+      0.0010892825666815042,
+      0.001098142913542688,
+      0.9964685440063477,
+      0.08443198725581169,
+      126.20542602539062,
+      0.2323088213801384,
+      116.25254592895507
+    ],
+    [
+      11700,
+      0.0010828548111021519,
+      0.0010908290278166533,
+      0.996489268541336,
+      0.08448080345988274,
+      126.19028167724609,
+      0.23232013434171678,
+      116.14497833251953
+    ],
+    [
+      11800,
+      0.0010931664146482944,
+      0.0010852559935301542,
+      0.9965068578720093,
+      0.08458057940006256,
+      126.1677375793457,
+      0.232316093146801,
+      116.01935577392578
+    ],
+    [
+      11900,
+      0.0010783979669213295,
+      0.0010831191786564887,
+      0.9965152978897095,
+      0.08465723022818565,
+      126.16901168823242,
+      0.2323262169957161,
+      115.92890930175781
+    ],
+    [
+      12000,
+      0.0010739187709987164,
+      0.0010767998057417572,
+      0.9965342700481414,
+      0.08470270484685898,
+      126.1489013671875,
+      0.23233110159635545,
+      115.82496719360351
+    ],
+    [
+      12100,
+      0.001081323716789484,
+      0.0010701419319957494,
+      0.996552151441574,
+      0.08482328951358795,
+      126.13193435668946,
+      0.23233904391527177,
+      115.71751480102539
+    ],
+    [
+      12200,
+      0.001069290447048843,
+      0.0010667500318959355,
+      0.9965674638748169,
+      0.08490719124674798,
+      126.12336807250976,
+      0.23233925700187683,
+      115.6395767211914
+    ],
+    [
+      12300,
+      0.0010576930362731218,
+      0.0010630268254317344,
+      0.9965775907039642,
+      0.08500452488660812,
+      126.11942672729492,
+      0.23234191834926604,
+      115.55313110351562
+    ],
+    [
+      12400,
+      0.0010579086374491453,
+      0.0010597162996418773,
+      0.9965911269187927,
+      0.0850234366953373,
+      126.10723190307617,
+      0.23234192579984664,
+      115.49040069580079
+    ],
+    [
+      12500,
+      0.00105884182266891,
+      0.0010591769474558531,
+      0.9965912997722626,
+      0.08509839251637459,
+      126.09261703491211,
+      0.23234910964965821,
+      115.39856796264648
+    ],
+    [
+      12600,
+      0.0010492673609405756,
+      0.0010523241246119142,
+      0.9966122388839722,
+      0.08515365049242973,
+      126.07911376953125,
+      0.23235163390636443,
+      115.31780624389648
+    ],
+    [
+      12700,
+      0.0010469818953424692,
+      0.001054246979765594,
+      0.9966103374958039,
+      0.0852161318063736,
+      126.06603088378907,
+      0.23235639184713364,
+      115.23497314453125
+    ],
+    [
+      12800,
+      0.001047896919772029,
+      0.0010474840528331697,
+      0.9966319680213929,
+      0.08526609987020492,
+      126.06686859130859,
+      0.23235168755054475,
+      115.18387069702149
+    ],
+    [
+      12900,
+      0.0010481865610927343,
+      0.0010426361113786697,
+      0.9966452360153198,
+      0.08531843423843384,
+      126.05129928588867,
+      0.23235325068235396,
+      115.1278694152832
+    ],
+    [
+      13000,
+      0.0010447215754538774,
+      0.0010429605143144726,
+      0.9966436862945557,
+      0.08536772951483726,
+      126.0511375427246,
+      0.232355035841465,
+      115.06888275146484
+    ],
+    [
+      13100,
+      0.0010408565867692232,
+      0.0010389916715212166,
+      0.9966553151607513,
+      0.08540127277374268,
+      126.04122009277344,
+      0.23236988335847855,
+      115.01573944091797
+    ],
+    [
+      13200,
+      0.0010365601629018784,
+      0.001039467053487897,
+      0.9966573119163513,
+      0.08544283732771873,
+      126.03194808959961,
+      0.23238099217414857,
+      114.96460800170898
+    ],
+    [
+      13300,
+      0.001031434629112482,
+      0.0010335963335819543,
+      0.9966719686985016,
+      0.08545634523034096,
+      126.02406997680664,
+      0.23237714171409607,
+      114.91546401977538
+    ],
+    [
+      13400,
+      0.0010409815004095435,
+      0.0010307785589247942,
+      0.9966779530048371,
+      0.085519240796566,
+      126.01416549682617,
+      0.23237623870372773,
+      114.85276947021484
+    ],
+    [
+      13500,
+      0.0010494472226127982,
+      0.0010319407447241247,
+      0.996679711341858,
+      0.08555940389633179,
+      126.0154899597168,
+      0.23237807154655457,
+      114.82555770874023
+    ],
+    [
+      13600,
+      0.0010468873661011457,
+      0.0010301783098839224,
+      0.9966850399971008,
+      0.08555840700864792,
+      126.01099853515625,
+      0.23238253146409987,
+      114.8093505859375
+    ],
+    [
+      13700,
+      0.0010318877175450325,
+      0.0010297604370862246,
+      0.9966854453086853,
+      0.08554633259773255,
+      126.00723648071289,
+      0.23238351345062255,
+      114.7804786682129
+    ],
+    [
+      13800,
+      0.0010282585863023996,
+      0.0010277235647663473,
+      0.9966923534870148,
+      0.08560490161180496,
+      126.00455627441406,
+      0.23238452821969985,
+      114.75637359619141
+    ],
+    [
+      13900,
+      0.0010335029801353812,
+      0.0010305034229531884,
+      0.9966853618621826,
+      0.08557852283120156,
+      126.00122451782227,
+      0.23237734586000441,
+      114.74615020751953
+    ],
+    [
+      14000,
+      0.0010252950014546514,
+      0.0010263724252581596,
+      0.9966955006122589,
+      0.08566009029746055,
+      125.99572372436523,
+      0.23237969726324081,
+      114.72825469970704
+    ],
+    [
+      14100,
+      0.0010331285884603858,
+      0.001025454804766923,
+      0.9966976642608643,
+      0.08560354635119438,
+      125.99196929931641,
+      0.23238455206155778,
+      114.7004997253418
+    ],
+    [
+      14200,
+      0.00103470659814775,
+      0.0010292757768183946,
+      0.9966898918151855,
+      0.08562596663832664,
+      125.99538650512696,
+      0.2323866531252861,
+      114.71183395385742
+    ],
+    [
+      14300,
+      0.0010391909163445234,
+      0.0010268417419865728,
+      0.9966950476169586,
+      0.0856340155005455,
+      125.9959602355957,
+      0.23238587826490403,
+      114.70362548828125
+    ],
+    [
+      14400,
+      0.0010277373949065804,
+      0.0010271820006892084,
+      0.9966925144195556,
+      0.08563636541366577,
+      125.9925651550293,
+      0.23238842636346818,
+      114.6863914489746
+    ],
+    [
+      14500,
+      0.001018531620502472,
+      0.001030394248664379,
+      0.9966863393783569,
+      0.08566709905862809,
+      125.99628982543945,
+      0.23237616568803787,
+      114.70411758422851
+    ],
+    [
+      14600,
+      0.0010264961747452617,
+      0.001025198120623827,
+      0.9966991305351257,
+      0.08562327846884728,
+      125.99188690185547,
+      0.23238871097564698,
+      114.68511047363282
+    ]
+  ],
+  "history_columns": [
+    "step",
+    "train_loss",
+    "test_mse",
+    "test_cos",
+    "test_cv",
+    "perplexity",
+    "address_margin",
+    "hard_perplexity"
+  ]
+}