{ "benchmark": "Selective ordinal memory", "training_examples": 12000, "training_sequence_length": 48, "test_examples_per_length": 4000, "results": { "selective_ssm": { "parameters": 4594, "epochs": 60, "length_48": { "accuracy": 0.8785, "cross_entropy": 0.29386404529213905 }, "length_96_zero_shot": { "accuracy": 0.87225, "cross_entropy": 0.3296811506152153 } }, "fixed_ssm": { "parameters": 3314, "epochs": 60, "length_48": { "accuracy": 0.54725, "cross_entropy": 1.1088726967573166 }, "length_96_zero_shot": { "accuracy": 0.37325, "cross_entropy": 2.4339514672756195 } }, "gru": { "parameters": 7146, "epochs": 60, "length_48": { "accuracy": 0.96225, "cross_entropy": 0.12647006940096617 }, "length_96_zero_shot": { "accuracy": 0.95625, "cross_entropy": 0.144393608905375 } } }, "training_history": { "selective_ssm": [ { "variant": "selective_ssm", "epoch": 1, "training_loss": 2.3363112287318453, "validation_accuracy": 0.107 }, { "variant": "selective_ssm", "epoch": 2, "training_loss": 2.302377868205943, "validation_accuracy": 0.1135 }, { "variant": "selective_ssm", "epoch": 3, "training_loss": 2.299270401609705, "validation_accuracy": 0.1245 }, { "variant": "selective_ssm", "epoch": 4, "training_loss": 2.283364103195515, "validation_accuracy": 0.158 }, { "variant": "selective_ssm", "epoch": 5, "training_loss": 2.18175703413943, "validation_accuracy": 0.3035 }, { "variant": "selective_ssm", "epoch": 6, "training_loss": 1.9254426524994221, "validation_accuracy": 0.3705 }, { "variant": "selective_ssm", "epoch": 7, "training_loss": 1.6743863339119769, "validation_accuracy": 0.3745 }, { "variant": "selective_ssm", "epoch": 8, "training_loss": 1.507324477459522, "validation_accuracy": 0.471 }, { "variant": "selective_ssm", "epoch": 9, "training_loss": 1.1284294813237292, "validation_accuracy": 0.603 }, { "variant": "selective_ssm", "epoch": 10, "training_loss": 0.9387250631413562, "validation_accuracy": 0.628 }, { "variant": "selective_ssm", "epoch": 11, "training_loss": 0.878574920461533, "validation_accuracy": 0.641 }, { "variant": "selective_ssm", "epoch": 12, "training_loss": 0.8409488391368947, "validation_accuracy": 0.6465 }, { "variant": "selective_ssm", "epoch": 13, "training_loss": 0.8119208305440051, "validation_accuracy": 0.659 }, { "variant": "selective_ssm", "epoch": 14, "training_loss": 0.7943221394051897, "validation_accuracy": 0.654 }, { "variant": "selective_ssm", "epoch": 15, "training_loss": 0.7786555886268616, "validation_accuracy": 0.6685 }, { "variant": "selective_ssm", "epoch": 16, "training_loss": 0.7590317941726522, "validation_accuracy": 0.6675 }, { "variant": "selective_ssm", "epoch": 17, "training_loss": 0.7496492076427379, "validation_accuracy": 0.658 }, { "variant": "selective_ssm", "epoch": 18, "training_loss": 0.747254380520354, "validation_accuracy": 0.665 }, { "variant": "selective_ssm", "epoch": 19, "training_loss": 0.7271188319997585, "validation_accuracy": 0.6615 }, { "variant": "selective_ssm", "epoch": 20, "training_loss": 0.722328726281511, "validation_accuracy": 0.6715 }, { "variant": "selective_ssm", "epoch": 21, "training_loss": 0.7130176348889128, "validation_accuracy": 0.687 }, { "variant": "selective_ssm", "epoch": 22, "training_loss": 0.6957160462724402, "validation_accuracy": 0.6795 }, { "variant": "selective_ssm", "epoch": 23, "training_loss": 0.6857281466747852, "validation_accuracy": 0.6975 }, { "variant": "selective_ssm", "epoch": 24, "training_loss": 0.6730769137118725, "validation_accuracy": 0.7045 }, { "variant": "selective_ssm", "epoch": 25, "training_loss": 0.6580752387959906, "validation_accuracy": 0.7165 }, { "variant": "selective_ssm", "epoch": 26, "training_loss": 0.6331596590102987, "validation_accuracy": 0.731 }, { "variant": "selective_ssm", "epoch": 27, "training_loss": 0.5979627256697797, "validation_accuracy": 0.754 }, { "variant": "selective_ssm", "epoch": 28, "training_loss": 0.5488030675877916, "validation_accuracy": 0.7725 }, { "variant": "selective_ssm", "epoch": 29, "training_loss": 0.5126242935657501, "validation_accuracy": 0.778 }, { "variant": "selective_ssm", "epoch": 30, "training_loss": 0.4852969525976384, "validation_accuracy": 0.7935 }, { "variant": "selective_ssm", "epoch": 31, "training_loss": 0.4674976905609699, "validation_accuracy": 0.7975 }, { "variant": "selective_ssm", "epoch": 32, "training_loss": 0.44658732541064, "validation_accuracy": 0.7935 }, { "variant": "selective_ssm", "epoch": 33, "training_loss": 0.4338401923788355, "validation_accuracy": 0.8085 }, { "variant": "selective_ssm", "epoch": 34, "training_loss": 0.4194187414138875, "validation_accuracy": 0.8135 }, { "variant": "selective_ssm", "epoch": 35, "training_loss": 0.41004132653804537, "validation_accuracy": 0.807 }, { "variant": "selective_ssm", "epoch": 36, "training_loss": 0.39800182999448575, "validation_accuracy": 0.815 }, { "variant": "selective_ssm", "epoch": 37, "training_loss": 0.38995797837034185, "validation_accuracy": 0.8195 }, { "variant": "selective_ssm", "epoch": 38, "training_loss": 0.3821834769654781, "validation_accuracy": 0.823 }, { "variant": "selective_ssm", "epoch": 39, "training_loss": 0.37109449188759985, "validation_accuracy": 0.824 }, { "variant": "selective_ssm", "epoch": 40, "training_loss": 0.3684192531286402, "validation_accuracy": 0.8245 }, { "variant": "selective_ssm", "epoch": 41, "training_loss": 0.35656488639243106, "validation_accuracy": 0.8305 }, { "variant": "selective_ssm", "epoch": 42, "training_loss": 0.3545714464593441, "validation_accuracy": 0.8275 }, { "variant": "selective_ssm", "epoch": 43, "training_loss": 0.3465287869280957, "validation_accuracy": 0.8325 }, { "variant": "selective_ssm", "epoch": 44, "training_loss": 0.3421127656672863, "validation_accuracy": 0.84 }, { "variant": "selective_ssm", "epoch": 45, "training_loss": 0.341991571035791, "validation_accuracy": 0.842 }, { "variant": "selective_ssm", "epoch": 46, "training_loss": 0.32973376931028164, "validation_accuracy": 0.846 }, { "variant": "selective_ssm", "epoch": 47, "training_loss": 0.32450218213365434, "validation_accuracy": 0.8445 }, { "variant": "selective_ssm", "epoch": 48, "training_loss": 0.32015241777643244, "validation_accuracy": 0.847 }, { "variant": "selective_ssm", "epoch": 49, "training_loss": 0.3111421180532334, "validation_accuracy": 0.8525 }, { "variant": "selective_ssm", "epoch": 50, "training_loss": 0.3019216720094072, "validation_accuracy": 0.855 }, { "variant": "selective_ssm", "epoch": 51, "training_loss": 0.30652734827488026, "validation_accuracy": 0.853 }, { "variant": "selective_ssm", "epoch": 52, "training_loss": 0.29805111821661606, "validation_accuracy": 0.8515 }, { "variant": "selective_ssm", "epoch": 53, "training_loss": 0.29405926199669535, "validation_accuracy": 0.856 }, { "variant": "selective_ssm", "epoch": 54, "training_loss": 0.29031611408324953, "validation_accuracy": 0.861 }, { "variant": "selective_ssm", "epoch": 55, "training_loss": 0.2839207395594171, "validation_accuracy": 0.8645 }, { "variant": "selective_ssm", "epoch": 56, "training_loss": 0.2776622001795059, "validation_accuracy": 0.8615 }, { "variant": "selective_ssm", "epoch": 57, "training_loss": 0.2726325773178263, "validation_accuracy": 0.8695 }, { "variant": "selective_ssm", "epoch": 58, "training_loss": 0.2648067499728913, "validation_accuracy": 0.8625 }, { "variant": "selective_ssm", "epoch": 59, "training_loss": 0.26297357615004197, "validation_accuracy": 0.874 }, { "variant": "selective_ssm", "epoch": 60, "training_loss": 0.26065608478607016, "validation_accuracy": 0.881 } ], "fixed_ssm": [ { "variant": "fixed_ssm", "epoch": 1, "training_loss": 2.3353592334909643, "validation_accuracy": 0.105 }, { "variant": "fixed_ssm", "epoch": 2, "training_loss": 2.3039215777782682, "validation_accuracy": 0.1075 }, { "variant": "fixed_ssm", "epoch": 3, "training_loss": 2.3022656795826366, "validation_accuracy": 0.117 }, { "variant": "fixed_ssm", "epoch": 4, "training_loss": 2.301634266021404, "validation_accuracy": 0.1125 }, { "variant": "fixed_ssm", "epoch": 5, "training_loss": 2.3001739775880856, "validation_accuracy": 0.1135 }, { "variant": "fixed_ssm", "epoch": 6, "training_loss": 2.2986721840310604, "validation_accuracy": 0.1135 }, { "variant": "fixed_ssm", "epoch": 7, "training_loss": 2.2957387021247375, "validation_accuracy": 0.1285 }, { "variant": "fixed_ssm", "epoch": 8, "training_loss": 2.2903502951277064, "validation_accuracy": 0.112 }, { "variant": "fixed_ssm", "epoch": 9, "training_loss": 2.280890424200829, "validation_accuracy": 0.128 }, { "variant": "fixed_ssm", "epoch": 10, "training_loss": 2.2619114125028568, "validation_accuracy": 0.1495 }, { "variant": "fixed_ssm", "epoch": 11, "training_loss": 2.2201107664311186, "validation_accuracy": 0.18 }, { "variant": "fixed_ssm", "epoch": 12, "training_loss": 2.1196168990845377, "validation_accuracy": 0.261 }, { "variant": "fixed_ssm", "epoch": 13, "training_loss": 1.9562980291691232, "validation_accuracy": 0.318 }, { "variant": "fixed_ssm", "epoch": 14, "training_loss": 1.8030793742930635, "validation_accuracy": 0.338 }, { "variant": "fixed_ssm", "epoch": 15, "training_loss": 1.7088419122898832, "validation_accuracy": 0.3545 }, { "variant": "fixed_ssm", "epoch": 16, "training_loss": 1.6587675936678623, "validation_accuracy": 0.363 }, { "variant": "fixed_ssm", "epoch": 17, "training_loss": 1.6198743759317602, "validation_accuracy": 0.3575 }, { "variant": "fixed_ssm", "epoch": 18, "training_loss": 1.597056954465014, "validation_accuracy": 0.365 }, { "variant": "fixed_ssm", "epoch": 19, "training_loss": 1.572468547110862, "validation_accuracy": 0.3745 }, { "variant": "fixed_ssm", "epoch": 20, "training_loss": 1.5531339569294706, "validation_accuracy": 0.375 }, { "variant": "fixed_ssm", "epoch": 21, "training_loss": 1.5309961526951892, "validation_accuracy": 0.3815 }, { "variant": "fixed_ssm", "epoch": 22, "training_loss": 1.5084752858953272, "validation_accuracy": 0.391 }, { "variant": "fixed_ssm", "epoch": 23, "training_loss": 1.4886164310130667, "validation_accuracy": 0.4055 }, { "variant": "fixed_ssm", "epoch": 24, "training_loss": 1.464285705951934, "validation_accuracy": 0.4155 }, { "variant": "fixed_ssm", "epoch": 25, "training_loss": 1.4398610591888428, "validation_accuracy": 0.4275 }, { "variant": "fixed_ssm", "epoch": 26, "training_loss": 1.4151407962149762, "validation_accuracy": 0.433 }, { "variant": "fixed_ssm", "epoch": 27, "training_loss": 1.3938992302468483, "validation_accuracy": 0.446 }, { "variant": "fixed_ssm", "epoch": 28, "training_loss": 1.3761005984975936, "validation_accuracy": 0.4495 }, { "variant": "fixed_ssm", "epoch": 29, "training_loss": 1.355471986405393, "validation_accuracy": 0.4585 }, { "variant": "fixed_ssm", "epoch": 30, "training_loss": 1.3363145336191704, "validation_accuracy": 0.4735 }, { "variant": "fixed_ssm", "epoch": 31, "training_loss": 1.3142326943417812, "validation_accuracy": 0.4765 }, { "variant": "fixed_ssm", "epoch": 32, "training_loss": 1.2969839598270172, "validation_accuracy": 0.489 }, { "variant": "fixed_ssm", "epoch": 33, "training_loss": 1.2858448307564918, "validation_accuracy": 0.4895 }, { "variant": "fixed_ssm", "epoch": 34, "training_loss": 1.266926187150022, "validation_accuracy": 0.4935 }, { "variant": "fixed_ssm", "epoch": 35, "training_loss": 1.2500894957400384, "validation_accuracy": 0.5025 }, { "variant": "fixed_ssm", "epoch": 36, "training_loss": 1.2322490545029336, "validation_accuracy": 0.5095 }, { "variant": "fixed_ssm", "epoch": 37, "training_loss": 1.2208100836327735, "validation_accuracy": 0.5095 }, { "variant": "fixed_ssm", "epoch": 38, "training_loss": 1.2055485223201996, "validation_accuracy": 0.521 }, { "variant": "fixed_ssm", "epoch": 39, "training_loss": 1.1940563917160034, "validation_accuracy": 0.5255 }, { "variant": "fixed_ssm", "epoch": 40, "training_loss": 1.1838379692524037, "validation_accuracy": 0.5185 }, { "variant": "fixed_ssm", "epoch": 41, "training_loss": 1.1703011938866148, "validation_accuracy": 0.5235 }, { "variant": "fixed_ssm", "epoch": 42, "training_loss": 1.1595663694625205, "validation_accuracy": 0.5245 }, { "variant": "fixed_ssm", "epoch": 43, "training_loss": 1.1505883069748575, "validation_accuracy": 0.5235 }, { "variant": "fixed_ssm", "epoch": 44, "training_loss": 1.1402675395316266, "validation_accuracy": 0.524 }, { "variant": "fixed_ssm", "epoch": 45, "training_loss": 1.13290928018854, "validation_accuracy": 0.5365 }, { "variant": "fixed_ssm", "epoch": 46, "training_loss": 1.1214076787867444, "validation_accuracy": 0.534 }, { "variant": "fixed_ssm", "epoch": 47, "training_loss": 1.11458838493266, "validation_accuracy": 0.5385 }, { "variant": "fixed_ssm", "epoch": 48, "training_loss": 1.1090772849448183, "validation_accuracy": 0.5415 }, { "variant": "fixed_ssm", "epoch": 49, "training_loss": 1.098906600728948, "validation_accuracy": 0.545 }, { "variant": "fixed_ssm", "epoch": 50, "training_loss": 1.0915721122254716, "validation_accuracy": 0.544 }, { "variant": "fixed_ssm", "epoch": 51, "training_loss": 1.0865838312088174, "validation_accuracy": 0.5475 }, { "variant": "fixed_ssm", "epoch": 52, "training_loss": 1.078594740400923, "validation_accuracy": 0.5515 }, { "variant": "fixed_ssm", "epoch": 53, "training_loss": 1.0691243232564722, "validation_accuracy": 0.5575 }, { "variant": "fixed_ssm", "epoch": 54, "training_loss": 1.0649417347096382, "validation_accuracy": 0.562 }, { "variant": "fixed_ssm", "epoch": 55, "training_loss": 1.0544955692392715, "validation_accuracy": 0.5565 }, { "variant": "fixed_ssm", "epoch": 56, "training_loss": 1.0504532278852259, "validation_accuracy": 0.56 }, { "variant": "fixed_ssm", "epoch": 57, "training_loss": 1.0466023138228884, "validation_accuracy": 0.5515 }, { "variant": "fixed_ssm", "epoch": 58, "training_loss": 1.0409651500113466, "validation_accuracy": 0.557 }, { "variant": "fixed_ssm", "epoch": 59, "training_loss": 1.037726326191679, "validation_accuracy": 0.56 }, { "variant": "fixed_ssm", "epoch": 60, "training_loss": 1.030308889581802, "validation_accuracy": 0.5615 } ], "gru": [ { "variant": "gru", "epoch": 1, "training_loss": 2.307496720171989, "validation_accuracy": 0.113 }, { "variant": "gru", "epoch": 2, "training_loss": 2.2951550382248898, "validation_accuracy": 0.1225 }, { "variant": "gru", "epoch": 3, "training_loss": 2.2853371894105954, "validation_accuracy": 0.143 }, { "variant": "gru", "epoch": 4, "training_loss": 2.2343535321824093, "validation_accuracy": 0.234 }, { "variant": "gru", "epoch": 5, "training_loss": 1.9750959086925426, "validation_accuracy": 0.3235 }, { "variant": "gru", "epoch": 6, "training_loss": 1.7028222895683127, "validation_accuracy": 0.376 }, { "variant": "gru", "epoch": 7, "training_loss": 1.502154619135755, "validation_accuracy": 0.3785 }, { "variant": "gru", "epoch": 8, "training_loss": 1.387926383221403, "validation_accuracy": 0.403 }, { "variant": "gru", "epoch": 9, "training_loss": 1.2961563227024484, "validation_accuracy": 0.4395 }, { "variant": "gru", "epoch": 10, "training_loss": 1.1968751131220068, "validation_accuracy": 0.487 }, { "variant": "gru", "epoch": 11, "training_loss": 1.1169310351635546, "validation_accuracy": 0.521 }, { "variant": "gru", "epoch": 12, "training_loss": 1.036836271590375, "validation_accuracy": 0.5545 }, { "variant": "gru", "epoch": 13, "training_loss": 0.9751845407993236, "validation_accuracy": 0.563 }, { "variant": "gru", "epoch": 14, "training_loss": 0.9272876584783514, "validation_accuracy": 0.574 }, { "variant": "gru", "epoch": 15, "training_loss": 0.8842355964031625, "validation_accuracy": 0.5975 }, { "variant": "gru", "epoch": 16, "training_loss": 0.8468549581284218, "validation_accuracy": 0.6095 }, { "variant": "gru", "epoch": 17, "training_loss": 0.8111054212489026, "validation_accuracy": 0.614 }, { "variant": "gru", "epoch": 18, "training_loss": 0.7773945978347291, "validation_accuracy": 0.6315 }, { "variant": "gru", "epoch": 19, "training_loss": 0.7439964362915527, "validation_accuracy": 0.646 }, { "variant": "gru", "epoch": 20, "training_loss": 0.7159112793333987, "validation_accuracy": 0.6545 }, { "variant": "gru", "epoch": 21, "training_loss": 0.6868720397036127, "validation_accuracy": 0.672 }, { "variant": "gru", "epoch": 22, "training_loss": 0.658954123233227, "validation_accuracy": 0.682 }, { "variant": "gru", "epoch": 23, "training_loss": 0.63022185259677, "validation_accuracy": 0.7045 }, { "variant": "gru", "epoch": 24, "training_loss": 0.6026503367626921, "validation_accuracy": 0.709 }, { "variant": "gru", "epoch": 25, "training_loss": 0.5758582455046634, "validation_accuracy": 0.7135 }, { "variant": "gru", "epoch": 26, "training_loss": 0.551946241170802, "validation_accuracy": 0.726 }, { "variant": "gru", "epoch": 27, "training_loss": 0.5226797683441893, "validation_accuracy": 0.753 }, { "variant": "gru", "epoch": 28, "training_loss": 0.49462918398228095, "validation_accuracy": 0.763 }, { "variant": "gru", "epoch": 29, "training_loss": 0.46910898989819466, "validation_accuracy": 0.7765 }, { "variant": "gru", "epoch": 30, "training_loss": 0.43757979920569884, "validation_accuracy": 0.808 }, { "variant": "gru", "epoch": 31, "training_loss": 0.40479996293149095, "validation_accuracy": 0.8185 }, { "variant": "gru", "epoch": 32, "training_loss": 0.3770996433623294, "validation_accuracy": 0.834 }, { "variant": "gru", "epoch": 33, "training_loss": 0.3512212298017867, "validation_accuracy": 0.8495 }, { "variant": "gru", "epoch": 34, "training_loss": 0.32752167861512366, "validation_accuracy": 0.8625 }, { "variant": "gru", "epoch": 35, "training_loss": 0.30428984475896714, "validation_accuracy": 0.868 }, { "variant": "gru", "epoch": 36, "training_loss": 0.28324485047066467, "validation_accuracy": 0.883 }, { "variant": "gru", "epoch": 37, "training_loss": 0.2649370038763006, "validation_accuracy": 0.8815 }, { "variant": "gru", "epoch": 38, "training_loss": 0.24939178437628645, "validation_accuracy": 0.8885 }, { "variant": "gru", "epoch": 39, "training_loss": 0.23531965721160808, "validation_accuracy": 0.897 }, { "variant": "gru", "epoch": 40, "training_loss": 0.21780814262146644, "validation_accuracy": 0.9045 }, { "variant": "gru", "epoch": 41, "training_loss": 0.20509977480198474, "validation_accuracy": 0.908 }, { "variant": "gru", "epoch": 42, "training_loss": 0.19458670755650134, "validation_accuracy": 0.908 }, { "variant": "gru", "epoch": 43, "training_loss": 0.18120360152518494, "validation_accuracy": 0.9105 }, { "variant": "gru", "epoch": 44, "training_loss": 0.17063416794259498, "validation_accuracy": 0.921 }, { "variant": "gru", "epoch": 45, "training_loss": 0.16205515823465713, "validation_accuracy": 0.9225 }, { "variant": "gru", "epoch": 46, "training_loss": 0.15119966230493911, "validation_accuracy": 0.925 }, { "variant": "gru", "epoch": 47, "training_loss": 0.14337273845647244, "validation_accuracy": 0.9335 }, { "variant": "gru", "epoch": 48, "training_loss": 0.13570780164383828, "validation_accuracy": 0.937 }, { "variant": "gru", "epoch": 49, "training_loss": 0.1279922604877898, "validation_accuracy": 0.94 }, { "variant": "gru", "epoch": 50, "training_loss": 0.12057814762947407, "validation_accuracy": 0.942 }, { "variant": "gru", "epoch": 51, "training_loss": 0.11308145110911512, "validation_accuracy": 0.945 }, { "variant": "gru", "epoch": 52, "training_loss": 0.10737714948172265, "validation_accuracy": 0.944 }, { "variant": "gru", "epoch": 53, "training_loss": 0.10131331739273478, "validation_accuracy": 0.949 }, { "variant": "gru", "epoch": 54, "training_loss": 0.0959858498040666, "validation_accuracy": 0.954 }, { "variant": "gru", "epoch": 55, "training_loss": 0.09025249519246689, "validation_accuracy": 0.9555 }, { "variant": "gru", "epoch": 56, "training_loss": 0.08463195243731458, "validation_accuracy": 0.9575 }, { "variant": "gru", "epoch": 57, "training_loss": 0.0816229392397911, "validation_accuracy": 0.96 }, { "variant": "gru", "epoch": 58, "training_loss": 0.07712231750818009, "validation_accuracy": 0.961 }, { "variant": "gru", "epoch": 59, "training_loss": 0.07365085858296841, "validation_accuracy": 0.961 }, { "variant": "gru", "epoch": 60, "training_loss": 0.07048324971122945, "validation_accuracy": 0.963 } ] } }