| { | |
| "benchmark": "Long-lag adding problem", | |
| "training_examples": 16000, | |
| "training_length": 100, | |
| "results": { | |
| "vanilla_rnn": { | |
| "parameters": 4825, | |
| "training_epochs": 65, | |
| "checkpoint_reused": true, | |
| "length_100": { | |
| "rmse": 0.385515958070755, | |
| "mae": 0.31562933325767517, | |
| "within_0.1": 0.19025 | |
| }, | |
| "length_200_zero_shot": { | |
| "rmse": 0.4009908437728882, | |
| "mae": 0.32702550292015076, | |
| "within_0.1": 0.1925 | |
| }, | |
| "length_400_zero_shot": { | |
| "rmse": 0.4056301414966583, | |
| "mae": 0.3313989043235779, | |
| "within_0.1": 0.1885 | |
| } | |
| }, | |
| "lstm": { | |
| "parameters": 4641, | |
| "training_epochs": 100, | |
| "checkpoint_reused": true, | |
| "length_100": { | |
| "rmse": 0.010518018156290054, | |
| "mae": 0.007944227196276188, | |
| "within_0.1": 1.0 | |
| }, | |
| "length_200_zero_shot": { | |
| "rmse": 0.0381074883043766, | |
| "mae": 0.032907769083976746, | |
| "within_0.1": 0.997 | |
| }, | |
| "length_400_zero_shot": { | |
| "rmse": 0.10497123003005981, | |
| "mae": 0.09410939365625381, | |
| "within_0.1": 0.56825 | |
| } | |
| }, | |
| "gru": { | |
| "parameters": 4357, | |
| "training_epochs": 60, | |
| "checkpoint_reused": false, | |
| "length_100": { | |
| "rmse": 0.012872785329818726, | |
| "mae": 0.00958452932536602, | |
| "within_0.1": 0.99975 | |
| }, | |
| "length_200_zero_shot": { | |
| "rmse": 0.19797414541244507, | |
| "mae": 0.17585906386375427, | |
| "within_0.1": 0.21025 | |
| }, | |
| "length_400_zero_shot": { | |
| "rmse": 0.4264245331287384, | |
| "mae": 0.37727734446525574, | |
| "within_0.1": 0.115 | |
| } | |
| } | |
| }, | |
| "training_history": { | |
| "vanilla_rnn": [], | |
| "lstm": [], | |
| "gru": [ | |
| { | |
| "variant": "gru", | |
| "epoch": 1, | |
| "training_mse": 0.27897960042196607, | |
| "validation_rmse": 0.41890600323677063 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 2, | |
| "training_mse": 0.1676346929300399, | |
| "validation_rmse": 0.41886889934539795 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 3, | |
| "training_mse": 0.16712906530925206, | |
| "validation_rmse": 0.4186388850212097 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 4, | |
| "training_mse": 0.1668442790470426, | |
| "validation_rmse": 0.4184664487838745 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 5, | |
| "training_mse": 0.16630058392645822, | |
| "validation_rmse": 0.41935667395591736 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 6, | |
| "training_mse": 0.165447969521795, | |
| "validation_rmse": 0.41495800018310547 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 7, | |
| "training_mse": 0.16104612369385976, | |
| "validation_rmse": 0.3994660973548889 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 8, | |
| "training_mse": 0.11790320377737756, | |
| "validation_rmse": 0.22453507781028748 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 9, | |
| "training_mse": 0.03282030183999311, | |
| "validation_rmse": 0.14513947069644928 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 10, | |
| "training_mse": 0.015436596828438932, | |
| "validation_rmse": 0.09309564530849457 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 11, | |
| "training_mse": 0.009214021206375152, | |
| "validation_rmse": 0.09240031987428665 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 12, | |
| "training_mse": 0.008095646365767433, | |
| "validation_rmse": 0.07105675339698792 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 13, | |
| "training_mse": 0.0050112293730120334, | |
| "validation_rmse": 0.08008020371198654 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 14, | |
| "training_mse": 0.0045446014948307524, | |
| "validation_rmse": 0.06406337022781372 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 15, | |
| "training_mse": 0.0036848297505269923, | |
| "validation_rmse": 0.0719640851020813 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 16, | |
| "training_mse": 0.0030352562677026505, | |
| "validation_rmse": 0.04874277859926224 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 17, | |
| "training_mse": 0.0025032314080153666, | |
| "validation_rmse": 0.046146493405103683 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 18, | |
| "training_mse": 0.0022726455843815256, | |
| "validation_rmse": 0.04251471161842346 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 19, | |
| "training_mse": 0.0017833743329411225, | |
| "validation_rmse": 0.040294766426086426 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 20, | |
| "training_mse": 0.001669998792561865, | |
| "validation_rmse": 0.04777347669005394 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 21, | |
| "training_mse": 0.0015638722827480663, | |
| "validation_rmse": 0.03726698085665703 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 22, | |
| "training_mse": 0.001392907346598804, | |
| "validation_rmse": 0.03497902676463127 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 23, | |
| "training_mse": 0.0013391529228390447, | |
| "validation_rmse": 0.033759646117687225 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 24, | |
| "training_mse": 0.0012772484056075058, | |
| "validation_rmse": 0.03317611664533615 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 25, | |
| "training_mse": 0.001164291207382958, | |
| "validation_rmse": 0.03242440149188042 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 26, | |
| "training_mse": 0.0009748986090666481, | |
| "validation_rmse": 0.032584499567747116 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 27, | |
| "training_mse": 0.0011779089155237352, | |
| "validation_rmse": 0.03799568861722946 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 28, | |
| "training_mse": 0.0009888880525804346, | |
| "validation_rmse": 0.03183925524353981 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 29, | |
| "training_mse": 0.0008991299823503054, | |
| "validation_rmse": 0.034730687737464905 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 30, | |
| "training_mse": 0.0007818559099668785, | |
| "validation_rmse": 0.02763444557785988 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 31, | |
| "training_mse": 0.0007222901931816032, | |
| "validation_rmse": 0.02985423244535923 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 32, | |
| "training_mse": 0.0007579267886097706, | |
| "validation_rmse": 0.025924278423190117 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 33, | |
| "training_mse": 0.0006937842796711872, | |
| "validation_rmse": 0.03327471762895584 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 34, | |
| "training_mse": 0.0006558693070445092, | |
| "validation_rmse": 0.023472504690289497 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 35, | |
| "training_mse": 0.0006232630472541565, | |
| "validation_rmse": 0.02608579955995083 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 36, | |
| "training_mse": 0.0006158285000960209, | |
| "validation_rmse": 0.022989388555288315 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 37, | |
| "training_mse": 0.0006331298666607056, | |
| "validation_rmse": 0.036010462790727615 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 38, | |
| "training_mse": 0.0006183669745315989, | |
| "validation_rmse": 0.021163983270525932 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 39, | |
| "training_mse": 0.00048328266111350365, | |
| "validation_rmse": 0.021558744832873344 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 40, | |
| "training_mse": 0.0004281246233620637, | |
| "validation_rmse": 0.019658120349049568 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 41, | |
| "training_mse": 0.00040030264801951864, | |
| "validation_rmse": 0.02244676649570465 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 42, | |
| "training_mse": 0.0004425118494841699, | |
| "validation_rmse": 0.022320907562971115 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 43, | |
| "training_mse": 0.00036702687216420024, | |
| "validation_rmse": 0.019741958007216454 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 44, | |
| "training_mse": 0.0003764253404889522, | |
| "validation_rmse": 0.020163239911198616 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 45, | |
| "training_mse": 0.00040674276048472774, | |
| "validation_rmse": 0.022408759221434593 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 46, | |
| "training_mse": 0.0002968242945840641, | |
| "validation_rmse": 0.02195422723889351 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 47, | |
| "training_mse": 0.0002811291208398336, | |
| "validation_rmse": 0.016174808144569397 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 48, | |
| "training_mse": 0.0002783976602903937, | |
| "validation_rmse": 0.016525471583008766 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 49, | |
| "training_mse": 0.00035174332590522393, | |
| "validation_rmse": 0.023897603154182434 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 50, | |
| "training_mse": 0.0003189754333727742, | |
| "validation_rmse": 0.017815887928009033 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 51, | |
| "training_mse": 0.00039808949042621647, | |
| "validation_rmse": 0.015075829811394215 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 52, | |
| "training_mse": 0.0002545547320060284, | |
| "validation_rmse": 0.014894144609570503 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 53, | |
| "training_mse": 0.00022400631883462508, | |
| "validation_rmse": 0.01521306298673153 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 54, | |
| "training_mse": 0.0002214045068415414, | |
| "validation_rmse": 0.014713034965097904 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 55, | |
| "training_mse": 0.000234982696600214, | |
| "validation_rmse": 0.013789931312203407 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 56, | |
| "training_mse": 0.0002446352727785115, | |
| "validation_rmse": 0.01484834123402834 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 57, | |
| "training_mse": 0.0004104178783202928, | |
| "validation_rmse": 0.0208186823874712 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 58, | |
| "training_mse": 0.00024293293222038458, | |
| "validation_rmse": 0.013977939262986183 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 59, | |
| "training_mse": 0.00022919440160631128, | |
| "validation_rmse": 0.013975833542644978 | |
| }, | |
| { | |
| "variant": "gru", | |
| "epoch": 60, | |
| "training_mse": 0.00022673859336738668, | |
| "validation_rmse": 0.014603943563997746 | |
| } | |
| ] | |
| } | |
| } |