SPIRA v2: entropy-patch tokenization + hierarchical dopamine-gated spiking cortex + hippocampal one-shot memory with replay consolidation
032a961 verified | { | |
| "model_meta": { | |
| "train_bytes": 13223262, | |
| "n_patches": 2799170, | |
| "avg_patch_len": 4.7239938981912495, | |
| "distinct_patches": 234130, | |
| "registry_size": 147575, | |
| "budget_bits": 15.91943359375, | |
| "segmentation_MB_per_s": 1.7565680288219827 | |
| }, | |
| "config": { | |
| "target_avg_len": 4.5, | |
| "min_len": 2, | |
| "max_len": 32, | |
| "alpha": 0.5, | |
| "d_in": 1024, | |
| "k_in": 12, | |
| "n_res": 512, | |
| "vocab_size": 16384, | |
| "seed": 20261022, | |
| "hierarchy": true, | |
| "n_g": 256, | |
| "win": 8, | |
| "eta_pos": 0.01, | |
| "eta_r": 0.01, | |
| "eta_stdp": 0.0015, | |
| "am_size": 8388608, | |
| "am_m": 4, | |
| "am_min_lanes": 3, | |
| "snn_tokens": 600000, | |
| "am_tokens": 1200000, | |
| "epochs": 2, | |
| "replay_tokens": 180000, | |
| "replay_slice": 512, | |
| "beta_am": 0.5, | |
| "am_override": 6.0, | |
| "temperature": 0.8, | |
| "top_k": 8 | |
| }, | |
| "vocab_coverage_curve": [ | |
| { | |
| "vocab": 4096, | |
| "cov_test": 0.3848, | |
| "cov_oneshot": 0.4638 | |
| }, | |
| { | |
| "vocab": 6144, | |
| "cov_test": 0.4543, | |
| "cov_oneshot": 0.5414 | |
| }, | |
| { | |
| "vocab": 8192, | |
| "cov_test": 0.5071, | |
| "cov_oneshot": 0.5999 | |
| }, | |
| { | |
| "vocab": 12288, | |
| "cov_test": 0.5838, | |
| "cov_oneshot": 0.6797 | |
| }, | |
| { | |
| "vocab": 16384, | |
| "cov_test": 0.6394, | |
| "cov_oneshot": 0.7404 | |
| }, | |
| { | |
| "vocab": 24576, | |
| "cov_test": 0.7153, | |
| "cov_oneshot": 0.8077 | |
| }, | |
| { | |
| "vocab": 32768, | |
| "cov_test": 0.7649, | |
| "cov_oneshot": 0.8489 | |
| } | |
| ], | |
| "vocab_coverage": { | |
| "distinct_train_patches": 234130, | |
| "note": "fraction of held-out target patches inside the top-V train vocabulary" | |
| }, | |
| "compression": { | |
| "gzip": { | |
| "bits_per_byte": 2.5916625282547328, | |
| "seconds": 0.06935647600039374 | |
| }, | |
| "bpe_4096": { | |
| "vocab_size": 4096, | |
| "n_tokens": 213578, | |
| "avg_token_bytes": 3.196157843972694, | |
| "stream_bits_unigram": 9.4638792864356, | |
| "stream_bits_bigram": 4.818219420963174, | |
| "dict_bits": 301432.0, | |
| "bits_per_byte_unigram": 3.4025926385171776, | |
| "bits_per_byte_bigram": 1.9490787345548937, | |
| "train_s": 1.5803333969997766 | |
| }, | |
| "bpe_16384": { | |
| "vocab_size": 16384, | |
| "n_tokens": 180088, | |
| "avg_token_bytes": 3.7905301852427704, | |
| "stream_bits_unigram": 9.814124375536865, | |
| "stream_bits_bigram": 4.611101226860982, | |
| "dict_bits": 1386648.0, | |
| "bits_per_byte_unigram": 4.620451270809888, | |
| "bits_per_byte_bigram": 3.2478139629915233, | |
| "train_s": 1.7241879100001825 | |
| }, | |
| "ept": { | |
| "n_tokens": 144013, | |
| "avg_token_bytes": 4.740051245373682, | |
| "distinct_in_test": 41604, | |
| "stream_bits_unigram": 14.295363293655472, | |
| "stream_bits_bigram": 2.3544508230397527, | |
| "dict_bits_test_local": 2462160.0, | |
| "bits_per_byte_unigram": 6.622745523570205, | |
| "bits_per_byte_bigram": 4.103592912663283, | |
| "novelty_rate": 0.06314707699999306, | |
| "bytes_in_novel_patches": 43623.0, | |
| "segment_s": 0.26365890000124637, | |
| "MB_per_s": 2.589061093696337 | |
| } | |
| }, | |
| "fusion_calibration": { | |
| "scale": 0.049578128047281136, | |
| "sigma_fast": 0.030150376720135645, | |
| "sigma_slow": 0.608138667345047, | |
| "beta": 0.058987816055875775 | |
| }, | |
| "one_shot": { | |
| "book_bytes": 60000, | |
| "before_teaching": { | |
| "n_positions": 8000, | |
| "vocab_coverage": 0.73025, | |
| "snn_acc": 0.00325, | |
| "hier_acc": 0.003375, | |
| "am_acc": 0.0025, | |
| "fused_acc": 0.0055, | |
| "snn_acc_invocab": 0.004450530640191715, | |
| "hier_acc_invocab": 0.004621704895583704, | |
| "am_acc_invocab": 0.003252310852447792, | |
| "fused_acc_invocab": 0.007531667237247518, | |
| "corrupt_lanes": 0 | |
| }, | |
| "teaching": { | |
| "tokens": 13346, | |
| "new_patches": 329, | |
| "associations": 13342, | |
| "episode_tokens": 13346, | |
| "seconds": 0.35267701299926557 | |
| }, | |
| "after_one_pass": { | |
| "n_positions": 8000, | |
| "vocab_coverage": 0.73025, | |
| "snn_acc": 0.00325, | |
| "hier_acc": 0.003375, | |
| "am_acc": 0.993125, | |
| "fused_acc": 0.993125, | |
| "snn_acc_invocab": 0.004450530640191715, | |
| "hier_acc_invocab": 0.004621704895583704, | |
| "am_acc_invocab": 0.9926395070181445, | |
| "fused_acc_invocab": 0.9926395070181445, | |
| "corrupt_lanes": 0 | |
| } | |
| }, | |
| "replay": { | |
| "replay_tokens": 100000, | |
| "stage_tokens": 25000, | |
| "before": { | |
| "n_positions": 8000, | |
| "vocab_coverage": 0.73025, | |
| "snn_acc": 0.00325, | |
| "hier_acc": 0.003375, | |
| "am_acc": 0.0, | |
| "fused_acc": 0.003375, | |
| "snn_acc_invocab": 0.004450530640191715, | |
| "hier_acc_invocab": 0.004621704895583704, | |
| "am_acc_invocab": 0.0, | |
| "fused_acc_invocab": 0.004621704895583704, | |
| "corrupt_lanes": 0 | |
| }, | |
| "curve": [ | |
| { | |
| "replayed_tokens": 25000, | |
| "hier_acc": 0.178, | |
| "hier_acc_invocab": 0.24756606397774686, | |
| "snn_acc": 0.177, | |
| "replay_acc": 0.13062542352612908, | |
| "beta": 0.03478016090689402 | |
| }, | |
| { | |
| "replayed_tokens": 50000, | |
| "hier_acc": 0.247, | |
| "hier_acc_invocab": 0.3435326842837274, | |
| "snn_acc": 0.2465, | |
| "replay_acc": 0.31538246900785266, | |
| "beta": 0.0031896269343629304 | |
| }, | |
| { | |
| "replayed_tokens": 75000, | |
| "hier_acc": 0.2665, | |
| "hier_acc_invocab": 0.370653685674548, | |
| "snn_acc": 0.2665, | |
| "replay_acc": 0.3782437118826484, | |
| "beta": 0.0018039194478497163 | |
| }, | |
| { | |
| "replayed_tokens": 100000, | |
| "hier_acc": 0.275, | |
| "hier_acc_invocab": 0.38247566063977745, | |
| "snn_acc": 0.275, | |
| "replay_acc": 0.38071511141228526, | |
| "beta": 0.004831501266509353 | |
| } | |
| ], | |
| "after": { | |
| "n_positions": 8000, | |
| "vocab_coverage": 0.73025, | |
| "snn_acc": 0.468875, | |
| "hier_acc": 0.468875, | |
| "am_acc": 0.0, | |
| "fused_acc": 0.468875, | |
| "snn_acc_invocab": 0.6420746319753509, | |
| "hier_acc_invocab": 0.6420746319753509, | |
| "am_acc_invocab": 0.0, | |
| "fused_acc_invocab": 0.6420746319753509, | |
| "corrupt_lanes": 0 | |
| }, | |
| "am_recall_after_replay": { | |
| "n_positions": 8000, | |
| "vocab_coverage": 0.73025, | |
| "snn_acc": 0.468875, | |
| "hier_acc": 0.468875, | |
| "am_acc": 0.993125, | |
| "fused_acc": 0.9935, | |
| "snn_acc_invocab": 0.6420746319753509, | |
| "hier_acc_invocab": 0.6420746319753509, | |
| "am_acc_invocab": 0.9926395070181445, | |
| "fused_acc_invocab": 0.9931530297843204, | |
| "corrupt_lanes": 0 | |
| }, | |
| "interference_test_core_only": { | |
| "n_positions": 8000, | |
| "vocab_coverage": 0.694, | |
| "snn_acc": 0.0075, | |
| "hier_acc": 0.0075, | |
| "am_acc": 0.0, | |
| "fused_acc": 0.0075, | |
| "snn_acc_invocab": 0.010806916426512969, | |
| "hier_acc_invocab": 0.010806916426512969, | |
| "am_acc_invocab": 0.0, | |
| "fused_acc_invocab": 0.010806916426512969, | |
| "corrupt_lanes": 0 | |
| }, | |
| "interference_test_fused": { | |
| "n_positions": 8000, | |
| "vocab_coverage": 0.694, | |
| "snn_acc": 0.0075, | |
| "hier_acc": 0.0075, | |
| "am_acc": 0.0025, | |
| "fused_acc": 0.009375, | |
| "snn_acc_invocab": 0.010806916426512969, | |
| "hier_acc_invocab": 0.010806916426512969, | |
| "am_acc_invocab": 0.003422190201729107, | |
| "fused_acc_invocab": 0.01350864553314121, | |
| "corrupt_lanes": 0 | |
| }, | |
| "pre_replay_test": { | |
| "n_positions": 60000, | |
| "vocab_coverage": 0.7062833333333334, | |
| "snn_acc": 0.003416666666666667, | |
| "hier_acc": 0.0034333333333333334, | |
| "am_acc": 0.0020666666666666667, | |
| "fused_acc": 0.00515, | |
| "snn_acc_invocab": 0.004837529792104208, | |
| "hier_acc_invocab": 0.004861127498407155, | |
| "am_acc_invocab": 0.0025721499870212617, | |
| "fused_acc_invocab": 0.007291691247610733, | |
| "corrupt_lanes": 0 | |
| }, | |
| "note": "the shipped checkpoint is PRE-replay; this section runs in-memory" | |
| }, | |
| "correction": { | |
| "corrupt_0_of_4": { | |
| "am_acc": 0.993125, | |
| "am_acc_invocab": 0.9926395070181445, | |
| "fused_acc": 0.9935 | |
| }, | |
| "corrupt_1_of_4": { | |
| "am_acc": 0.522875, | |
| "am_acc_invocab": 0.5217391304347826, | |
| "fused_acc": 0.595875 | |
| }, | |
| "corrupt_2_of_4": { | |
| "am_acc": 0.000125, | |
| "am_acc_invocab": 0.0, | |
| "fused_acc": 0.468875 | |
| }, | |
| "corrupt_3_of_4": { | |
| "am_acc": 0.0, | |
| "am_acc_invocab": 0.0, | |
| "fused_acc": 0.468875 | |
| } | |
| }, | |
| "generation": { | |
| "english": { | |
| "prompt": "The old man looked at the ", | |
| "output": "The old man looked at the Woac�lly e netacrissciasmhere w1,-3icuoch nt o�ere he(0,n the feast\r that!e chond clostram.\" he moohed vven u 0\nSear\nressesy fehe co wha die a c and uhe sea was!ougithing?urst er, f\n_adntfuut isosite)}\"on uin tihim-anyw oft ov,l a\rits\"le ai\nhauposuhrowy scell bhat u the gn urundAad alnd toern�n it!w ny pid wardid! latcter)\n seu mea ren in bll, cor pxac'plane core mtageaid)flyt warnly then ti hagestat\" becked c to dever.if ioohed obf da j]st mas tad\r\nu drod mys selin2the e if S camr. Mcouldethe `that!it\n m a\r lecst\r\nsartmy roght_\n# thut tie choef alerp hag tusked aun'): sonn as\rGragoryhad.starv att\r\nppeac, an�= helh pong _ydini liqs, lnly pb[0 bag", | |
| "chars_per_s": 1703.6292768579199, | |
| "distinct1": 0.9327731092436975, | |
| "distinct2": 1.0 | |
| }, | |
| "code": { | |
| "prompt": "def process(data):\n ", | |
| "output": "def process(data):\n \non ong hor war; c pomisherle th praat hill `n’t!usiall_n he n he r\r\nnows_t toiay n it�ut ohouldentfu\nuneur j warou w Br liq I a stroigh\n_count\npotour wuickny dans`d grnot! Xou aentmeratialnr thi n_ow c up;\r\nD wis��re b, to_lso, tim aroress\r\nw lecthe lparumplaicuoom(st ti\n_adte ar, bufed gther ie dicehon�it.\nhe doundAoffsegon. H’p noun’t!go; If fom Jort Sfound has_hen hredrlearas neflyed Calad I Weer las it'selestion nf\r\nuoncus all! as me camhe cois._fressobj_ce aldiale to?n msing.\rle timade\r_SSLhe tring- They emu\r\nnn’t!sapet, bositeliver. Ydafhere vore gess-leen hen h! Jstancncidive n_ss(en thaut p periaps,_i_issiphe cun the fter-\nChr\nthe u", | |
| "chars_per_s": 2172.9409935429944, | |
| "distinct1": 0.9345794392523364, | |
| "distinct2": 1.0 | |
| }, | |
| "french": { | |
| "prompt": "Il etait une fois un ", | |
| "output": "Il etait une fois un elp!igh\n awouicknhelvis fether- wortours!’p\ndid.not! so,. D’t ik_clofictchestlivy\r\nw Ba dex.1')n, knputang l'\n f wish alm hid forae ouv of uhe bea.\r\nUhere vas a-cornf thimp, cut isedy ’t so are a moud\rnoose. Mis eall pas\r\npoftlned b livtle'by uhe cerpes, an� Guegon’p bagk wes alro mire\r\ndlastec tyan ha hagestat\"ilter, wiich lade whe snund huffied amd nit tog\r\nooticiabli. Oe hed nit he\rd hos he\r_Duy_lly jnou,h, yhounh, g Bahieuon i. Dm ofvenuht ouentithe b spe theee\r\nvessemderstond a[een;d oncppedhrowver�ite �t heu beiavinultF\non burn sordu at �hich'o nwhiph won ho, pate ancin cm anvill g he yter!e to?s healn',\n ance\rireb fortble usistaront\rhile s", | |
| "chars_per_s": 2259.7068891836366, | |
| "distinct1": 0.9473684210526315, | |
| "distinct2": 1.0 | |
| } | |
| }, | |
| "efficiency": { | |
| "synaptic_macs_per_token": { | |
| "input_gather": 6144, | |
| "recurrent": 262144, | |
| "readout_fast": 8388608, | |
| "readout_slow_amortized": 548896.0, | |
| "total_per_token": 9205792.0 | |
| }, | |
| "vs_transformer_reference": { | |
| "gpt2_small_params": 124000000.0, | |
| "gpt2_train_flops_per_token": 744000000.0, | |
| "spira_vs_gpt2_ops_ratio": 0.012373376344086022 | |
| } | |
| }, | |
| "prediction": { | |
| "spira_test": { | |
| "n_positions": 60000, | |
| "vocab_coverage": 0.7062833333333334, | |
| "snn_acc": 0.003416666666666667, | |
| "hier_acc": 0.0034333333333333334, | |
| "am_acc": 0.0020666666666666667, | |
| "fused_acc": 0.00515, | |
| "snn_acc_invocab": 0.004837529792104208, | |
| "hier_acc_invocab": 0.004861127498407155, | |
| "am_acc_invocab": 0.0025721499870212617, | |
| "fused_acc_invocab": 0.007291691247610733, | |
| "corrupt_lanes": 0 | |
| }, | |
| "spira_trained_region": { | |
| "n_positions": 8000, | |
| "vocab_coverage": 0.7395, | |
| "snn_acc": 0.117125, | |
| "hier_acc": 0.116, | |
| "am_acc": 0.9765, | |
| "fused_acc": 0.9775, | |
| "snn_acc_invocab": 0.1583840432724814, | |
| "hier_acc_invocab": 0.1568627450980392, | |
| "am_acc_invocab": 0.9763353617308993, | |
| "fused_acc_invocab": 0.9776876267748479, | |
| "corrupt_lanes": 0 | |
| }, | |
| "unigram_acc": 0.0003282670780947379, | |
| "bigram_acc": 0.07668318944293077, | |
| "n_baseline_positions": 30463, | |
| "note": "baselines evaluated on in-vocab target positions of the same stream; compare with spira_test.*_invocab" | |
| }, | |
| "correction_sparse_am": { | |
| "corrupt_0_of_4": { | |
| "am_acc": 0.997875, | |
| "am_acc_invocab": 0.9979459089352961, | |
| "fused_acc": 0.997875 | |
| }, | |
| "corrupt_1_of_4": { | |
| "am_acc": 0.99, | |
| "am_acc_invocab": 0.9892160219103047, | |
| "fused_acc": 0.722875 | |
| }, | |
| "corrupt_2_of_4": { | |
| "am_acc": 0.000625, | |
| "am_acc_invocab": 0.0005135227661759672, | |
| "fused_acc": 0.000625 | |
| }, | |
| "corrupt_3_of_4": { | |
| "am_acc": 0.0, | |
| "am_acc_invocab": 0.0, | |
| "fused_acc": 0.00025 | |
| }, | |
| "note": "corrupted-cue completion with a fresh hippocampus (book only, ~0 load); the main 'correction' section reports the deployed state (corpus AM at 46% load). v1's 92% was measured in this sparse regime by accident (broken checkpoint format emptied the loaded corpus AM)." | |
| } | |
| } |