Add precomputed results for p=9

.gitattributes

@@ -74,3 +74,10 @@ precomputed_results/p_031/p031_phase_align_approx1.png filter=lfs diff=lfs merge
 precomputed_results/p_031/p031_phase_align_approx2.png filter=lfs diff=lfs merge=lfs -text
 precomputed_results/p_031/p031_single_freq_quad.png filter=lfs diff=lfs merge=lfs -text
 precomputed_results/p_031/p031_single_freq_relu.png filter=lfs diff=lfs merge=lfs -text
+precomputed_results/p_009/p009_lineplot_in.png filter=lfs diff=lfs merge=lfs -text
+precomputed_results/p_009/p009_lineplot_out.png filter=lfs diff=lfs merge=lfs -text
+precomputed_results/p_009/p009_phase_align_approx1.png filter=lfs diff=lfs merge=lfs -text
+precomputed_results/p_009/p009_phase_align_approx2.png filter=lfs diff=lfs merge=lfs -text
+precomputed_results/p_009/p009_phase_align_quad.png filter=lfs diff=lfs merge=lfs -text
+precomputed_results/p_009/p009_single_freq_quad.png filter=lfs diff=lfs merge=lfs -text
+precomputed_results/p_009/p009_single_freq_relu.png filter=lfs diff=lfs merge=lfs -text

precomputed_results/p_009/p009_logits_interactive.json

@@ -0,0 +1 @@
+{"pairs": [[0, 0], [1, 0], [2, 0], [3, 0], [4, 0], [5, 0], [6, 0], [7, 0], [8, 0]], "correct_answers": [0, 1, 2, 3, 4, 5, 6, 7, 8], "logits": [[6.225973129272461, -0.42272233963012695, 0.43343156576156616, -0.029559314250946045, -0.7333509922027588, -0.2956560552120209, -0.14366161823272705, -1.0385000705718994, -0.42348670959472656], [1.0056105852127075, 6.976110935211182, 0.3679962158203125, -1.2561242580413818, -0.1353779435157776, -0.9141680598258972, -1.2088048458099365, -1.8852434158325195, -1.3283441066741943], [0.4929104447364807, -1.090764045715332, 7.166391372680664, 0.044684022665023804, 0.9852460026741028, -0.24106650054454803, -1.4014173746109009, -1.0038530826568604, -0.4544332027435303], [-0.9498991370201111, -1.2759813070297241, -1.1999588012695312, 6.500079154968262, -0.8172470927238464, -0.9236076474189758, -0.42031610012054443, -0.9317978620529175, -0.9628182649612427], [0.668920636177063, -0.6930823922157288, -1.1369543075561523, -2.616199016571045, 6.708083629608154, -0.9288009405136108, -0.6079546213150024, -0.2772216796875, -0.1483113020658493], [1.0260368585586548, 0.40010982751846313, -1.4863635301589966, -0.5103893280029297, -1.3092235326766968, 7.060690879821777, -1.075552225112915, -0.3440393805503845, -0.196609228849411], [-0.6294415593147278, -1.250927209854126, -0.5517414212226868, -0.7401149272918701, -0.86446213722229, -0.5588915348052979, 6.655684471130371, -0.7264660596847534, -0.9677294492721558], [0.2382621169090271, -0.7757992744445801, -1.6823946237564087, -1.5511764287948608, -0.6635268926620483, 0.45432135462760925, -0.74192214012146, 6.850571155548096, -0.07050973176956177], [0.3924682140350342, -1.5858150720596313, -0.5013530254364014, -0.8568658828735352, -0.906563937664032, -0.6670979857444763, -1.786723017692566, 0.471148282289505, 6.753711223602295]], "output_classes": [0, 1, 2, 3, 4, 5, 6, 7, 8]}

precomputed_results/p_009/p009_metadata.json

@@ -0,0 +1,82 @@
+{
+  "prime": 9,
+  "d_mlp": 512,
+  "training_runs": {
+    "standard": {
+      "act_type": "ReLU",
+      "lr": 5e-05,
+      "weight_decay": 0,
+      "num_epochs": 5000,
+      "frac_train": 1.0,
+      "init_type": "random",
+      "init_scale": 0.1,
+      "optimizer": "AdamW"
+    },
+    "grokking": {
+      "act_type": "ReLU",
+      "lr": 0.0001,
+      "weight_decay": 2.0,
+      "num_epochs": 50000,
+      "frac_train": 0.75,
+      "init_type": "random",
+      "init_scale": 0.1,
+      "optimizer": "AdamW"
+    },
+    "quad_random": {
+      "act_type": "Quad",
+      "lr": 5e-05,
+      "weight_decay": 0,
+      "num_epochs": 5000,
+      "frac_train": 1.0,
+      "init_type": "random",
+      "init_scale": 0.1,
+      "optimizer": "AdamW"
+    },
+    "quad_single_freq": {
+      "act_type": "Quad",
+      "lr": 0.1,
+      "weight_decay": 0,
+      "num_epochs": 10000,
+      "frac_train": 1.0,
+      "init_type": "single-freq",
+      "init_scale": 0.02,
+      "optimizer": "SGD"
+    },
+    "relu_single_freq": {
+      "act_type": "ReLU",
+      "lr": 0.01,
+      "weight_decay": 0,
+      "num_epochs": 10000,
+      "frac_train": 1.0,
+      "init_type": "single-freq",
+      "init_scale": 0.002,
+      "optimizer": "SGD"
+    }
+  },
+  "final_metrics": {
+    "standard": {
+      "train_acc": 1.0,
+      "test_acc": 1.0,
+      "train_loss": 0.0063102878630161285,
+      "test_loss": 0.0063102878630161285
+    },
+    "quad_random": {
+      "train_acc": 1.0,
+      "test_acc": 1.0,
+      "train_loss": 0.0009679726208560169,
+      "test_loss": 0.0009679726208560169
+    },
+    "quad_single_freq": {
+      "train_acc": 1.0,
+      "test_acc": 1.0,
+      "train_loss": 0.0006334710051305592,
+      "test_loss": 0.0006334710051305592
+    },
+    "relu_single_freq": {
+      "train_acc": 1.0,
+      "test_acc": 1.0,
+      "train_loss": 1.90578031539917,
+      "test_loss": 1.90578031539917
+    }
+  }
+}

precomputed_results/p_009/p009_neuron_spectra.json

@@ -0,0 +1 @@
+{"fourier_basis_names": ["Const", "cos 1", "sin 1", "cos 2", "sin 2", "cos 3", "sin 3", "cos 4", "sin 4"], "neurons": {"neuron_0": {"global_index": 194, "dominant_freq": 1, "fourier_magnitudes_in": [0.04556606709957123, 0.7075158357620239, 0.01477275975048542, 0.06358962506055832, 0.029284019023180008, 0.19491873681545258, 0.037093278020620346, 0.06358937919139862, 0.018153563141822815], "fourier_magnitudes_out": [0.12855540215969086, 0.6557085514068604, 0.0019015403231605887, 0.14729472994804382, 0.0380711555480957, 0.1944604068994522, 0.0059073735028505325, 0.17148973047733307, 0.03219221904873848]}, "neuron_1": {"global_index": 462, "dominant_freq": 2, "fourier_magnitudes_in": [0.014603463001549244, 0.09475266933441162, 0.09797219932079315, 0.6773117184638977, 0.09537884593009949, 0.1419333666563034, 0.09086033701896667, 0.023294536396861076, 0.10845661163330078], "fourier_magnitudes_out": [0.06521307677030563, 0.012767848558723927, 0.14206629991531372, 0.5898648500442505, 0.23595809936523438, 0.05329431593418121, 0.2055550068616867, 0.03077513910830021, 0.10581227391958237]}, "neuron_2": {"global_index": 187, "dominant_freq": 3, "fourier_magnitudes_in": [0.2679072916507721, 0.013741005212068558, 0.022147903218865395, 0.030267465859651566, 0.009254897944629192, 0.7351370453834534, 0.005455600097775459, 0.0008451284957118332, 0.0011635097907856107], "fourier_magnitudes_out": [0.27944251894950867, 0.014594748616218567, 0.017899110913276672, 0.026402968913316727, 0.04373963177204132, 0.7651328444480896, 0.01958717405796051, 0.03470202907919884, 0.061075758188962936]}, "neuron_3": {"global_index": 454, "dominant_freq": 4, "fourier_magnitudes_in": [0.0644875317811966, 0.04758625477552414, 0.04107993096113205, 0.04665329307317734, 0.13201099634170532, 0.0935632735490799, 0.2020636796951294, 0.044916704297065735, 0.6721738576889038], "fourier_magnitudes_out": [0.0019817871507257223, 0.028786981478333473, 0.0400628037750721, 0.07547871023416519, 0.07952755689620972, 0.18762347102165222, 0.11578872799873352, 0.6922436952590942, 0.1151832714676857]}, "neuron_4": {"global_index": 229, "dominant_freq": 1, "fourier_magnitudes_in": [0.007823213934898376, 0.00848317239433527, 0.6542045474052429, 0.0019336719997227192, 0.05336437374353409, 0.005509205162525177, 0.19237396121025085, 0.010373630560934544, 0.10163972526788712], "fourier_magnitudes_out": [0.010092316195368767, 0.6610961556434631, 0.13506387174129486, 0.02691642940044403, 0.026329388841986656, 0.11019282042980194, 0.16101327538490295, 0.04585515707731247, 0.08854486793279648]}, "neuron_5": {"global_index": 305, "dominant_freq": 2, "fourier_magnitudes_in": [0.09287853538990021, 0.09342782199382782, 0.08276072889566422, 0.12007783353328705, 0.6376784443855286, 0.12719489634037018, 0.18489407002925873, 0.12791258096694946, 0.03645048663020134], "fourier_magnitudes_out": [0.014028268866240978, 0.00665174750611186, 0.12490340322256088, 0.6215116381645203, 0.200845867395401, 0.21179541945457458, 0.07161443680524826, 0.046846676617860794, 0.0024303016252815723]}, "neuron_6": {"global_index": 143, "dominant_freq": 3, "fourier_magnitudes_in": [0.24318702518939972, 0.01683039218187332, 0.016376161947846413, 0.021652307361364365, 0.018389523029327393, 0.7044114470481873, 0.023189393803477287, 0.0041490099392831326, 0.015950337052345276], "fourier_magnitudes_out": [0.20374228060245514, 0.03435537591576576, 0.02851407788693905, 0.01778445392847061, 0.04882989078760147, 0.6915525197982788, 0.06344490498304367, 0.0009965592762455344, 0.0003306497528683394]}, "neuron_7": {"global_index": 55, "dominant_freq": 4, "fourier_magnitudes_in": [0.008687797002494335, 0.02102574147284031, 0.051543012261390686, 0.05665332451462746, 0.08549883961677551, 0.1605195850133896, 0.09977293014526367, 0.6402161717414856, 0.10331743955612183], "fourier_magnitudes_out": [0.0671042874455452, 0.07280283421278, 0.015759073197841644, 0.0036165493074804544, 0.14137911796569824, 0.147895947098732, 0.15355251729488373, 0.6940174102783203, 0.18873102962970734]}, "neuron_8": {"global_index": 321, "dominant_freq": 1, "fourier_magnitudes_in": [0.015606512315571308, 0.6482261419296265, 0.12996789813041687, 0.014802457764744759, 0.04838133230805397, 0.15714262425899506, 0.07434836030006409, 0.06638091802597046, 0.09020452201366425], "fourier_magnitudes_out": [0.13653092086315155, 0.5379490852355957, 0.35423699021339417, 0.00902603380382061, 0.06401548534631729, 0.025478189811110497, 0.13637642562389374, 0.1257622241973877, 0.005323226097971201]}, "neuron_9": {"global_index": 479, "dominant_freq": 2, "fourier_magnitudes_in": [0.043047983199357986, 0.03357972577214241, 0.11081115901470184, 0.6329029202461243, 0.08245278894901276, 0.11723186075687408, 0.10282392054796219, 0.04521496221423149, 0.07510803639888763], "fourier_magnitudes_out": [0.037037137895822525, 0.10274569690227509, 0.046994779258966446, 0.6587742567062378, 0.07715131342411041, 0.21582429111003876, 0.07790253311395645, 0.07500675320625305, 0.019327519461512566]}, "neuron_10": {"global_index": 423, "dominant_freq": 3, "fourier_magnitudes_in": [0.227887824177742, 0.006603864952921867, 0.03722241893410683, 0.02213597483932972, 0.0051648905500769615, 0.6697366833686829, 0.029999522492289543, 0.02892250008881092, 0.008863229304552078], "fourier_magnitudes_out": [0.1529143899679184, 0.08375406265258789, 0.021797504276037216, 0.023128241300582886, 0.0205706637352705, 0.7245540618896484, 0.13639536499977112, 0.0161798857152462, 0.08216705918312073]}, "neuron_11": {"global_index": 169, "dominant_freq": 4, "fourier_magnitudes_in": [0.006920161657035351, 0.057928718626499176, 0.14484986662864685, 0.07099834084510803, 0.07699048519134521, 0.09032289683818817, 0.06344086676836014, 0.63166743516922, 0.10641064494848251], "fourier_magnitudes_out": [0.05375326797366142, 0.052196260541677475, 0.032804880291223526, 0.06684347242116928, 0.011910884641110897, 0.06621582806110382, 0.10604951530694962, 0.6707427501678467, 0.3055097162723541]}, "neuron_12": {"global_index": 12, "dominant_freq": 1, "fourier_magnitudes_in": [0.0016631855396553874, 0.6280817985534668, 0.11299477517604828, 0.013779918663203716, 0.03257715702056885, 0.16222554445266724, 0.09501868486404419, 0.05556334927678108, 0.06557807326316833], "fourier_magnitudes_out": [0.05867023020982742, 0.5556254386901855, 0.4024606943130493, 0.031530480831861496, 0.07172378152608871, 0.049981407821178436, 0.18301628530025482, 0.13198404014110565, 0.009522262960672379]}, "neuron_13": {"global_index": 100, "dominant_freq": 2, "fourier_magnitudes_in": [0.004790664184838533, 0.05843503028154373, 0.0770687535405159, 0.6302691698074341, 0.10213211178779602, 0.15655295550823212, 0.07772661000490189, 0.00575099466368556, 0.049994636327028275], "fourier_magnitudes_out": [0.09429313242435455, 0.036216944456100464, 0.00015795837680343539, 0.6198323369026184, 0.2023189663887024, 0.14326666295528412, 0.10106367617845535, 0.08336293697357178, 0.011841433122754097]}, "neuron_14": {"global_index": 470, "dominant_freq": 3, "fourier_magnitudes_in": [0.2347741425037384, 0.0057095433585345745, 0.005282387603074312, 0.001597126480191946, 0.00568043440580368, 0.6649599075317383, 0.007264954037964344, 0.0017257127910852432, 0.000850675452966243], "fourier_magnitudes_out": [0.10598786175251007, 0.010004456155002117, 0.0496375747025013, 0.042280104011297226, 0.042828645557165146, 0.6770966053009033, 0.03471143916249275, 0.07155783474445343, 0.015630172565579414]}, "neuron_15": {"global_index": 490, "dominant_freq": 4, "fourier_magnitudes_in": [0.015618078410625458, 0.009593948721885681, 0.07284942269325256, 0.011990257538855076, 0.13356254994869232, 0.018863124772906303, 0.16548992693424225, 0.008327699266374111, 0.6286185383796692], "fourier_magnitudes_out": [0.053604550659656525, 0.09811535477638245, 0.007479648105800152, 0.16013196110725403, 0.015547624789178371, 0.2239801585674286, 0.05095396563410759, 0.7230298519134521, 0.04352070763707161]}, "neuron_16": {"global_index": 204, "dominant_freq": 1, "fourier_magnitudes_in": [4.753288976644399e-06, 0.6255450248718262, 0.11029492318630219, 0.01645151525735855, 0.04519115760922432, 0.16385583579540253, 0.09460344910621643, 0.06240963190793991, 0.07439173758029938], "fourier_magnitudes_out": [0.02945011854171753, 0.6611279249191284, 0.25245630741119385, 0.03432958573102951, 0.0075307744555175304, 0.04700898379087448, 0.10317488014698029, 0.041911300271749496, 0.03427358716726303]}, "neuron_17": {"global_index": 206, "dominant_freq": 2, "fourier_magnitudes_in": [1.6427964055765187e-06, 0.06299953162670135, 0.0750863254070282, 0.6298892498016357, 0.11107011884450912, 0.1646782010793686, 0.0950813740491867, 0.01675749011337757, 0.04603338986635208], "fourier_magnitudes_out": [0.03424932435154915, 0.001119195600040257, 0.04855567216873169, 0.6086327433586121, 0.19734898209571838, 0.14668184518814087, 0.09288988262414932, 0.026265379041433334, 0.06573057919740677]}, "neuron_18": {"global_index": 511, "dominant_freq": 3, "fourier_magnitudes_in": [0.23276901245117188, 4.444841579243075e-06, 1.7078823475458194e-06, 1.333908471679024e-06, 2.816646258452238e-07, 0.6583778262138367, 1.7445668163418304e-06, 4.5167030293669086e-07, 1.46244894949632e-06], "fourier_magnitudes_out": [0.23774519562721252, 0.00557613093405962, 0.02705295942723751, 0.031218549236655235, 0.027251027524471283, 0.7312431335449219, 0.042479075491428375, 0.07423919439315796, 0.04400001838803291]}, "neuron_19": {"global_index": 42, "dominant_freq": 4, "fourier_magnitudes_in": [0.0028952781576663256, 0.01483286265283823, 0.05233801528811455, 0.06704171746969223, 0.08040229976177216, 0.1703699827194214, 0.09600017219781876, 0.628589928150177, 0.10648850351572037], "fourier_magnitudes_out": [0.04571361467242241, 0.057035110890865326, 0.04022432118654251, 0.05519811436533928, 0.033818602561950684, 0.057209525257349014, 0.15763752162456512, 0.610483705997467, 0.29397743940353394]}}}

precomputed_results/p_009/p009_overview.json

@@ -0,0 +1 @@
+{"std_epochs": [0, 200, 400, 600, 800, 1000, 1200, 1400, 1600, 1800, 2000, 2200, 2400, 2600, 2800, 3000, 3200, 3400, 3600, 3800, 4000, 4200, 4400, 4600, 4800], "std_ipr": [0.39460474252700806, 0.41012611985206604, 0.4442066550254822, 0.48425188660621643, 0.5224266052246094, 0.5551608800888062, 0.5817892551422119, 0.6034306287765503, 0.6212095022201538, 0.6357609033584595, 0.6478009819984436, 0.6582198143005371, 0.667588472366333, 0.6761120557785034, 0.6835013628005981, 0.690115213394165, 0.6960419416427612, 0.7012717723846436, 0.7059794664382935, 0.710098147392273, 0.7138159275054932, 0.7171499729156494, 0.7200719714164734, 0.7227238416671753, 0.7251557111740112], "std_train_loss": [2.1961324214935303, 2.1486423015594482, 2.089108943939209, 2.0132877826690674, 1.9193778038024902, 1.8077919483184814, 1.6797209978103638, 1.537805199623108, 1.3847278356552124, 1.22403085231781, 1.060030460357666, 0.8977756500244141, 0.7420556545257568, 0.5972861647605896, 0.4675646722316742, 0.35559743642807007, 0.26278191804885864, 0.18895843625068665, 0.13241712749004364, 0.09066127240657806, 0.06076653301715851, 0.03994970768690109, 0.025796949863433838, 0.016376368701457977, 0.010232066735625267]}