Upload 41 files

07_22_2024_Conway_2_State_Jump_Rot_Pos_On_Masking_On_Broad_Entrpoy_Homog_2024-07-23 10-37-31/07_22_2024_Conway_2_State_Jump_Rot_Pos_On_Masking_On_Broad_Entrpoy_Homog_info.txt

@@ -0,0 +1,428 @@
+Model Name: 07_22_2024_Conway_2_State_Jump_Rot_Pos_On_Masking_On_Broad_Entrpoy_Homog
+Model Created @ 2024-07-23 10-37-31 Eastern Time
+Number of trainable parameters: 12735744
+Model Architecture:
+AutoregressiveWrapper(
+  (net): TransformerWrapper(
+    (token_emb): TokenEmbedding(
+      (emb): Embedding(256, 256)
+    )
+    (post_emb_norm): Identity()
+    (emb_dropout): Dropout(p=0.0, inplace=False)
+    (project_emb): Identity()
+    (attn_layers): Decoder(
+      (layers): ModuleList(
+        (0): ModuleList(
+          (0): ModuleList(
+            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+            (1-2): 2 x None
+          )
+          (1): Attention(
+            (to_q): Linear(in_features=256, out_features=512, bias=False)
+            (to_k): Linear(in_features=256, out_features=512, bias=False)
+            (to_v): Linear(in_features=256, out_features=512, bias=False)
+            (attend): Attend(
+              (attn_dropout): Dropout(p=0.0, inplace=False)
+            )
+            (to_out): Linear(in_features=512, out_features=256, bias=False)
+          )
+          (2): Residual()
+        )
+        (1): ModuleList(
+          (0): ModuleList(
+            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+            (1-2): 2 x None
+          )
+          (1): FeedForward(
+            (ff): Sequential(
+              (0): Sequential(
+                (0): Linear(in_features=256, out_features=1024, bias=True)
+                (1): GELU(approximate='none')
+              )
+              (1): Dropout(p=0.0, inplace=False)
+              (2): Linear(in_features=1024, out_features=256, bias=True)
+            )
+          )
+          (2): Residual()
+        )
+        (2): ModuleList(
+          (0): ModuleList(
+            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+            (1-2): 2 x None
+          )
+          (1): Attention(
+            (to_q): Linear(in_features=256, out_features=512, bias=False)
+            (to_k): Linear(in_features=256, out_features=512, bias=False)
+            (to_v): Linear(in_features=256, out_features=512, bias=False)
+            (attend): Attend(
+              (attn_dropout): Dropout(p=0.0, inplace=False)
+            )
+            (to_out): Linear(in_features=512, out_features=256, bias=False)
+          )
+          (2): Residual()
+        )
+        (3): ModuleList(
+          (0): ModuleList(
+            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+            (1-2): 2 x None
+          )
+          (1): FeedForward(
+            (ff): Sequential(
+              (0): Sequential(
+                (0): Linear(in_features=256, out_features=1024, bias=True)
+                (1): GELU(approximate='none')
+              )
+              (1): Dropout(p=0.0, inplace=False)
+              (2): Linear(in_features=1024, out_features=256, bias=True)
+            )
+          )
+          (2): Residual()
+        )
+        (4): ModuleList(
+          (0): ModuleList(
+            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+            (1-2): 2 x None
+          )
+          (1): Attention(
+            (to_q): Linear(in_features=256, out_features=512, bias=False)
+            (to_k): Linear(in_features=256, out_features=512, bias=False)
+            (to_v): Linear(in_features=256, out_features=512, bias=False)
+            (attend): Attend(
+              (attn_dropout): Dropout(p=0.0, inplace=False)
+            )
+            (to_out): Linear(in_features=512, out_features=256, bias=False)
+          )
+          (2): Residual()
+        )
+        (5): ModuleList(
+          (0): ModuleList(
+            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+            (1-2): 2 x None
+          )
+          (1): FeedForward(
+            (ff): Sequential(
+              (0): Sequential(
+                (0): Linear(in_features=256, out_features=1024, bias=True)
+                (1): GELU(approximate='none')
+              )
+              (1): Dropout(p=0.0, inplace=False)
+              (2): Linear(in_features=1024, out_features=256, bias=True)
+            )
+          )
+          (2): Residual()
+        )
+        (6): ModuleList(
+          (0): ModuleList(
+            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+            (1-2): 2 x None
+          )
+          (1): Attention(
+            (to_q): Linear(in_features=256, out_features=512, bias=False)
+            (to_k): Linear(in_features=256, out_features=512, bias=False)
+            (to_v): Linear(in_features=256, out_features=512, bias=False)
+            (attend): Attend(
+              (attn_dropout): Dropout(p=0.0, inplace=False)
+            )
+            (to_out): Linear(in_features=512, out_features=256, bias=False)
+          )
+          (2): Residual()
+        )
+        (7): ModuleList(
+          (0): ModuleList(
+            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+            (1-2): 2 x None
+          )
+          (1): FeedForward(
+            (ff): Sequential(
+              (0): Sequential(
+                (0): Linear(in_features=256, out_features=1024, bias=True)
+                (1): GELU(approximate='none')
+              )
+              (1): Dropout(p=0.0, inplace=False)
+              (2): Linear(in_features=1024, out_features=256, bias=True)
+            )
+          )
+          (2): Residual()
+        )
+        (8): ModuleList(
+          (0): ModuleList(
+            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+            (1-2): 2 x None
+          )
+          (1): Attention(
+            (to_q): Linear(in_features=256, out_features=512, bias=False)
+            (to_k): Linear(in_features=256, out_features=512, bias=False)
+            (to_v): Linear(in_features=256, out_features=512, bias=False)
+            (attend): Attend(
+              (attn_dropout): Dropout(p=0.0, inplace=False)
+            )
+            (to_out): Linear(in_features=512, out_features=256, bias=False)
+          )
+          (2): Residual()
+        )
+        (9): ModuleList(
+          (0): ModuleList(
+            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+            (1-2): 2 x None
+          )
+          (1): FeedForward(
+            (ff): Sequential(
+              (0): Sequential(
+                (0): Linear(in_features=256, out_features=1024, bias=True)
+                (1): GELU(approximate='none')
+              )
+              (1): Dropout(p=0.0, inplace=False)
+              (2): Linear(in_features=1024, out_features=256, bias=True)
+            )
+          )
+          (2): Residual()
+        )
+        (10): ModuleList(
+          (0): ModuleList(
+            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+            (1-2): 2 x None
+          )
+          (1): Attention(
+            (to_q): Linear(in_features=256, out_features=512, bias=False)
+            (to_k): Linear(in_features=256, out_features=512, bias=False)
+            (to_v): Linear(in_features=256, out_features=512, bias=False)
+            (attend): Attend(
+              (attn_dropout): Dropout(p=0.0, inplace=False)
+            )
+            (to_out): Linear(in_features=512, out_features=256, bias=False)
+          )
+          (2): Residual()
+        )
+        (11): ModuleList(
+          (0): ModuleList(
+            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+            (1-2): 2 x None
+          )
+          (1): FeedForward(
+            (ff): Sequential(
+              (0): Sequential(
+                (0): Linear(in_features=256, out_features=1024, bias=True)
+                (1): GELU(approximate='none')
+              )
+              (1): Dropout(p=0.0, inplace=False)
+              (2): Linear(in_features=1024, out_features=256, bias=True)
+            )
+          )
+          (2): Residual()
+        )
+        (12): ModuleList(
+          (0): ModuleList(
+            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+            (1-2): 2 x None
+          )
+          (1): Attention(
+            (to_q): Linear(in_features=256, out_features=512, bias=False)
+            (to_k): Linear(in_features=256, out_features=512, bias=False)
+            (to_v): Linear(in_features=256, out_features=512, bias=False)
+            (attend): Attend(
+              (attn_dropout): Dropout(p=0.0, inplace=False)
+            )
+            (to_out): Linear(in_features=512, out_features=256, bias=False)
+          )
+          (2): Residual()
+        )
+        (13): ModuleList(
+          (0): ModuleList(
+            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+            (1-2): 2 x None
+          )
+          (1): FeedForward(
+            (ff): Sequential(
+              (0): Sequential(
+                (0): Linear(in_features=256, out_features=1024, bias=True)
+                (1): GELU(approximate='none')
+              )
+              (1): Dropout(p=0.0, inplace=False)
+              (2): Linear(in_features=1024, out_features=256, bias=True)
+            )
+          )
+          (2): Residual()
+        )
+        (14): ModuleList(
+          (0): ModuleList(
+            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+            (1-2): 2 x None
+          )
+          (1): Attention(
+            (to_q): Linear(in_features=256, out_features=512, bias=False)
+            (to_k): Linear(in_features=256, out_features=512, bias=False)
+            (to_v): Linear(in_features=256, out_features=512, bias=False)
+            (attend): Attend(
+              (attn_dropout): Dropout(p=0.0, inplace=False)
+            )
+            (to_out): Linear(in_features=512, out_features=256, bias=False)
+          )
+          (2): Residual()
+        )
+        (15): ModuleList(
+          (0): ModuleList(
+            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+            (1-2): 2 x None
+          )
+          (1): FeedForward(
+            (ff): Sequential(
+              (0): Sequential(
+                (0): Linear(in_features=256, out_features=1024, bias=True)
+                (1): GELU(approximate='none')
+              )
+              (1): Dropout(p=0.0, inplace=False)
+              (2): Linear(in_features=1024, out_features=256, bias=True)
+            )
+          )
+          (2): Residual()
+        )
+        (16): ModuleList(
+          (0): ModuleList(
+            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+            (1-2): 2 x None
+          )
+          (1): Attention(
+            (to_q): Linear(in_features=256, out_features=512, bias=False)
+            (to_k): Linear(in_features=256, out_features=512, bias=False)
+            (to_v): Linear(in_features=256, out_features=512, bias=False)
+            (attend): Attend(
+              (attn_dropout): Dropout(p=0.0, inplace=False)
+            )
+            (to_out): Linear(in_features=512, out_features=256, bias=False)
+          )
+          (2): Residual()
+        )
+        (17): ModuleList(
+          (0): ModuleList(
+            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+            (1-2): 2 x None
+          )
+          (1): FeedForward(
+            (ff): Sequential(
+              (0): Sequential(
+                (0): Linear(in_features=256, out_features=1024, bias=True)
+                (1): GELU(approximate='none')
+              )
+              (1): Dropout(p=0.0, inplace=False)
+              (2): Linear(in_features=1024, out_features=256, bias=True)
+            )
+          )
+          (2): Residual()
+        )
+        (18): ModuleList(
+          (0): ModuleList(
+            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+            (1-2): 2 x None
+          )
+          (1): Attention(
+            (to_q): Linear(in_features=256, out_features=512, bias=False)
+            (to_k): Linear(in_features=256, out_features=512, bias=False)
+            (to_v): Linear(in_features=256, out_features=512, bias=False)
+            (attend): Attend(
+              (attn_dropout): Dropout(p=0.0, inplace=False)
+            )
+            (to_out): Linear(in_features=512, out_features=256, bias=False)
+          )
+          (2): Residual()
+        )
+        (19): ModuleList(
+          (0): ModuleList(
+            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+            (1-2): 2 x None
+          )
+          (1): FeedForward(
+            (ff): Sequential(
+              (0): Sequential(
+                (0): Linear(in_features=256, out_features=1024, bias=True)
+                (1): GELU(approximate='none')
+              )
+              (1): Dropout(p=0.0, inplace=False)
+              (2): Linear(in_features=1024, out_features=256, bias=True)
+            )
+          )
+          (2): Residual()
+        )
+        (20): ModuleList(
+          (0): ModuleList(
+            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+            (1-2): 2 x None
+          )
+          (1): Attention(
+            (to_q): Linear(in_features=256, out_features=512, bias=False)
+            (to_k): Linear(in_features=256, out_features=512, bias=False)
+            (to_v): Linear(in_features=256, out_features=512, bias=False)
+            (attend): Attend(
+              (attn_dropout): Dropout(p=0.0, inplace=False)
+            )
+            (to_out): Linear(in_features=512, out_features=256, bias=False)
+          )
+          (2): Residual()
+        )
+        (21): ModuleList(
+          (0): ModuleList(
+            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+            (1-2): 2 x None
+          )
+          (1): FeedForward(
+            (ff): Sequential(
+              (0): Sequential(
+                (0): Linear(in_features=256, out_features=1024, bias=True)
+                (1): GELU(approximate='none')
+              )
+              (1): Dropout(p=0.0, inplace=False)
+              (2): Linear(in_features=1024, out_features=256, bias=True)
+            )
+          )
+          (2): Residual()
+        )
+        (22): ModuleList(
+          (0): ModuleList(
+            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+            (1-2): 2 x None
+          )
+          (1): Attention(
+            (to_q): Linear(in_features=256, out_features=512, bias=False)
+            (to_k): Linear(in_features=256, out_features=512, bias=False)
+            (to_v): Linear(in_features=256, out_features=512, bias=False)
+            (attend): Attend(
+              (attn_dropout): Dropout(p=0.0, inplace=False)
+            )
+            (to_out): Linear(in_features=512, out_features=256, bias=False)
+          )
+          (2): Residual()
+        )
+        (23): ModuleList(
+          (0): ModuleList(
+            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+            (1-2): 2 x None
+          )
+          (1): FeedForward(
+            (ff): Sequential(
+              (0): Sequential(
+                (0): Linear(in_features=256, out_features=1024, bias=True)
+                (1): GELU(approximate='none')
+              )
+              (1): Dropout(p=0.0, inplace=False)
+              (2): Linear(in_features=1024, out_features=256, bias=True)
+            )
+          )
+          (2): Residual()
+        )
+      )
+      (rotary_pos_emb): RotaryEmbedding()
+      (final_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+    )
+    (to_logits): Linear(in_features=256, out_features=256, bias=False)
+  )
+)
+Model Parameters:
+num_tokens: 256
+max_seq_len: 2071
+dim: 256
+depth: 12
+heads: 8
+attn_dim_head: 64
+rotary_pos_emb: True
+attn_flash: True
+
+Note: July-22-This model is testing whether or not masking will affect the osutcome of the model. We are still using a broad entropy/ homogeneous training set for this run.

07_22_2024_Conway_2_State_Jump_Rot_Pos_On_Masking_On_Broad_Entrpoy_Homog_2024-07-23 10-37-31/Conway_GPT_v7_epoch_10.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f0a7404efe04f04e8de78e7b9b4851594b9f0a73f5a199382f6be44ff941ae2c
+size 51000826

07_22_2024_Conway_2_State_Jump_Rot_Pos_On_Masking_On_Broad_Entrpoy_Homog_2024-07-23 10-37-31/Conway_GPT_v7_epoch_12.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f53b0ce18b220866010134b8b2e60f91ca95b2ade0aafaff988c73b4173bd13f
+size 51000826

07_22_2024_Conway_2_State_Jump_Rot_Pos_On_Masking_On_Broad_Entrpoy_Homog_2024-07-23 10-37-31/Conway_GPT_v7_epoch_14.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d80b7df6c2515b89aa9f250d33060994e2c3bc5c32a77918e27f957d5b164b87
+size 51000826

07_22_2024_Conway_2_State_Jump_Rot_Pos_On_Masking_On_Broad_Entrpoy_Homog_2024-07-23 10-37-31/Conway_GPT_v7_epoch_16.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:434f8dccf0fd0d71e10c91f7b4146004fa0018e50b2e4f5cc4cbbe009f98a516
+size 51000826

07_22_2024_Conway_2_State_Jump_Rot_Pos_On_Masking_On_Broad_Entrpoy_Homog_2024-07-23 10-37-31/Conway_GPT_v7_epoch_18.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9be1ff42c135741e832f028bc4ce8d86eef4db5fa5dd434c47979266c1dcb4ca
+size 51000826

07_22_2024_Conway_2_State_Jump_Rot_Pos_On_Masking_On_Broad_Entrpoy_Homog_2024-07-23 10-37-31/Conway_GPT_v7_epoch_2.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:89db1da561f6f9bce4f296f71507311516dd0074aa9fa1b0dd0bcea7826b2f9b
+size 51000634

07_22_2024_Conway_2_State_Jump_Rot_Pos_On_Masking_On_Broad_Entrpoy_Homog_2024-07-23 10-37-31/Conway_GPT_v7_epoch_20.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e6537fdf571aa0d1e8c487af07fad7fa61a352f781b5f393a37c4489cc4cd26
+size 51000826

07_22_2024_Conway_2_State_Jump_Rot_Pos_On_Masking_On_Broad_Entrpoy_Homog_2024-07-23 10-37-31/Conway_GPT_v7_epoch_22.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:53265a179b8e767d34253f16f119e1098d862ffdc5e2b472766dbe9c0739e84f
+size 51000826

07_22_2024_Conway_2_State_Jump_Rot_Pos_On_Masking_On_Broad_Entrpoy_Homog_2024-07-23 10-37-31/Conway_GPT_v7_epoch_24.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd0d58a56df6e1f189d1ce75cc6859298a2692d3747751308986892b312ddb30
+size 51000826

07_22_2024_Conway_2_State_Jump_Rot_Pos_On_Masking_On_Broad_Entrpoy_Homog_2024-07-23 10-37-31/Conway_GPT_v7_epoch_26.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:151383dea7ee7aade68fc3b529e37564ce6f83d77c9e51d892aea25cef557f27
+size 51000826

07_22_2024_Conway_2_State_Jump_Rot_Pos_On_Masking_On_Broad_Entrpoy_Homog_2024-07-23 10-37-31/Conway_GPT_v7_epoch_28.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d06bd0a7ca00ccf37447aeb800d12f082124b0cd4411afd3338689ac06f8b362
+size 51000826

07_22_2024_Conway_2_State_Jump_Rot_Pos_On_Masking_On_Broad_Entrpoy_Homog_2024-07-23 10-37-31/Conway_GPT_v7_epoch_30.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:495dbeb6eee67a18c57b0464d79b44e752a4b2af50df007e97288f5f6ad0dc0b
+size 51000826

07_22_2024_Conway_2_State_Jump_Rot_Pos_On_Masking_On_Broad_Entrpoy_Homog_2024-07-23 10-37-31/Conway_GPT_v7_epoch_32.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d8524ad6b2139d38095d7c01387285226c99d7e5d773dc37db0961d398d92cf
+size 51000826

07_22_2024_Conway_2_State_Jump_Rot_Pos_On_Masking_On_Broad_Entrpoy_Homog_2024-07-23 10-37-31/Conway_GPT_v7_epoch_34.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9fc286f03bdea264f7762b668d555df62696411e3f0f7766adf3e20580a51bb
+size 51000826

07_22_2024_Conway_2_State_Jump_Rot_Pos_On_Masking_On_Broad_Entrpoy_Homog_2024-07-23 10-37-31/Conway_GPT_v7_epoch_36.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4dee45d28cc8fbecfc84c69c43197ea1a0fd8589132e786b1a8cab5a55a42a31
+size 51000826

07_22_2024_Conway_2_State_Jump_Rot_Pos_On_Masking_On_Broad_Entrpoy_Homog_2024-07-23 10-37-31/Conway_GPT_v7_epoch_38.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99c43c5627e61e995424e3c50359b918192bc9e8b27586fd7cd5fb2739719adc
+size 51000826

07_22_2024_Conway_2_State_Jump_Rot_Pos_On_Masking_On_Broad_Entrpoy_Homog_2024-07-23 10-37-31/Conway_GPT_v7_epoch_4.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:839c244713f259a4de2779a6f4b0b6b2ff8d8e1a48ca03653110435403f58e8f
+size 51000634

07_22_2024_Conway_2_State_Jump_Rot_Pos_On_Masking_On_Broad_Entrpoy_Homog_2024-07-23 10-37-31/Conway_GPT_v7_epoch_40.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9758032867d77997a2e625e85c103f6ad8e1cbdeeca76681b842463a447c1550
+size 51000826

07_22_2024_Conway_2_State_Jump_Rot_Pos_On_Masking_On_Broad_Entrpoy_Homog_2024-07-23 10-37-31/Conway_GPT_v7_epoch_42.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea6fccf32571daab0c04f40309dffc82d43b73d4886482d7b63976814ca4a8f7
+size 51000826

07_22_2024_Conway_2_State_Jump_Rot_Pos_On_Masking_On_Broad_Entrpoy_Homog_2024-07-23 10-37-31/Conway_GPT_v7_epoch_44.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da3467dbf4f796df323f17d878cb1463132d6176b005d8b9e7aa1b22038f5fb2
+size 51000826

07_22_2024_Conway_2_State_Jump_Rot_Pos_On_Masking_On_Broad_Entrpoy_Homog_2024-07-23 10-37-31/Conway_GPT_v7_epoch_46.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e80f6f9648014a6064a4f9178cc2160121a5390d168b0ea237efa8255bc2a36
+size 51000826

07_22_2024_Conway_2_State_Jump_Rot_Pos_On_Masking_On_Broad_Entrpoy_Homog_2024-07-23 10-37-31/Conway_GPT_v7_epoch_48.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:57443942053f3e04b173e0492e5152782e2391758af90b342cbd7ad8808f666b
+size 51000826

07_22_2024_Conway_2_State_Jump_Rot_Pos_On_Masking_On_Broad_Entrpoy_Homog_2024-07-23 10-37-31/Conway_GPT_v7_epoch_50.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:43ff077757252cfc7485cdb2c71cf8f661ebcbde2ddde5d647587e30930ccfc3
+size 51000826

07_22_2024_Conway_2_State_Jump_Rot_Pos_On_Masking_On_Broad_Entrpoy_Homog_2024-07-23 10-37-31/Conway_GPT_v7_epoch_6.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16fb2e4381d1748d7503c2181f876974ba4e0f3f1baf4f3bf44b8a4b4ffe4537
+size 51000634

07_22_2024_Conway_2_State_Jump_Rot_Pos_On_Masking_On_Broad_Entrpoy_Homog_2024-07-23 10-37-31/Conway_GPT_v7_epoch_8.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32da1a08333ae6f53b402443066bb6f4b7d33805780ee958c464414c714d0e70
+size 51000634

08_14_2024_Conway_2_State_Jump_Rot_Pos_On_Masking_Off_High_Entrpoy_Homog_2024-08-14 12-24-10/08_14_2024_Conway_2_State_Jump_Rot_Pos_On_Masking_Off_High_Entrpoy_Homog_info.txt

@@ -0,0 +1,428 @@
+Model Name: 08_14_2024_Conway_2_State_Jump_Rot_Pos_On_Masking_Off_High_Entrpoy_Homog
+Model Created @ 2024-08-14 12-24-10 Eastern Time
+Number of trainable parameters: 12735744
+Model Architecture:
+AutoregressiveWrapper(
+  (net): TransformerWrapper(
+    (token_emb): TokenEmbedding(
+      (emb): Embedding(256, 256)
+    )
+    (post_emb_norm): Identity()
+    (emb_dropout): Dropout(p=0.0, inplace=False)
+    (project_emb): Identity()
+    (attn_layers): Decoder(
+      (layers): ModuleList(
+        (0): ModuleList(
+          (0): ModuleList(
+            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+            (1-2): 2 x None
+          )
+          (1): Attention(
+            (to_q): Linear(in_features=256, out_features=512, bias=False)
+            (to_k): Linear(in_features=256, out_features=512, bias=False)
+            (to_v): Linear(in_features=256, out_features=512, bias=False)
+            (attend): Attend(
+              (attn_dropout): Dropout(p=0.0, inplace=False)
+            )
+            (to_out): Linear(in_features=512, out_features=256, bias=False)
+          )
+          (2): Residual()
+        )
+        (1): ModuleList(
+          (0): ModuleList(
+            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+            (1-2): 2 x None
+          )
+          (1): FeedForward(
+            (ff): Sequential(
+              (0): Sequential(
+                (0): Linear(in_features=256, out_features=1024, bias=True)
+                (1): GELU(approximate='none')
+              )
+              (1): Dropout(p=0.0, inplace=False)
+              (2): Linear(in_features=1024, out_features=256, bias=True)
+            )
+          )
+          (2): Residual()
+        )
+        (2): ModuleList(
+          (0): ModuleList(
+            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+            (1-2): 2 x None
+          )
+          (1): Attention(
+            (to_q): Linear(in_features=256, out_features=512, bias=False)
+            (to_k): Linear(in_features=256, out_features=512, bias=False)
+            (to_v): Linear(in_features=256, out_features=512, bias=False)
+            (attend): Attend(
+              (attn_dropout): Dropout(p=0.0, inplace=False)
+            )
+            (to_out): Linear(in_features=512, out_features=256, bias=False)
+          )
+          (2): Residual()
+        )
+        (3): ModuleList(
+          (0): ModuleList(
+            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+            (1-2): 2 x None
+          )
+          (1): FeedForward(
+            (ff): Sequential(
+              (0): Sequential(
+                (0): Linear(in_features=256, out_features=1024, bias=True)
+                (1): GELU(approximate='none')
+              )
+              (1): Dropout(p=0.0, inplace=False)
+              (2): Linear(in_features=1024, out_features=256, bias=True)
+            )
+          )
+          (2): Residual()
+        )
+        (4): ModuleList(
+          (0): ModuleList(
+            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+            (1-2): 2 x None
+          )
+          (1): Attention(
+            (to_q): Linear(in_features=256, out_features=512, bias=False)
+            (to_k): Linear(in_features=256, out_features=512, bias=False)
+            (to_v): Linear(in_features=256, out_features=512, bias=False)
+            (attend): Attend(
+              (attn_dropout): Dropout(p=0.0, inplace=False)
+            )
+            (to_out): Linear(in_features=512, out_features=256, bias=False)
+          )
+          (2): Residual()
+        )
+        (5): ModuleList(
+          (0): ModuleList(
+            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+            (1-2): 2 x None
+          )
+          (1): FeedForward(
+            (ff): Sequential(
+              (0): Sequential(
+                (0): Linear(in_features=256, out_features=1024, bias=True)
+                (1): GELU(approximate='none')
+              )
+              (1): Dropout(p=0.0, inplace=False)
+              (2): Linear(in_features=1024, out_features=256, bias=True)
+            )
+          )
+          (2): Residual()
+        )
+        (6): ModuleList(
+          (0): ModuleList(
+            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+            (1-2): 2 x None
+          )
+          (1): Attention(
+            (to_q): Linear(in_features=256, out_features=512, bias=False)
+            (to_k): Linear(in_features=256, out_features=512, bias=False)
+            (to_v): Linear(in_features=256, out_features=512, bias=False)
+            (attend): Attend(
+              (attn_dropout): Dropout(p=0.0, inplace=False)
+            )
+            (to_out): Linear(in_features=512, out_features=256, bias=False)
+          )
+          (2): Residual()
+        )
+        (7): ModuleList(
+          (0): ModuleList(
+            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+            (1-2): 2 x None
+          )
+          (1): FeedForward(
+            (ff): Sequential(
+              (0): Sequential(
+                (0): Linear(in_features=256, out_features=1024, bias=True)
+                (1): GELU(approximate='none')
+              )
+              (1): Dropout(p=0.0, inplace=False)
+              (2): Linear(in_features=1024, out_features=256, bias=True)
+            )
+          )
+          (2): Residual()
+        )
+        (8): ModuleList(
+          (0): ModuleList(
+            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+            (1-2): 2 x None
+          )
+          (1): Attention(
+            (to_q): Linear(in_features=256, out_features=512, bias=False)
+            (to_k): Linear(in_features=256, out_features=512, bias=False)
+            (to_v): Linear(in_features=256, out_features=512, bias=False)
+            (attend): Attend(
+              (attn_dropout): Dropout(p=0.0, inplace=False)
+            )
+            (to_out): Linear(in_features=512, out_features=256, bias=False)
+          )
+          (2): Residual()
+        )
+        (9): ModuleList(
+          (0): ModuleList(
+            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+            (1-2): 2 x None
+          )
+          (1): FeedForward(
+            (ff): Sequential(
+              (0): Sequential(
+                (0): Linear(in_features=256, out_features=1024, bias=True)
+                (1): GELU(approximate='none')
+              )
+              (1): Dropout(p=0.0, inplace=False)
+              (2): Linear(in_features=1024, out_features=256, bias=True)
+            )
+          )
+          (2): Residual()
+        )
+        (10): ModuleList(
+          (0): ModuleList(
+            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+            (1-2): 2 x None
+          )
+          (1): Attention(
+            (to_q): Linear(in_features=256, out_features=512, bias=False)
+            (to_k): Linear(in_features=256, out_features=512, bias=False)
+            (to_v): Linear(in_features=256, out_features=512, bias=False)
+            (attend): Attend(
+              (attn_dropout): Dropout(p=0.0, inplace=False)
+            )
+            (to_out): Linear(in_features=512, out_features=256, bias=False)
+          )
+          (2): Residual()
+        )
+        (11): ModuleList(
+          (0): ModuleList(
+            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+            (1-2): 2 x None
+          )
+          (1): FeedForward(
+            (ff): Sequential(
+              (0): Sequential(
+                (0): Linear(in_features=256, out_features=1024, bias=True)
+                (1): GELU(approximate='none')
+              )
+              (1): Dropout(p=0.0, inplace=False)
+              (2): Linear(in_features=1024, out_features=256, bias=True)
+            )
+          )
+          (2): Residual()
+        )
+        (12): ModuleList(
+          (0): ModuleList(
+            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+            (1-2): 2 x None
+          )
+          (1): Attention(
+            (to_q): Linear(in_features=256, out_features=512, bias=False)
+            (to_k): Linear(in_features=256, out_features=512, bias=False)
+            (to_v): Linear(in_features=256, out_features=512, bias=False)
+            (attend): Attend(
+              (attn_dropout): Dropout(p=0.0, inplace=False)
+            )
+            (to_out): Linear(in_features=512, out_features=256, bias=False)
+          )
+          (2): Residual()
+        )
+        (13): ModuleList(
+          (0): ModuleList(
+            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+            (1-2): 2 x None
+          )
+          (1): FeedForward(
+            (ff): Sequential(
+              (0): Sequential(
+                (0): Linear(in_features=256, out_features=1024, bias=True)
+                (1): GELU(approximate='none')
+              )
+              (1): Dropout(p=0.0, inplace=False)
+              (2): Linear(in_features=1024, out_features=256, bias=True)
+            )
+          )
+          (2): Residual()
+        )
+        (14): ModuleList(
+          (0): ModuleList(
+            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+            (1-2): 2 x None
+          )
+          (1): Attention(
+            (to_q): Linear(in_features=256, out_features=512, bias=False)
+            (to_k): Linear(in_features=256, out_features=512, bias=False)
+            (to_v): Linear(in_features=256, out_features=512, bias=False)
+            (attend): Attend(
+              (attn_dropout): Dropout(p=0.0, inplace=False)
+            )
+            (to_out): Linear(in_features=512, out_features=256, bias=False)
+          )
+          (2): Residual()
+        )
+        (15): ModuleList(
+          (0): ModuleList(
+            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+            (1-2): 2 x None
+          )
+          (1): FeedForward(
+            (ff): Sequential(
+              (0): Sequential(
+                (0): Linear(in_features=256, out_features=1024, bias=True)
+                (1): GELU(approximate='none')
+              )
+              (1): Dropout(p=0.0, inplace=False)
+              (2): Linear(in_features=1024, out_features=256, bias=True)
+            )
+          )
+          (2): Residual()
+        )
+        (16): ModuleList(
+          (0): ModuleList(
+            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+            (1-2): 2 x None
+          )
+          (1): Attention(
+            (to_q): Linear(in_features=256, out_features=512, bias=False)
+            (to_k): Linear(in_features=256, out_features=512, bias=False)
+            (to_v): Linear(in_features=256, out_features=512, bias=False)
+            (attend): Attend(
+              (attn_dropout): Dropout(p=0.0, inplace=False)
+            )
+            (to_out): Linear(in_features=512, out_features=256, bias=False)
+          )
+          (2): Residual()
+        )
+        (17): ModuleList(
+          (0): ModuleList(
+            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+            (1-2): 2 x None
+          )
+          (1): FeedForward(
+            (ff): Sequential(
+              (0): Sequential(
+                (0): Linear(in_features=256, out_features=1024, bias=True)
+                (1): GELU(approximate='none')
+              )
+              (1): Dropout(p=0.0, inplace=False)
+              (2): Linear(in_features=1024, out_features=256, bias=True)
+            )
+          )
+          (2): Residual()
+        )
+        (18): ModuleList(
+          (0): ModuleList(
+            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+            (1-2): 2 x None
+          )
+          (1): Attention(
+            (to_q): Linear(in_features=256, out_features=512, bias=False)
+            (to_k): Linear(in_features=256, out_features=512, bias=False)
+            (to_v): Linear(in_features=256, out_features=512, bias=False)
+            (attend): Attend(
+              (attn_dropout): Dropout(p=0.0, inplace=False)
+            )
+            (to_out): Linear(in_features=512, out_features=256, bias=False)
+          )
+          (2): Residual()
+        )
+        (19): ModuleList(
+          (0): ModuleList(
+            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+            (1-2): 2 x None
+          )
+          (1): FeedForward(
+            (ff): Sequential(
+              (0): Sequential(
+                (0): Linear(in_features=256, out_features=1024, bias=True)
+                (1): GELU(approximate='none')
+              )
+              (1): Dropout(p=0.0, inplace=False)
+              (2): Linear(in_features=1024, out_features=256, bias=True)
+            )
+          )
+          (2): Residual()
+        )
+        (20): ModuleList(
+          (0): ModuleList(
+            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+            (1-2): 2 x None
+          )
+          (1): Attention(
+            (to_q): Linear(in_features=256, out_features=512, bias=False)
+            (to_k): Linear(in_features=256, out_features=512, bias=False)
+            (to_v): Linear(in_features=256, out_features=512, bias=False)
+            (attend): Attend(
+              (attn_dropout): Dropout(p=0.0, inplace=False)
+            )
+            (to_out): Linear(in_features=512, out_features=256, bias=False)
+          )
+          (2): Residual()
+        )
+        (21): ModuleList(
+          (0): ModuleList(
+            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+            (1-2): 2 x None
+          )
+          (1): FeedForward(
+            (ff): Sequential(
+              (0): Sequential(
+                (0): Linear(in_features=256, out_features=1024, bias=True)
+                (1): GELU(approximate='none')
+              )
+              (1): Dropout(p=0.0, inplace=False)
+              (2): Linear(in_features=1024, out_features=256, bias=True)
+            )
+          )
+          (2): Residual()
+        )
+        (22): ModuleList(
+          (0): ModuleList(
+            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+            (1-2): 2 x None
+          )
+          (1): Attention(
+            (to_q): Linear(in_features=256, out_features=512, bias=False)
+            (to_k): Linear(in_features=256, out_features=512, bias=False)
+            (to_v): Linear(in_features=256, out_features=512, bias=False)
+            (attend): Attend(
+              (attn_dropout): Dropout(p=0.0, inplace=False)
+            )
+            (to_out): Linear(in_features=512, out_features=256, bias=False)
+          )
+          (2): Residual()
+        )
+        (23): ModuleList(
+          (0): ModuleList(
+            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+            (1-2): 2 x None
+          )
+          (1): FeedForward(
+            (ff): Sequential(
+              (0): Sequential(
+                (0): Linear(in_features=256, out_features=1024, bias=True)
+                (1): GELU(approximate='none')
+              )
+              (1): Dropout(p=0.0, inplace=False)
+              (2): Linear(in_features=1024, out_features=256, bias=True)
+            )
+          )
+          (2): Residual()
+        )
+      )
+      (rotary_pos_emb): RotaryEmbedding()
+      (final_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+    )
+    (to_logits): Linear(in_features=256, out_features=256, bias=False)
+  )
+)
+Model Parameters:
+num_tokens: 256
+max_seq_len: 2071
+dim: 256
+depth: 12
+heads: 8
+attn_dim_head: 64
+rotary_pos_emb: True
+attn_flash: True
+
+Note:Aug 14, 2024 - this model is a test using rot pos enc, no extra masking, and high entropy data on a 32 by 32 toroidal grid. Poor performance on the test set is anticipated.

08_14_2024_Conway_2_State_Jump_Rot_Pos_On_Masking_Off_High_Entrpoy_Homog_2024-08-14 12-24-10/Conway_GPT_v7_epoch_18.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f31cc4e972c4c39401bab9141abb0ec759ac3ce7c7926fcc6a0688347cbbc86a
+size 51000826

08_14_2024_Conway_2_State_Jump_Rot_Pos_On_Masking_Off_High_Entrpoy_Homog_2024-08-14 12-24-10/Conway_GPT_v7_epoch_24.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7d8eed1245a57d1f1cd750c14d563b74e37dcc89a6a7851d9bd8007673b6a35
+size 51000826

08_14_2024_Conway_2_State_Jump_Rot_Pos_On_Masking_Off_High_Entrpoy_Homog_2024-08-14 12-24-10/Conway_GPT_v7_epoch_26.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e4ef11b905f81f1d4281e710675fafeaf717a21adfb829089ff9f2f4f21491a3
+size 51000826

08_14_2024_Conway_2_State_Jump_Rot_Pos_On_Masking_Off_High_Entrpoy_Homog_2024-08-14 12-24-10/Conway_GPT_v7_epoch_30.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3dcf2219da1458017aa9cf0f744e17523297d9332666512e4d36a1e3c5740e8e
+size 51000826

08_14_2024_Conway_2_State_Jump_Rot_Pos_On_Masking_Off_High_Entrpoy_Homog_2024-08-14 12-24-10/Conway_GPT_v7_epoch_32.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3db6059b9acd53e96d997791d532ff4fe0486976fbb90f71db840f3c93f90cf1
+size 51000826

08_14_2024_Conway_2_State_Jump_Rot_Pos_On_Masking_Off_High_Entrpoy_Homog_2024-08-14 12-24-10/Conway_GPT_v7_epoch_34.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:605b45658b0597c53f206c240a9b1c2d5225c8cc4f95f8591f85de35bd3ef367
+size 51000826

08_14_2024_Conway_2_State_Jump_Rot_Pos_On_Masking_Off_High_Entrpoy_Homog_2024-08-14 12-24-10/Conway_GPT_v7_epoch_40.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e1c1256d5edc4660a9c2437cb87463d45c938cdacc211502bf30f79ab9fbf62
+size 51000826

08_14_2024_Conway_2_State_Jump_Rot_Pos_On_Masking_Off_High_Entrpoy_Homog_2024-08-14 12-24-10/Conway_GPT_v7_epoch_42.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16af24853c96b9a2af347f77ca7263f17ce8c8fcc1c43ca380749f70b0ebc164
+size 51000826

08_14_2024_Conway_2_State_Jump_Rot_Pos_On_Masking_Off_High_Entrpoy_Homog_2024-08-14 12-24-10/Conway_GPT_v7_epoch_44.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e263f49bba8cde18936615c2963908d08cff4fdf10133047dc815fd7c421bd2d
+size 51000826

08_14_2024_Conway_2_State_Jump_Rot_Pos_On_Masking_Off_High_Entrpoy_Homog_2024-08-14 12-24-10/Conway_GPT_v7_epoch_48.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc2ea470325b9ef1cc52d89759a1c32bc0f33e347b9feb4366fc6c31410e6394
+size 51000826

08_14_2024_Conway_2_State_Jump_Rot_Pos_On_Masking_Off_High_Entrpoy_Homog_2024-08-14 12-24-10/Conway_GPT_v7_epoch_50.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2bce33853b2ae2ec3e35eff23c7472337e5a45f95065cd668d26edc5d5eeeef
+size 51000826

08_14_2024_Conway_2_State_Jump_Rot_Pos_On_Masking_Off_High_Entrpoy_Homog_2024-08-14 12-24-10/Conway_GPT_v7_epoch_8.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa8331e5ee3a744705cd0b2cece4d994e3a32c812a04f5b8d7d17c0dbbe470ee
+size 51000634