ReactiveAI
/

RxT-Beta-Memory-Attention-Base

model_hub_mixin

pytorch_model_hub_mixin

🇪🇺 Region: EU

Model card Files Files and versions

AdamF92 commited on Feb 23

Commit

5dfd38e

·

verified ·

1 Parent(s): fa73705

Epoch 0 - Val loss -0.9678

Files changed (2) hide show

config.json +2 -1
model.safetensors +2 -2

config.json CHANGED Viewed

@@ -25,7 +25,8 @@
   "residual_gate_slot_status_type": "mean",
   "residual_gate_type": "elementwise",
   "residual_per_slot_gate": true,
-  "seq_len": 4096,
   "stm_size": 4096,
   "use_flash_attention": true,
   "use_gated_residual": true,

   "residual_gate_slot_status_type": "mean",
   "residual_gate_type": "elementwise",
   "residual_per_slot_gate": true,
+  "rope_base": 100000,
+  "seq_len": 8192,
   "stm_size": 4096,
   "use_flash_attention": true,
   "use_gated_residual": true,

model.safetensors CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:30cc3f3d7857bde9e95fdd545d25bde7d367872ee12a250000d2655d0c16ba9b
-size 132917168

 version https://git-lfs.github.com/spec/v1
+oid sha256:7d1fac900a7c9256fbc94994934bb0ead9487bf4fbf0fb572fd54027cd04e7b8
+size 133048240