Release 15 DAMI paper checkpoints and model card

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 DAMI authors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,3 +1,86 @@
 ---
 license: mit
+language:
+- en
+tags:
+- dami
+- linear-attention
+- long-context
+- initialization
+- ruler
 ---
+
+# DAMI: Dual-Axis Memory Initialization for Enhanced Long-Context Retrieval in Gated Linear Attention
+
+Zhangchi Feng · Harry Jiang · Yangyifei Luo · Ray Zhang · Richong Zhang
+
+[Project page](https://speedai.com/research/dami) · [Code, evaluation data and scripts](https://cnb.cool/speed-ai/research/DAMI)
+
+DAMI allocates memory timescales across heads and input responses across layers to improve long-context retrieval. This repository contains the 15 paper checkpoints: five architectures (GDN, KDA, GDN2, Mamba-2, RWKV-7) with three initialization conditions (Random, PoST, DAMI). Each model has approximately 100M parameters and was trained for approximately 5B tokens with a 4096-token training context.
+
+## Checkpoints
+
+All checkpoints are stored under `checkpoints/`, using the directory names below. Each directory contains the full model weights (`model.safetensors`), `config.json`, `generation_config.json`, `tokenizer.json`, and `tokenizer_config.json`.
+
+| Architecture | Random | PoST | DAMI |
+| --- | --- | --- | --- |
+| GDN | `main_gdn_random` | `main_gdn_PoST` | `main_gdn_DAMI` |
+| KDA | `main_kda_random` | `main_kda_PoST` | `main_kda_DAMI` |
+| GDN2 | `main_gdn2_random` | `main_gdn2_PoST` | `main_gdn2_DAMI` |
+| Mamba-2 | `main_mamba2_random` | `main_mamba2_PoST` | `main_mamba2_DAMI` |
+| RWKV-7 | `main_rwkv7_random` | `main_rwkv7_PoST` | `main_rwkv7_DAMI` |
+
+## Download and evaluation
+
+Clone the code repository and install its dependencies as described in its README. From the code repository root, download the checkpoints:
+
+```python
+from huggingface_hub import snapshot_download
+
+snapshot_download(
+    repo_id="SpeedAI-Lab/DAMI",
+    local_dir=".",
+    allow_patterns=["checkpoints/**"],
+)
+```
+
+To download only GDN + DAMI, use `allow_patterns=["checkpoints/main_gdn_DAMI/*"]`. Access to this repository requires authentication while it is private.
+
+Run all 15 checkpoints on the fixed RULER evaluation data in Linux/WSL with CUDA:
+
+```bash
+python evaluate_ruler.py --preflight --all --models-root checkpoints
+python evaluate_ruler.py --all --models-root checkpoints \
+  --output-dir outputs/ruler-retest --skip-done
+```
+
+For a single checkpoint, replace `--all` with `--architecture gdn --initialization dami`. The evaluator includes the architecture-specific FLA and RWKV adaptations required by these checkpoints; use the provided loading path for evaluation.
+
+The code repository includes 5,500 fixed samples across 11 RULER task/length settings and 82,500 original predictions. Evaluation uses greedy decoding with up to 128 new tokens. Scores use case-insensitive answer containment; the average weights all 11 settings equally.
+
+## RULER results
+
+Average accuracy (%), as reported in the paper:
+
+| Architecture | Random | PoST | DAMI |
+| --- | ---: | ---: | ---: |
+| GDN | 30.42 | 33.49 | 44.36 |
+| KDA | 26.38 | 33.67 | 43.09 |
+| GDN2 | 30.27 | 33.47 | 43.71 |
+| Mamba-2 | 9.95 | 13.73 | 24.42 |
+| RWKV-7 | 23.87 | 20.69 | 29.16 |
+
+## Citation
+
+```bibtex
+@misc{feng2026dami,
+  title={DAMI: Dual-Axis Memory Initialization for Enhanced Long-Context Retrieval in Gated Linear Attention},
+  author={Feng, Zhangchi and Jiang, Harry and Luo, Yangyifei and Zhang, Ray and Zhang, Richong},
+  year={2026},
+  url={https://speedai.com/research/dami}
+}
+```
+
+## License
+
+These checkpoints are released under the MIT License. See the LICENSE file. Benchmark attribution and data notices are provided in the code repository.

checkpoints/main_gdn2_DAMI/config.json

@@ -0,0 +1,47 @@
+{
+  "allow_neg_eigval": false,
+  "architectures": [
+    "GDN2ForCausalLM"
+  ],
+  "attn": null,
+  "attn_mode": "chunk",
+  "attnres_block_size": null,
+  "bos_token_id": 1,
+  "conv_size": 4,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "expand_v": 2,
+  "fuse_cross_entropy": true,
+  "fuse_linear_cross_entropy": false,
+  "fuse_norm": true,
+  "fuse_swiglu": true,
+  "head_dim": 64,
+  "head_init_design": null,
+  "hidden_act": "swish",
+  "hidden_ratio": 4,
+  "hidden_size": 512,
+  "initialization_manifest": {
+    "A_log_rule": "GDN v3_layerdt3 design target A_log (PoST tau by head + layerwise sensitivity)",
+    "design_config": "/root/linear-attention/flame/configs/gated_deltanet_100M_v3_layerdt3_a.json",
+    "dt_bias_rule": "head means aligned to gdn_v3_layerdt3 with deterministic small channel noise; channel_std=0.03",
+    "experiment": "v5_layerdt3_b",
+    "family": "GDN2",
+    "gdn_source_csv": null
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": null,
+  "max_position_embeddings": 4096,
+  "model_type": "gdn2",
+  "norm_eps": 1e-06,
+  "num_heads": 6,
+  "num_hidden_layers": 18,
+  "num_v_heads": 6,
+  "pad_token_id": 0,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.9.0",
+  "use_cache": true,
+  "use_gate": true,
+  "use_l2warp": false,
+  "use_short_conv": true,
+  "vocab_size": 32000
+}

checkpoints/main_gdn2_DAMI/generation_config.json

@@ -0,0 +1,10 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "pad_token_id": 0,
+  "transformers_version": "5.9.0",
+  "use_cache": true
+}

checkpoints/main_gdn2_DAMI/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e5e93f40b6c97da4ca97dd039a2398db8b9db91d483336590735be783cdc1bb6
+size 449050472

checkpoints/main_gdn2_DAMI/tokenizer_config.json

@@ -0,0 +1,16 @@
+{
+  "add_prefix_space": null,
+  "backend": "tokenizers",
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": null,
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

checkpoints/main_gdn2_PoST/config.json

@@ -0,0 +1,46 @@
+{
+  "allow_neg_eigval": false,
+  "architectures": [
+    "GDN2ForCausalLM"
+  ],
+  "attn": null,
+  "attn_mode": "chunk",
+  "attnres_block_size": null,
+  "bos_token_id": 1,
+  "conv_size": 4,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "expand_v": 2,
+  "fuse_cross_entropy": true,
+  "fuse_linear_cross_entropy": false,
+  "fuse_norm": true,
+  "fuse_swiglu": true,
+  "head_dim": 64,
+  "head_init_design": null,
+  "hidden_act": "swish",
+  "hidden_ratio": 4,
+  "hidden_size": 512,
+  "initialization_manifest": {
+    "A_log_rule": "PoST head-column values; all other tensors copied from the same seeded base initialization",
+    "dt_bias_rule": "GDN-aligned head means with deterministic small channel noise; channel_std=0.03",
+    "experiment": "v5_post_a",
+    "family": "GDN2",
+    "gdn_source_csv": "/root/checkpoints/gdn/training/gdn100m-v3-random-5b-v3_random_a/initial_head_params.csv"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": null,
+  "max_position_embeddings": 4096,
+  "model_type": "gdn2",
+  "norm_eps": 1e-06,
+  "num_heads": 6,
+  "num_hidden_layers": 18,
+  "num_v_heads": 6,
+  "pad_token_id": 0,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.9.0",
+  "use_cache": true,
+  "use_gate": true,
+  "use_l2warp": false,
+  "use_short_conv": true,
+  "vocab_size": 32000
+}

checkpoints/main_gdn2_PoST/generation_config.json

@@ -0,0 +1,10 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "pad_token_id": 0,
+  "transformers_version": "5.9.0",
+  "use_cache": true
+}

checkpoints/main_gdn2_PoST/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:893d33dd84873b17586298a2ce3f34736ad7bebdcd2098042f13d25e61ee407d
+size 449050472

checkpoints/main_gdn2_PoST/tokenizer_config.json

@@ -0,0 +1,16 @@
+{
+  "add_prefix_space": null,
+  "backend": "tokenizers",
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": null,
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

checkpoints/main_gdn2_random/config.json

@@ -0,0 +1,46 @@
+{
+  "allow_neg_eigval": false,
+  "architectures": [
+    "GDN2ForCausalLM"
+  ],
+  "attn": null,
+  "attn_mode": "chunk",
+  "attnres_block_size": null,
+  "bos_token_id": 1,
+  "conv_size": 4,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "expand_v": 2,
+  "fuse_cross_entropy": true,
+  "fuse_linear_cross_entropy": false,
+  "fuse_norm": true,
+  "fuse_swiglu": true,
+  "head_dim": 64,
+  "head_init_design": null,
+  "hidden_act": "swish",
+  "hidden_ratio": 4,
+  "hidden_size": 512,
+  "initialization_manifest": {
+    "A_log_rule": "GDN v3_random_a source A_log",
+    "dt_bias_rule": "GDN-aligned head means with deterministic small channel noise; channel_std=0.03",
+    "experiment": "v5_random_a",
+    "family": "GDN2",
+    "gdn_source_csv": "/root/checkpoints/gdn/training/gdn100m-v3-random-5b-v3_random_a/initial_head_params.csv"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": null,
+  "max_position_embeddings": 4096,
+  "model_type": "gdn2",
+  "norm_eps": 1e-06,
+  "num_heads": 6,
+  "num_hidden_layers": 18,
+  "num_v_heads": 6,
+  "pad_token_id": 0,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.9.0",
+  "use_cache": true,
+  "use_gate": true,
+  "use_l2warp": false,
+  "use_short_conv": true,
+  "vocab_size": 32000
+}

checkpoints/main_gdn2_random/generation_config.json

@@ -0,0 +1,10 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "pad_token_id": 0,
+  "transformers_version": "5.9.0",
+  "use_cache": true
+}

checkpoints/main_gdn2_random/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6bfa974dec7c458b14f394d89ac33cee41086f3d7beda2bb6ab058652d7a875
+size 449050472

checkpoints/main_gdn2_random/tokenizer_config.json

@@ -0,0 +1,16 @@
+{
+  "add_prefix_space": null,
+  "backend": "tokenizers",
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": null,
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

checkpoints/main_gdn_DAMI/config.json

@@ -0,0 +1,42 @@
+{
+  "allow_neg_eigval": false,
+  "architectures": [
+    "GatedDeltaNetForCausalLM"
+  ],
+  "attn": null,
+  "attn_mode": "chunk",
+  "attnres_block_size": null,
+  "bos_token_id": 1,
+  "conv_size": 4,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "expand_v": 2,
+  "fuse_cross_entropy": true,
+  "fuse_linear_cross_entropy": false,
+  "fuse_norm": true,
+  "fuse_swiglu": true,
+  "head_dim": 64,
+  "head_init_design": null,
+  "head_retention_scale": null,
+  "hidden_act": "swish",
+  "hidden_ratio": 4,
+  "hidden_size": 512,
+  "initial_full_weights_path": null,
+  "initializer_range": 0.02,
+  "intermediate_size": null,
+  "max_position_embeddings": 4096,
+  "model_type": "gated_deltanet",
+  "norm_eps": 1e-06,
+  "num_heads": 6,
+  "num_hidden_layers": 18,
+  "num_v_heads": null,
+  "pad_token_id": 0,
+  "position_adaptive_scaling": null,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.9.0",
+  "use_cache": true,
+  "use_gate": true,
+  "use_l2warp": false,
+  "use_short_conv": true,
+  "vocab_size": 32000
+}

checkpoints/main_gdn_DAMI/generation_config.json

@@ -0,0 +1,10 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "pad_token_id": 0,
+  "transformers_version": "5.9.0",
+  "use_cache": true
+}

checkpoints/main_gdn_DAMI/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:13dfd749678ff1f0578e57765e08cf262903aa6fbedf07ceba679f9ab291e9e2
+size 415192624

checkpoints/main_gdn_DAMI/tokenizer_config.json

@@ -0,0 +1,16 @@
+{
+  "add_prefix_space": null,
+  "backend": "tokenizers",
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": null,
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

checkpoints/main_gdn_PoST/config.json

@@ -0,0 +1,42 @@
+{
+  "allow_neg_eigval": false,
+  "architectures": [
+    "GatedDeltaNetForCausalLM"
+  ],
+  "attn": null,
+  "attn_mode": "chunk",
+  "attnres_block_size": null,
+  "bos_token_id": 1,
+  "conv_size": 4,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "expand_v": 2,
+  "fuse_cross_entropy": true,
+  "fuse_linear_cross_entropy": false,
+  "fuse_norm": true,
+  "fuse_swiglu": true,
+  "head_dim": 64,
+  "head_init_design": null,
+  "head_retention_scale": null,
+  "hidden_act": "swish",
+  "hidden_ratio": 4,
+  "hidden_size": 512,
+  "initial_full_weights_path": null,
+  "initializer_range": 0.02,
+  "intermediate_size": null,
+  "max_position_embeddings": 4096,
+  "model_type": "gated_deltanet",
+  "norm_eps": 1e-06,
+  "num_heads": 6,
+  "num_hidden_layers": 18,
+  "num_v_heads": null,
+  "pad_token_id": 0,
+  "position_adaptive_scaling": null,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.9.0",
+  "use_cache": true,
+  "use_gate": true,
+  "use_l2warp": false,
+  "use_short_conv": true,
+  "vocab_size": 32000
+}

checkpoints/main_gdn_PoST/generation_config.json

@@ -0,0 +1,10 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "pad_token_id": 0,
+  "transformers_version": "5.9.0",
+  "use_cache": true
+}

checkpoints/main_gdn_PoST/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0875da62fdcf7760f134daab4593c6d7d250c82b9899da15344deb893ffb20ba
+size 415192624

checkpoints/main_gdn_PoST/tokenizer_config.json

@@ -0,0 +1,16 @@
+{
+  "add_prefix_space": null,
+  "backend": "tokenizers",
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": null,
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

checkpoints/main_gdn_random/config.json

@@ -0,0 +1,42 @@
+{
+  "allow_neg_eigval": false,
+  "architectures": [
+    "GatedDeltaNetForCausalLM"
+  ],
+  "attn": null,
+  "attn_mode": "chunk",
+  "attnres_block_size": null,
+  "bos_token_id": 1,
+  "conv_size": 4,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "expand_v": 2,
+  "fuse_cross_entropy": true,
+  "fuse_linear_cross_entropy": false,
+  "fuse_norm": true,
+  "fuse_swiglu": true,
+  "head_dim": 64,
+  "head_init_design": null,
+  "head_retention_scale": null,
+  "hidden_act": "swish",
+  "hidden_ratio": 4,
+  "hidden_size": 512,
+  "initial_full_weights_path": null,
+  "initializer_range": 0.02,
+  "intermediate_size": null,
+  "max_position_embeddings": 4096,
+  "model_type": "gated_deltanet",
+  "norm_eps": 1e-06,
+  "num_heads": 6,
+  "num_hidden_layers": 18,
+  "num_v_heads": null,
+  "pad_token_id": 0,
+  "position_adaptive_scaling": null,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.9.0",
+  "use_cache": true,
+  "use_gate": true,
+  "use_l2warp": false,
+  "use_short_conv": true,
+  "vocab_size": 32000
+}

checkpoints/main_gdn_random/generation_config.json

@@ -0,0 +1,10 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "pad_token_id": 0,
+  "transformers_version": "5.9.0",
+  "use_cache": true
+}

checkpoints/main_gdn_random/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d051a268ba74185e4f53952817ae3545c3e4ff22a7dac1c72d970150f5e0231c
+size 415192624

checkpoints/main_gdn_random/tokenizer_config.json

@@ -0,0 +1,16 @@
+{
+  "add_prefix_space": null,
+  "backend": "tokenizers",
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": null,
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

checkpoints/main_kda_DAMI/config.json

@@ -0,0 +1,49 @@
+{
+  "allow_neg_eigval": false,
+  "architectures": [
+    "KDAForCausalLM"
+  ],
+  "attn": null,
+  "attn_mode": "chunk",
+  "attnres_block_size": null,
+  "bos_token_id": 1,
+  "conv_size": 4,
+  "disable_recompute": true,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "expand_v": 2,
+  "fuse_cross_entropy": true,
+  "fuse_linear_cross_entropy": false,
+  "fuse_norm": true,
+  "fuse_swiglu": true,
+  "head_dim": 64,
+  "head_init_design": null,
+  "hidden_act": "swish",
+  "hidden_ratio": 4,
+  "hidden_size": 512,
+  "initialization_manifest": {
+    "A_log_rule": "per-head A_log copied exactly from GDN v3_layerdt3 targets",
+    "design": "v4 layerdt3: GDN v3_layerdt3 (PoST tau by head + low-high-low layerwise sensitivity) targets",
+    "design_config": "/root/linear-attention/flame/configs/gated_deltanet_100M_v3_layerdt3_a.json",
+    "dt_bias_rule": "per-head mean equals GDN v3_layerdt3 target dt_bias; 64 channels use deterministic centered Gaussian noise, std=0.03",
+    "experiment": "v4_layerdt3_b",
+    "non_ad_tensors": "same seeded base KDA initialization (seed 42) as v4_random_a / v4_post_a"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": null,
+  "lower_bound": null,
+  "max_position_embeddings": 4096,
+  "model_type": "kda",
+  "norm_eps": 1e-06,
+  "num_heads": 6,
+  "num_hidden_layers": 18,
+  "num_v_heads": 6,
+  "pad_token_id": 0,
+  "safe_gate": false,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.9.0",
+  "use_cache": true,
+  "use_l2warp": false,
+  "use_short_conv": true,
+  "vocab_size": 32000
+}

checkpoints/main_kda_DAMI/generation_config.json

@@ -0,0 +1,10 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "pad_token_id": 0,
+  "transformers_version": "5.9.0",
+  "use_cache": true
+}

checkpoints/main_kda_DAMI/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1ff8524598a136af2d32bf70001cd446a12a7f738ca4204877a7e11215c69c8
+size 406802360

checkpoints/main_kda_DAMI/tokenizer_config.json

@@ -0,0 +1,16 @@
+{
+  "add_prefix_space": null,
+  "backend": "tokenizers",
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": null,
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

checkpoints/main_kda_PoST/config.json

@@ -0,0 +1,47 @@
+{
+  "allow_neg_eigval": false,
+  "architectures": [
+    "KDAForCausalLM"
+  ],
+  "attn": null,
+  "attn_mode": "chunk",
+  "attnres_block_size": null,
+  "bos_token_id": 1,
+  "conv_size": 4,
+  "disable_recompute": true,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "expand_v": 2,
+  "fuse_cross_entropy": true,
+  "fuse_linear_cross_entropy": false,
+  "fuse_norm": true,
+  "fuse_swiglu": true,
+  "head_dim": 64,
+  "head_init_design": null,
+  "hidden_act": "swish",
+  "hidden_ratio": 4,
+  "hidden_size": 512,
+  "initialization_manifest": {
+    "A_log_rule": "PoST head-column values 0..8.317766; all other tensors copied from the same seeded base initialization",
+    "dt_bias_rule": "GDN-aligned head means with deterministic small channel noise; channel_std=0.03",
+    "experiment": "v4_post_a",
+    "gdn_source_csv": "/root/checkpoints/gdn/training/gdn100m-v3-random-5b-v3_random_a/initial_head_params.csv"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": null,
+  "lower_bound": null,
+  "max_position_embeddings": 4096,
+  "model_type": "kda",
+  "norm_eps": 1e-06,
+  "num_heads": 6,
+  "num_hidden_layers": 18,
+  "num_v_heads": 6,
+  "pad_token_id": 0,
+  "safe_gate": false,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.9.0",
+  "use_cache": true,
+  "use_l2warp": false,
+  "use_short_conv": true,
+  "vocab_size": 32000
+}

checkpoints/main_kda_PoST/generation_config.json

@@ -0,0 +1,10 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "pad_token_id": 0,
+  "transformers_version": "5.9.0",
+  "use_cache": true
+}

checkpoints/main_kda_PoST/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6be28267dc38fb84c4caafe9beba572ba861f7f4751bb5caa8218c07cb6155d
+size 406802360

checkpoints/main_kda_PoST/tokenizer_config.json

@@ -0,0 +1,16 @@
+{
+  "add_prefix_space": null,
+  "backend": "tokenizers",
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": null,
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

checkpoints/main_kda_random/config.json

@@ -0,0 +1,52 @@
+{
+  "allow_neg_eigval": false,
+  "architectures": [
+    "KDAForCausalLM"
+  ],
+  "attn": null,
+  "attn_mode": "chunk",
+  "attnres_block_size": null,
+  "bos_token_id": 1,
+  "conv_size": 4,
+  "disable_recompute": true,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "expand_v": 2,
+  "fuse_cross_entropy": true,
+  "fuse_linear_cross_entropy": false,
+  "fuse_norm": true,
+  "fuse_swiglu": true,
+  "head_dim": 64,
+  "head_init_design": null,
+  "hidden_act": "swish",
+  "hidden_ratio": 4,
+  "hidden_size": 512,
+  "initialization_manifest": {
+    "A_log_rule": "GDN v3_random_a source A_log",
+    "design_note": "KDA aligned random repeat: GDN v3_random_a head-level A_log/dtbias, per-head channel dtbias centered noise std=0.03",
+    "dt_bias_rule": "GDN-aligned head means with deterministic small channel noise; channel_std=0.03",
+    "experiment": "v4_random_c",
+    "gdn_source_csv": "/root/checkpoints/gdn/training/gdn100m-v3-random-5b-v3_random_a/initial_head_params.csv",
+    "repeat_of": "init_v4_random_a",
+    "repeat_rule": "identical initialization and training configuration to the source run; weights copied verbatim so run-to-run spread reflects training non-determinism only",
+    "source_package": "/root/checkpoints/kda/evals/_init_v4_random_a",
+    "source_weights_sha256": "f4333d94db0e652c24de2392ff77a3ba98064b3cbe7773fbb71d17edd64a9153"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": null,
+  "lower_bound": null,
+  "max_position_embeddings": 4096,
+  "model_type": "kda",
+  "norm_eps": 1e-06,
+  "num_heads": 6,
+  "num_hidden_layers": 18,
+  "num_v_heads": 6,
+  "pad_token_id": 0,
+  "safe_gate": false,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.9.0",
+  "use_cache": true,
+  "use_l2warp": false,
+  "use_short_conv": true,
+  "vocab_size": 32000
+}

checkpoints/main_kda_random/generation_config.json

@@ -0,0 +1,10 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "pad_token_id": 0,
+  "transformers_version": "5.9.0",
+  "use_cache": true
+}

checkpoints/main_kda_random/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6033721f703a952664942d1a51e7704d45688fba56ae432d5fd656a9ea1b6c1
+size 406802360

checkpoints/main_kda_random/tokenizer_config.json

@@ -0,0 +1,16 @@
+{
+  "add_prefix_space": null,
+  "backend": "tokenizers",
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": null,
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

checkpoints/main_mamba2_DAMI/config.json

@@ -0,0 +1,58 @@
+{
+  "A_init_range": [
+    1,
+    16
+  ],
+  "D_has_hdim": false,
+  "architectures": [
+    "Mamba2ForCausalLM"
+  ],
+  "bos_token_id": 1,
+  "chunk_size": 256,
+  "conv_init": null,
+  "conv_kernel": 4,
+  "dt_init_floor": 0.0001,
+  "dt_limit": [
+    0.0,
+    {
+      "__float__": "Infinity"
+    }
+  ],
+  "dt_max": 0.1,
+  "dt_min": 0.001,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "expand": 2,
+  "fuse_cross_entropy": true,
+  "fuse_linear_cross_entropy": false,
+  "fuse_norm": true,
+  "head_dim": 64,
+  "head_init_design": null,
+  "hidden_act": "silu",
+  "hidden_size": 512,
+  "initializer_range": 0.02,
+  "max_position_embeddings": 4096,
+  "model_type": "mamba2",
+  "n_groups": 1,
+  "norm_before_gate": false,
+  "norm_eps": 1e-05,
+  "num_heads": 16,
+  "num_hidden_layers": 42,
+  "pad_token_id": 0,
+  "rescale_prenorm_residual": true,
+  "residual_in_fp32": true,
+  "rms_norm": true,
+  "rmsnorm": true,
+  "state_size": 128,
+  "tie_word_embeddings": false,
+  "time_step_floor": 0.0001,
+  "time_step_max": 0.1,
+  "time_step_min": 0.001,
+  "time_step_rank": 128,
+  "transformers_version": "5.9.0",
+  "use_bias": false,
+  "use_cache": true,
+  "use_conv_bias": true,
+  "use_l2warp": false,
+  "vocab_size": 32000
+}

checkpoints/main_mamba2_DAMI/generation_config.json

@@ -0,0 +1,10 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "pad_token_id": 0,
+  "transformers_version": "5.9.0",
+  "use_cache": true
+}

checkpoints/main_mamba2_DAMI/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1cc2b9b0acd06f27717d2079ac2e1aa2d08babeb9c45fa78d559715555a2b2be
+size 210066632