Upload 3 files

README.md

@@ -1,3 +1,47 @@
----
-license: cc-by-nc-nd-4.0
----
+# Waveformer
+
+Continuous-time language model using Kuramoto phase synchronization instead of attention. O(1) memory scaling across sequence lengths. Trains on 8GB consumer GPU.
+
+## Paper
+
+"Emergence Over Attention: Continuous-Time Phase Synchronization as a Computational Primitive"
+DOI: [10.5281/zenodo.20741536](https://doi.org/10.5281/zenodo.20741536)
+
+Prior work: "Deterministic Layer Freezing in Autoregressive Language Models via Continuous Phase Coherence Analysis"
+DOI: [10.5281/zenodo.20720827](https://doi.org/10.5281/zenodo.20720827)
+
+## Usage
+
+```python
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+model = AutoModelForCausalLM.from_pretrained(
+    "Wannavf/Waveformer-207M-Chat",
+    trust_remote_code=True
+)
+tokenizer = AutoTokenizer.from_pretrained("gpt2")
+tokenizer.add_special_tokens({
+    'additional_special_tokens': ['<|im_start|>', '<|im_end|>']
+})
+
+prompt = "<|im_start|>user\nHello! What can you do?<|im_end|>\n<|im_start|>assistant\n"
+inputs = tokenizer(prompt, return_tensors="pt")
+outputs = model.generate(**inputs, max_new_tokens=50, temperature=0.7, top_p=0.8, top_k=40)
+```
+
+## Architecture
+
+| Component | Description |
+|-----------|-------------|
+| Attention | Kuramoto phase synchronization (no Q/K/V, no softmax) |
+| Position | KAM irrational frequency (zero learned parameters) |
+| Memory | O(1) in sequence length, flat VRAM at any context length |
+| Backward | Reversible integration (no intermediate activation storage) |
+
+## VRAM
+
+Flat at all sequence lengths. Attention mechanism contributes zero additional memory per token.
+
+## License
+
+This model is provided for research purposes. Commercial use requires explicit permission.

configuration_waveformer.py

@@ -0,0 +1,16 @@
+from transformers import PretrainedConfig
+
+class WaveformerConfig(PretrainedConfig):
+    model_type = "waveformer"
+
+    def __init__(self,
+                 vocab_size=50257,
+                 d_model=1024,
+                 n_layers=12,
+                 max_seq_len=32768,
+                 **kwargs):
+        super().__init__(**kwargs)
+        self.vocab_size = vocab_size
+        self.d_model = d_model
+        self.n_layers = n_layers
+        self.max_seq_len = max_seq_len

modeling_waveformer.py

@@ -0,0 +1,99 @@
+"""Waveformer model for HuggingFace transformers."""
+import torch, torch.nn as nn, math
+from transformers import PreTrainedModel
+from transformers.modeling_outputs import CausalLMOutputWithPast
+from transformers.generation import GenerationMixin
+try:
+    from .configuration_waveformer import WaveformerConfig
+except ImportError:
+    from configuration_waveformer import WaveformerConfig
+
+
+class _OscillatorAttention(nn.Module):
+    def __init__(self, d_model, d_out, n_osc):
+        super().__init__()
+        self.n_osc = n_osc
+        self.perturb = nn.Linear(d_model, n_osc, bias=False)
+        self.readout = nn.Linear(n_osc, d_out, bias=False)
+        omega = (torch.arange(n_osc).float() * 1.618033988749895).fmod(1.0) * 2 * math.pi
+        self.register_buffer('omega', omega)
+        idx = torch.arange(n_osc)
+        dist = (idx.unsqueeze(1) - idx.unsqueeze(0)).abs().float()
+        K = torch.zeros(n_osc, n_osc)
+        K[dist > 0] = torch.exp(-dist[dist > 0] / 100.0)
+        self.register_buffer('coupling', K)
+
+    def forward(self, x):
+        B, S, D = x.shape
+        theta = self.perturb(x.mean(1))
+        for _ in range(3):
+            sd = torch.sin(theta.unsqueeze(-1) - theta.unsqueeze(-2))
+            theta = theta + 0.1 * (self.omega + (self.coupling * sd).sum(-1))
+        return self.readout(torch.cos(theta))
+
+
+class WaveformerPreTrainedModel(PreTrainedModel):
+    config_class = WaveformerConfig
+    base_model_prefix = "waveformer"
+
+    def _init_weights(self, module):
+        if isinstance(module, nn.Linear):
+            module.weight.data.normal_(mean=0.0, std=0.02)
+            if module.bias is not None:
+                module.bias.data.zero_()
+
+
+class WaveformerLayer(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        D = config.d_model
+        self.osc = _OscillatorAttention(D, D, D * 2)
+        self.norm1 = nn.RMSNorm(D, eps=1e-5)
+        self.norm2 = nn.RMSNorm(D, eps=1e-5)
+        self.ffn = nn.Sequential(
+            nn.Linear(D, D * 8 // 3, bias=False),
+            nn.SiLU(),
+            nn.Linear(D * 8 // 3, D, bias=False),
+        )
+
+    def forward(self, x):
+        return x + self.osc(self.norm1(x)).unsqueeze(1).expand(-1, x.shape[1], -1) + self.ffn(self.norm2(x))
+
+
+class WaveformerModel(WaveformerPreTrainedModel, GenerationMixin):
+    def __init__(self, config):
+        super().__init__(config)
+        self.embed = nn.Embedding(config.vocab_size, config.d_model)
+
+        pos = torch.arange(config.max_seq_len).float()
+        omega = (pos * 1.618033988749895).fmod(1.0) * 2 * math.pi
+        sub = torch.arange(config.d_model).float() * 0.01
+        theta = omega.unsqueeze(1) + sub.unsqueeze(0)
+        self.register_buffer('kam_sin', torch.sin(theta))
+        self.register_buffer('kam_cos', torch.cos(theta))
+
+        self.layers = nn.ModuleList([
+            WaveformerLayer(config) for _ in range(config.n_layers)
+        ])
+        self.norm_f = nn.RMSNorm(config.d_model, eps=1e-5)
+        self.lm_head = nn.Linear(config.d_model, config.vocab_size, bias=False)
+        self.lm_head.weight = self.embed.weight
+        self.post_init()
+
+    def forward(self, input_ids, attention_mask=None, **kwargs):
+        B, S = input_ids.shape
+        sp = min(S, self.kam_sin.shape[0])
+        x = self.embed(input_ids)
+        x = x * self.kam_cos[:sp].unsqueeze(0) + x.roll(1, -1) * self.kam_sin[:sp].unsqueeze(0)
+        for layer in self.layers:
+            x = layer(x)
+        return CausalLMOutputWithPast(logits=self.lm_head(self.norm_f(x)))
+
+    def prepare_inputs_for_generation(self, input_ids, **kwargs):
+        return {"input_ids": input_ids}
+
+    def get_input_embeddings(self):
+        return self.embed
+
+    def set_input_embeddings(self, value):
+        self.embed = value