modeling_hfp.py

# Hyper Flux Projection (HFP) — O(1)-memory causal language model
# Copyright (C) 2026 Kayrahan Yılmaz
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <https://www.gnu.org/licenses/>.

import torch
import torch.nn as nn
import math
from transformers import PreTrainedModel
from transformers.modeling_outputs import CausalLMOutputWithPast, BaseModelOutputWithPast
from .configuration_hfp import HFPConfig

from .hfp_bulk_state import HFPBulkState
from .bulk_trigger_decoder import BulkTriggerDecoderLayer

class SinusoidalPositionalEncoding(nn.Module):
    def __init__(self, hidden_size, max_len=5000, pe_scale=0.3):
        super().__init__()
        self.max_len = max_len
        # [FIX K7] PE olcegi. Onceden ham (norm=sqrt(d/2)=8) eklenirken embedding
        # normu 0.02*sqrt(d)=0.23 idi -> PE, token icerigini ~35x BOGUYORDU;
        # anahtar/deger'ler ~%97 pozisyon oluyor, icerik-tabanli recall imkansiz
        # (MQAR loss ln(val_space)'te sabitlenip binding hic ogrenilmiyordu).
        # embed *sqrt(d) (bkz. HFPModel) + PE *0.3 ile normlar dengelenir.
        self.pe_scale = pe_scale
        pe = torch.zeros(max_len, hidden_size)
        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)
        div_term = torch.exp(torch.arange(0, hidden_size, 2).float() * (-math.log(10000.0) / hidden_size))
        pe[:, 0::2] = torch.sin(position * div_term)
        pe[:, 1::2] = torch.cos(position * div_term)
        self.register_buffer('pe', pe.unsqueeze(0))

    def forward(self, x, offset: int = 0):
        # [FIX A2] Streaming/chunked prefill'de her chunk'a 0..L degil, GLOBAL pozisyon eklenir.
        seq_len = x.size(1)
        if offset + seq_len > self.max_len:
            offset = max(0, self.max_len - seq_len)
        return x + self.pe_scale * self.pe[:, offset:offset + seq_len, :].to(x.device)

class HFPPreTrainedModel(PreTrainedModel):
    config_class = HFPConfig
    base_model_prefix = "hfp"
    _supports_cache_class = False

    def _init_weights(self, module):
        if isinstance(module, nn.Linear):
            module.weight.data.normal_(mean=0.0, std=0.02)
            if module.bias is not None:
                # importance_gate bias'i bilerek -2.0 (gate-collapse onlemi); ezme.
                if torch.all(module.bias.data == -2.0):
                    pass
                else:
                    module.bias.data.zero_()
        elif isinstance(module, nn.Embedding):
            module.weight.data.normal_(mean=0.0, std=0.02)
            if module.padding_idx is not None:
                module.weight.data[module.padding_idx].zero_()
        elif isinstance(module, nn.LayerNorm):
            module.bias.data.zero_()
            module.weight.data.fill_(1.0)

class HFPModel(HFPPreTrainedModel):
    def __init__(self, config):
        super().__init__(config)
        self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size)
        # [FIX K7] Vaswani sqrt(d) olcegi: emb (init std 0.02) tek basina norm~0.23;
        # sqrt(d) ile ~2.56'ya cikar ve sonuchennmus PE (*0.3, norm~2.4) ile dengelenir.
        self.embed_scale = math.sqrt(config.hidden_size)
        self.pos_encoder = SinusoidalPositionalEncoding(
            config.hidden_size, max_len=config.max_position_embeddings,
            pe_scale=getattr(config, "pe_scale", 0.3))

        self.layers = nn.ModuleList([
            BulkTriggerDecoderLayer(
                hidden_size=config.hidden_size,
                num_heads=config.num_attention_heads,
                feedforward_dim=config.intermediate_size,
                bulk_dim=config.bulk_dim,
                vocab_size=None,
                local_window=getattr(config, "local_window", None),   # [FIX K5]
                dropout_p=getattr(config, "dropout_p", 0.1),          # [FIX K3]
                ffn_type=getattr(config, "ffn_type", "entangled")     # [HFP-SCALE]
            )
            for _ in range(config.num_hidden_layers)
        ])

        # Katman-basi recurrent bellek (physics-inspired 'Bulk' analojisi;
        # teknik olarak: decay'li lineer-attention state'i M, z)
        # [TEMIZLIK B1] medium_freq/long_freq/medium_momentum kaldirildi.
        self.bulk_states = nn.ModuleList([
            HFPBulkState(
                hidden_size=config.hidden_size,
                short_len=config.short_len,
                max_short_len=getattr(config, "max_short_len", None), # [FIX K4]
                rec_block=getattr(config, "rec_block", 64),           # [K2]
                decay_mode=getattr(config, "decay_mode", "exp"),      # [HFP-CORE]
                conv_kernel=getattr(config, "conv_kernel", 3),        # [FIX K8] binding
                key_feature_map=getattr(config, "key_feature_map", "elu"),  # [HFP-CAP]
                dpfp_nu=getattr(config, "dpfp_nu", 2),                # [HFP-CAP]
                write_rule=getattr(config, "write_rule", "additive")  # [HFP-DELTA]
            )
            for _ in range(config.num_hidden_layers)
        ])
        # [K2] TBPTT: True ise chunk'lar arasi state detach edilmez
        self._detach_state = not getattr(config, "bptt_across_chunks", False)

        self.norm = nn.LayerNorm(config.hidden_size)
        self.post_init()

    @staticmethod
    def _offset_from_state(past_key_values_list):
        # state tuple: (short_memory, M, z, token_count, short_len_dynamic, write_idx, conv_state)
        first = past_key_values_list[0]
        if first is not None and len(first) >= 4 and isinstance(first[3], int):
            return int(first[3])
        return 0

    def forward(self, input_ids, attention_mask=None, past_key_values=None, use_cache=False, **kwargs):
        x = self.embed_tokens(input_ids) * self.embed_scale  # [FIX K7]

        if past_key_values is None or not isinstance(past_key_values, (tuple, list)):
            past_key_values_list = [None] * len(self.layers)
        else:
            past_key_values_list = past_key_values

        # [FIX A2] Global pozisyon offseti (chunked prefill icin)
        offset = self._offset_from_state(past_key_values_list)
        x = self.pos_encoder(x, offset=offset)

        new_past_key_values = []
        gate_entropies = []
        for i, (layer, bulk_state) in enumerate(zip(self.layers, self.bulk_states)):
            x, _, new_past_state = layer(x, bulk_state, past_state=past_key_values_list[i],
                                         return_past_state=True, detach_state=self._detach_state)

            # [C1] Gradyanli gate-entropy topla (yalnizca agirlik > 0 iken loss'a katilir)
            if self.training and getattr(bulk_state, "_gate_entropy_live", None) is not None:
                gate_entropies.append(bulk_state._gate_entropy_live)

            if use_cache:
                new_past_key_values.append(new_past_state)

        x = self.norm(x)

        out = BaseModelOutputWithPast(
            last_hidden_state=x,
            past_key_values=new_past_key_values if use_cache else None
        )
        out.gate_entropy = (torch.stack(gate_entropies).mean() if gate_entropies else None)
        return out

from transformers.generation import GenerationMixin

class HFPForCausalLM(HFPPreTrainedModel, GenerationMixin):
    # [D1] lm_head, embed_tokens'a bagli (weight tying) — safetensors kaydinin
    # paylasilan tensoru dogru ele almasi icin bildirilmeli.
    # transformers v5: dict (hedef -> kaynak); v4'te de sorunsuz calisir.
    _tied_weights_keys = {"lm_head.weight": "hfp.embed_tokens.weight"}

    def __init__(self, config):
        super().__init__(config)
        self.hfp = HFPModel(config)
        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
        # [D1] Weight tying (GPT standardi): embedding ve lm_head paylasimi
        self.lm_head.weight = self.hfp.embed_tokens.weight
        self.post_init()

    # [D1] tie_weights()'in from_pretrained SONRASI baglantiyi yeniden kurabilmesi
    # icin standart erisimciler (bunlarsiz yukleme tying'i sessizce koparir).
    def get_input_embeddings(self):
        return self.hfp.embed_tokens

    def set_input_embeddings(self, value):
        self.hfp.embed_tokens = value

    def get_output_embeddings(self):
        return self.lm_head

    def set_output_embeddings(self, value):
        self.lm_head = value

    def forward(self, input_ids, attention_mask=None, labels=None, past_key_values=None, use_cache=False, **kwargs):
        outputs = self.hfp(input_ids, attention_mask=attention_mask, past_key_values=past_key_values, use_cache=use_cache, **kwargs)
        hidden_states = outputs.last_hidden_state
        logits = self.lm_head(hidden_states)

        loss = None
        if labels is not None:
            shift_logits = logits[..., :-1, :].contiguous()
            shift_labels = labels[..., 1:].contiguous()
            loss_fct = nn.CrossEntropyLoss()
            loss = loss_fct(shift_logits.view(-1, self.config.vocab_size), shift_labels.view(-1))

            # [C1] Opsiyonel gate-entropy duzenleyici - default kapali (weight=0.0).
            w = float(getattr(self.config, "aux_gate_entropy_weight", 0.0))
            if w > 0.0 and getattr(outputs, "gate_entropy", None) is not None:
                loss = loss + w * outputs.gate_entropy

            # [ORTHO] EntangledFFN ortogonallik duzenleyicisi. Paylasilan W_bulk'tan
            # turetilen P_A/P_B'nin farkli ("tek bulk'in iki ayri golgesi") kalmasi
            # icin baski. Onceden implement edilmis ama loss'a HIC bagli degildi
            # (olu kod); artik opsiyonel + default kapali (weight=0.0 => baseline degismez).
            w_o = float(getattr(self.config, "aux_ortho_weight", 0.0))
            if w_o > 0.0:
                ortho = sum(layer.ffn.get_orthogonality_loss() for layer in self.hfp.layers)
                loss = loss + w_o * ortho

        return CausalLMOutputWithPast(
            loss=loss,
            logits=logits,
            past_key_values=outputs.past_key_values
        )

    def prepare_inputs_for_generation(self, input_ids, past_key_values=None, **kwargs):
        if past_key_values:
            input_ids = input_ids[:, -1:]
        return {
            "input_ids": input_ids,
            "past_key_values": past_key_values,
            "use_cache": kwargs.get("use_cache", True)
        }