Duplicate from PhysiQuanty/Patenty-Test2-Radix-65536

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,10 @@
+
+---
+# BinaryLLM (HF export)
+
+Tokenizer-free / base-N model export.
+
+## Load
+```python
+from transformers import AutoModelForCausalLM
+m = AutoModelForCausalLM.from_pretrained("./hf_binaryllm_repo", trust_remote_code=True)

SAVE_tokenizer_config_SAVE.json

@@ -0,0 +1,31 @@
+{
+  "added_tokens_decoder": {
+    "65536": {
+      "content": "<BOS>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65537": {
+      "content": "<UNK>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<BOS>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<EOS>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<EOS>",
+  "tokenizer_class": "tokenization_binaryllm.BinaryLLMTokenizer",
+  "unk_token": "<UNK>",
+  "auto_map": {
+    "AutoTokenizer": "tokenization_binaryllm.BinaryLLMTokenizer"
+  }
+}

__init__.py

@@ -0,0 +1,3 @@
+from .configuration_binaryllm import BinaryLLMConfig
+from .modeling_binaryllm import BinaryLLMForCausalLM
+from .tokenization_binaryllm import BinaryLLMTokenizer

binaryllm_vocab.json

@@ -0,0 +1,12 @@
+{
+  "base_vocab_size": 65536,
+  "vocab_size": 65538,
+  "bos_token": "<BOS>",
+  "bos_token_id": 65536,
+  "eos_token": "<EOS>",
+  "eos_token_id": 65537,
+  "unk_token": "<EOS>",
+  "unk_token_id": 65537,
+  "pad_token": "<EOS>",
+  "pad_token_id": 65537
+}

config.json

@@ -0,0 +1,22 @@
+{
+  "model_type": "binaryllm",
+  "architectures": ["BinaryLLMForCausalLM"],
+  "auto_map": {
+    "AutoConfig": "configuration_binaryllm.BinaryLLMConfig",
+    "AutoModelForCausalLM": "modeling_binaryllm.BinaryLLMForCausalLM",
+    "AutoTokenizer": "tokenization_binaryllm.BinaryLLMTokenizer"
+  },
+  "vocab_size": 65538,
+  "bos_token_id": 65536,
+  "eos_token_id": 65537,
+  "pad_token_id": 65537,
+  "hidden_size": 512,
+  "num_hidden_layers": 4,
+  "num_attention_heads": 4,
+  "intermediate_size": 2048,
+  "max_position_embeddings": 2048,
+  "dropout": 0.1,
+  "activation": "gelu",
+  "attn_backend": "auto",
+  "torch_dtype": "float32"
+}

configuration_binaryllm.py

@@ -0,0 +1,37 @@
+from transformers import PretrainedConfig
+
+
+class BinaryLLMConfig(PretrainedConfig):
+    model_type = "binaryllm"
+
+    def __init__(
+        self,
+        vocab_size: int = 65538,
+        hidden_size: int = 512,
+        num_hidden_layers: int = 4,
+        num_attention_heads: int = 4,
+        intermediate_size: int = 2048,
+        max_position_embeddings: int = 2048,
+        dropout: float = 0.1,
+        activation: str = "gelu",
+        attn_backend: str = "auto",
+        bos_token_id: int = 65536,
+        eos_token_id: int = 65537,
+        pad_token_id: int = 65537,
+        **kwargs,
+    ):
+        self.vocab_size = int(vocab_size)
+        self.hidden_size = int(hidden_size)
+        self.num_hidden_layers = int(num_hidden_layers)
+        self.num_attention_heads = int(num_attention_heads)
+        self.intermediate_size = int(intermediate_size)
+        self.max_position_embeddings = int(max_position_embeddings)
+        self.dropout = float(dropout)
+        self.activation = str(activation)
+        self.attn_backend = str(attn_backend)
+
+        self.bos_token_id = int(bos_token_id)
+        self.eos_token_id = int(eos_token_id)
+        self.pad_token_id = int(pad_token_id)
+
+        super().__init__(**kwargs)

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c7c56c25632fdb81d24a2ee74fe46a050b4faddfba5fe62612974b64ee5d660
+size 318893256

modeling_binaryllm.py

@@ -0,0 +1,556 @@
+import math
+from dataclasses import dataclass
+from typing import Optional
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from transformers import PreTrainedModel
+from transformers.modeling_outputs import CausalLMOutput
+
+from .configuration_binaryllm import BinaryLLMConfig
+
+try:
+    import flash_attn_v100_cuda
+    _FLASH_V100_AVAILABLE = True
+except Exception:
+    flash_attn_v100_cuda = None
+    _FLASH_V100_AVAILABLE = False
+
+
+class PositionalEncoding(nn.Module):
+    """
+    Sinusoidal positional encoding, stocké en fp32,
+    puis casté au dtype de x à chaque forward.
+    """
+
+    def __init__(self, d_model: int, max_len: int) -> None:
+        super().__init__()
+        pe = torch.zeros(max_len, d_model, dtype=torch.float32)
+        position = torch.arange(0, max_len, dtype=torch.float32).unsqueeze(1)
+        div_term = torch.exp(
+            torch.arange(0, d_model, 2, dtype=torch.float32)
+            * (-torch.log(torch.tensor(10000.0)) / d_model)
+        )
+        pe[:, 0::2] = torch.sin(position * div_term)
+        pe[:, 1::2] = torch.cos(position * div_term)
+        pe = pe.unsqueeze(0)
+        self.register_buffer("pe", pe, persistent=False)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        t = x.size(1)
+        pe = self.pe[:, :t, :].to(device=x.device, dtype=x.dtype)
+        return x + pe
+
+
+@dataclass
+class _InnerCfg:
+    block_size: int
+    embed_dim: int
+    vocab_size: int
+    num_heads: int
+    num_layers: int
+    ff_hidden_dim: int
+    dropout: float
+    layernorm_dim: Optional[int] = None
+    head_dim: Optional[int] = None
+    attn_backend: str = "auto"
+
+
+class FlashSelfAttentionPortable(nn.Module):
+    def __init__(
+        self,
+        embed_dim: int,
+        num_heads: int,
+        dropout: float = 0.0,
+        causal: bool = True,
+        backend: str = "auto",
+    ) -> None:
+        super().__init__()
+
+        if embed_dim % num_heads != 0:
+            raise ValueError(
+                f"embed_dim ({embed_dim}) doit être divisible par num_heads ({num_heads})"
+            )
+
+        self.embed_dim = embed_dim
+        self.num_heads = num_heads
+        self.head_dim = embed_dim // num_heads
+        self.dropout = float(dropout)
+        self.causal = bool(causal)
+        self.backend = str(backend)
+        self.softmax_scale = self.head_dim ** -0.5
+
+        self.q_proj = nn.Linear(embed_dim, embed_dim, bias=True)
+        self.k_proj = nn.Linear(embed_dim, embed_dim, bias=True)
+        self.v_proj = nn.Linear(embed_dim, embed_dim, bias=True)
+        self.out_proj = nn.Linear(embed_dim, embed_dim, bias=True)
+
+    def _shape_qkv(
+        self,
+        x: torch.Tensor,
+    ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.dtype]:
+        bsz, seqlen, _ = x.shape
+        residual_dtype = x.dtype
+
+        proj_dtype = self.q_proj.weight.dtype
+        if x.dtype != proj_dtype:
+            x = x.to(proj_dtype)
+
+        q = self.q_proj(x)
+        k = self.k_proj(x)
+        v = self.v_proj(x)
+
+        q = q.view(bsz, seqlen, self.num_heads, self.head_dim).transpose(1, 2).contiguous()
+        k = k.view(bsz, seqlen, self.num_heads, self.head_dim).transpose(1, 2).contiguous()
+        v = v.view(bsz, seqlen, self.num_heads, self.head_dim).transpose(1, 2).contiguous()
+
+        return q, k, v, residual_dtype
+
+    def _merge_heads(self, x: torch.Tensor) -> torch.Tensor:
+        bsz, nheads, seqlen, head_dim = x.shape
+        return x.transpose(1, 2).contiguous().view(bsz, seqlen, nheads * head_dim)
+
+    def _can_use_v100_kernel(self, q: torch.Tensor, padding_mask: Optional[torch.Tensor]) -> bool:
+        if not _FLASH_V100_AVAILABLE:
+            return False
+
+        if not q.is_cuda:
+            return False
+
+        if padding_mask is not None and bool(padding_mask.any().item()):
+            return False
+
+        cc = torch.cuda.get_device_capability(q.device)
+        if cc != (7, 0):
+            return False
+
+        hd = q.size(-1)
+        if hd % 2 != 0:
+            return False
+        if hd % 8 != 0:
+            return False
+        if hd > 256:
+            return False
+
+        return True
+
+    def _flash_attn_v100(
+        self,
+        q: torch.Tensor,
+        k: torch.Tensor,
+        v: torch.Tensor,
+    ) -> torch.Tensor:
+        if q.dtype != torch.float16:
+            q = q.to(torch.float16)
+        if k.dtype != torch.float16:
+            k = k.to(torch.float16)
+        if v.dtype != torch.float16:
+            v = v.to(torch.float16)
+
+        result = flash_attn_v100_cuda.fwd(
+            q,
+            k,
+            v,
+            None,
+            None,
+            0.0,
+            self.softmax_scale,
+            self.causal,
+            -1,
+            -1,
+            0.0,
+            False,
+            None,
+        )
+
+        out = result[0]
+        return out
+
+    def _sdpa_attn(
+        self,
+        q: torch.Tensor,
+        k: torch.Tensor,
+        v: torch.Tensor,
+        padding_mask: Optional[torch.Tensor] = None,
+    ) -> torch.Tensor:
+        bsz, nheads, tq, _ = q.shape
+        tk = k.size(-2)
+
+        attn_mask = None
+        if padding_mask is not None:
+            key_mask = padding_mask[:, None, None, :].to(device=q.device, dtype=torch.bool)
+            key_mask = key_mask.expand(bsz, nheads, tq, tk)
+            attn_mask = ~key_mask
+
+        dropout_p = self.dropout if self.training else 0.0
+
+        with torch.backends.cuda.sdp_kernel(
+            enable_flash=True,
+            enable_mem_efficient=True,
+            enable_math=True,
+        ):
+            out = F.scaled_dot_product_attention(
+                q,
+                k,
+                v,
+                attn_mask=attn_mask,
+                dropout_p=dropout_p,
+                is_causal=self.causal if attn_mask is None else False,
+                scale=self.softmax_scale,
+            )
+
+        return out
+
+    def _eager_attn(
+        self,
+        q: torch.Tensor,
+        k: torch.Tensor,
+        v: torch.Tensor,
+        padding_mask: Optional[torch.Tensor] = None,
+    ) -> torch.Tensor:
+        scores = torch.matmul(q.float(), k.float().transpose(-2, -1)) * self.softmax_scale
+
+        if self.causal:
+            tq = q.size(-2)
+            tk = k.size(-2)
+            causal_mask = torch.triu(
+                torch.ones(tq, tk, device=scores.device, dtype=torch.bool),
+                diagonal=1,
+            )
+            scores = scores.masked_fill(causal_mask.unsqueeze(0).unsqueeze(0), float("-inf"))
+
+        if padding_mask is not None:
+            key_mask = padding_mask[:, None, None, :].to(device=scores.device, dtype=torch.bool)
+            scores = scores.masked_fill(key_mask, float("-inf"))
+
+        probs = torch.softmax(scores, dim=-1)
+
+        if self.training and self.dropout > 0.0:
+            probs = F.dropout(probs, p=self.dropout)
+
+        out = torch.matmul(probs, v.float())
+        return out.to(q.dtype)
+
+    def forward(
+        self,
+        x: torch.Tensor,
+        padding_mask: Optional[torch.Tensor] = None,
+    ) -> torch.Tensor:
+        q, k, v, residual_dtype = self._shape_qkv(x)
+
+        if padding_mask is not None:
+            padding_mask = padding_mask.to(device=x.device, dtype=torch.bool)
+
+        backend = self.backend
+
+        if backend == "v100":
+            if not self._can_use_v100_kernel(q, padding_mask):
+                raise RuntimeError(
+                    "backend='v100' demandé mais indisponible "
+                    "(flash_attn_v100_cuda absent, GPU non sm70/V100, padding présent, "
+                    "ou head_dim incompatible)."
+                )
+            out = self._flash_attn_v100(q, k, v)
+
+        elif backend == "sdpa":
+            out = self._sdpa_attn(q, k, v, padding_mask=padding_mask)
+
+        elif backend == "eager":
+            out = self._eager_attn(q, k, v, padding_mask=padding_mask)
+
+        elif backend == "auto":
+            if self._can_use_v100_kernel(q, padding_mask):
+                out = self._flash_attn_v100(q, k, v)
+            else:
+                out = self._sdpa_attn(q, k, v, padding_mask=padding_mask)
+
+        else:
+            raise ValueError(f"backend d'attention non supporté: {backend}")
+
+        out = self._merge_heads(out)
+
+        out_proj_dtype = self.out_proj.weight.dtype
+        if out.dtype != out_proj_dtype:
+            out = out.to(out_proj_dtype)
+
+        out = self.out_proj(out)
+
+        if out.dtype != residual_dtype:
+            out = out.to(residual_dtype)
+
+        return out
+
+
+class FlashTransformerEncoderLayerPortable(nn.Module):
+    def __init__(
+        self,
+        d_model: int,
+        nhead: int,
+        dim_feedforward: int,
+        dropout: float = 0.1,
+        activation: str = "gelu",
+        batch_first: bool = True,
+        attn_backend: str = "auto",
+    ) -> None:
+        super().__init__()
+
+        if not batch_first:
+            raise ValueError("Cette implémentation supporte batch_first=True uniquement.")
+
+        self.self_attn = FlashSelfAttentionPortable(
+            embed_dim=d_model,
+            num_heads=nhead,
+            dropout=dropout,
+            causal=True,
+            backend=attn_backend,
+        )
+
+        self.linear1 = nn.Linear(d_model, dim_feedforward)
+        self.linear2 = nn.Linear(dim_feedforward, d_model)
+
+        self.norm1 = nn.LayerNorm(d_model)
+        self.norm2 = nn.LayerNorm(d_model)
+
+        self.dropout = nn.Dropout(dropout)
+        self.dropout1 = nn.Dropout(dropout)
+        self.dropout2 = nn.Dropout(dropout)
+
+        if activation == "gelu":
+            self.activation = F.gelu
+        elif activation == "relu":
+            self.activation = F.relu
+        else:
+            raise ValueError(f"activation non supportée: {activation}")
+
+    def _sa_block(
+        self,
+        x: torch.Tensor,
+        src_key_padding_mask: Optional[torch.Tensor],
+    ) -> torch.Tensor:
+        x = self.self_attn(x, padding_mask=src_key_padding_mask)
+        x = self.dropout1(x)
+        return x
+
+    def _ff_block(self, x: torch.Tensor) -> torch.Tensor:
+        ff_dtype = self.linear1.weight.dtype
+        x_ff = x if x.dtype == ff_dtype else x.to(ff_dtype)
+
+        x_ff = self.linear1(x_ff)
+        x_ff = self.activation(x_ff)
+        x_ff = self.dropout(x_ff)
+        x_ff = self.linear2(x_ff)
+        x_ff = self.dropout2(x_ff)
+
+        if x_ff.dtype != x.dtype:
+            x_ff = x_ff.to(x.dtype)
+
+        return x_ff
+
+    def forward(
+        self,
+        src: torch.Tensor,
+        src_mask: Optional[torch.Tensor] = None,
+        src_key_padding_mask: Optional[torch.Tensor] = None,
+    ) -> torch.Tensor:
+        x = src
+        x = self.norm1(x + self._sa_block(x, src_key_padding_mask))
+        x = self.norm2(x + self._ff_block(x))
+        return x
+
+
+class FlashTransformerEncoderPortable(nn.Module):
+    def __init__(
+        self,
+        encoder_layer: FlashTransformerEncoderLayerPortable,
+        num_layers: int,
+        attn_backend: str = "auto",
+    ) -> None:
+        super().__init__()
+
+        d_model = encoder_layer.norm1.normalized_shape[0]
+        nhead = encoder_layer.self_attn.num_heads
+        dim_feedforward = encoder_layer.linear1.out_features
+        dropout = encoder_layer.dropout.p
+
+        self.layers = nn.ModuleList(
+            [
+                FlashTransformerEncoderLayerPortable(
+                    d_model=d_model,
+                    nhead=nhead,
+                    dim_feedforward=dim_feedforward,
+                    dropout=dropout,
+                    activation="gelu",
+                    batch_first=True,
+                    attn_backend=attn_backend,
+                )
+                for _ in range(num_layers)
+            ]
+        )
+
+    def forward(
+        self,
+        src: torch.Tensor,
+        mask: Optional[torch.Tensor] = None,
+        src_key_padding_mask: Optional[torch.Tensor] = None,
+    ) -> torch.Tensor:
+        x = src
+        for layer in self.layers:
+            x = layer(x, src_mask=mask, src_key_padding_mask=src_key_padding_mask)
+        return x
+
+
+class TinyTransformerLM(nn.Module):
+    def __init__(self, cfg: _InnerCfg) -> None:
+        super().__init__()
+        self.cfg = cfg
+
+        vocab_size = cfg.vocab_size
+        self.tok_embed = nn.Embedding(vocab_size, cfg.embed_dim)
+        self.pos_encoding = PositionalEncoding(cfg.embed_dim, cfg.block_size)
+
+        encoder_layer = FlashTransformerEncoderLayerPortable(
+            d_model=cfg.embed_dim,
+            nhead=cfg.num_heads,
+            dim_feedforward=cfg.ff_hidden_dim,
+            dropout=cfg.dropout,
+            activation="gelu",
+            batch_first=True,
+            attn_backend=cfg.attn_backend,
+        )
+        self.encoder = FlashTransformerEncoderPortable(
+            encoder_layer,
+            num_layers=cfg.num_layers,
+            attn_backend=cfg.attn_backend,
+        )
+
+        ln_dim = cfg.layernorm_dim or cfg.embed_dim
+        head_dim = cfg.head_dim or ln_dim
+
+        self.pre_ln_proj: Optional[nn.Linear] = None
+        if ln_dim != cfg.embed_dim:
+            self.pre_ln_proj = nn.Linear(cfg.embed_dim, ln_dim)
+
+        self.ln = nn.LayerNorm(ln_dim)
+
+        self.head_pre: Optional[nn.Linear] = None
+        if head_dim != ln_dim:
+            self.head_pre = nn.Linear(ln_dim, head_dim)
+
+        self.head = nn.Linear(head_dim, vocab_size, bias=False)
+
+        if self.pre_ln_proj is None and self.head_pre is None and head_dim == cfg.embed_dim:
+            self.head.weight = self.tok_embed.weight
+
+        causal = torch.triu(torch.ones(cfg.block_size, cfg.block_size, dtype=torch.bool), diagonal=1)
+        self.register_buffer("causal_mask", causal, persistent=False)
+
+    def forward(self, tokens: torch.Tensor, padding_mask: Optional[torch.Tensor] = None) -> torch.Tensor:
+        x = self.tok_embed(tokens)
+        x = self.pos_encoding(x)
+
+        seq_len = tokens.size(1)
+        attn_mask = self.causal_mask[:seq_len, :seq_len].to(device=tokens.device)
+
+        if padding_mask is not None:
+            padding_mask = padding_mask[:, :seq_len].to(device=tokens.device, dtype=torch.bool)
+
+        x = self.encoder(x, mask=attn_mask, src_key_padding_mask=padding_mask)
+
+        if self.pre_ln_proj is not None:
+            proj_dtype = self.pre_ln_proj.weight.dtype
+            if x.dtype != proj_dtype:
+                x = x.to(proj_dtype)
+            x = self.pre_ln_proj(x)
+
+        ln_dtype = self.ln.weight.dtype
+        if x.dtype != ln_dtype:
+            x = x.to(ln_dtype)
+        x = self.ln(x)
+
+        if self.head_pre is not None:
+            head_pre_dtype = self.head_pre.weight.dtype
+            if x.dtype != head_pre_dtype:
+                x = x.to(head_pre_dtype)
+            x = self.head_pre(x)
+
+        head_dtype = self.head.weight.dtype
+        if x.dtype != head_dtype:
+            x = x.to(head_dtype)
+
+        return self.head(x)
+
+
+class BinaryLLMForCausalLM(PreTrainedModel):
+    config_class = BinaryLLMConfig
+    main_input_name = "input_ids"
+
+    def __init__(self, config: BinaryLLMConfig):
+        super().__init__(config)
+
+        attn_backend = getattr(config, "attn_backend", "auto")
+
+        inner = _InnerCfg(
+            block_size=int(config.max_position_embeddings),
+            embed_dim=int(config.hidden_size),
+            vocab_size=int(config.vocab_size),
+            num_heads=int(config.num_attention_heads),
+            num_layers=int(config.num_hidden_layers),
+            ff_hidden_dim=int(config.intermediate_size),
+            dropout=float(getattr(config, "dropout", 0.0)),
+            layernorm_dim=None,
+            head_dim=None,
+            attn_backend=str(attn_backend),
+        )
+        self.model = TinyTransformerLM(inner)
+
+        self.post_init()
+
+    def get_input_embeddings(self) -> nn.Module:
+        return self.model.tok_embed
+
+    def set_input_embeddings(self, value: nn.Module) -> None:
+        self.model.tok_embed = value
+
+    def get_output_embeddings(self) -> nn.Module:
+        return self.model.head
+
+    def set_output_embeddings(self, new_embeddings: nn.Module) -> None:
+        self.model.head = new_embeddings
+
+    def prepare_inputs_for_generation(
+        self,
+        input_ids: torch.LongTensor,
+        attention_mask: Optional[torch.Tensor] = None,
+        **kwargs,
+    ):
+        return {
+            "input_ids": input_ids,
+            "attention_mask": attention_mask,
+        }
+
+    def forward(
+        self,
+        input_ids: torch.LongTensor,
+        attention_mask: Optional[torch.Tensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+        **kwargs,
+    ) -> CausalLMOutput:
+        padding_mask = None
+        if attention_mask is not None:
+            padding_mask = ~attention_mask.to(torch.bool)
+
+        logits = self.model(input_ids, padding_mask=padding_mask)
+
+        loss = None
+        if labels is not None:
+            shift_logits = logits[:, :-1, :].contiguous()
+            shift_labels = labels[:, 1:].contiguous()
+            loss = F.cross_entropy(
+                shift_logits.view(-1, self.config.vocab_size),
+                shift_labels.view(-1),
+                ignore_index=-100,
+            )
+
+        return CausalLMOutput(loss=loss, logits=logits)

special_tokens_map.json

@@ -0,0 +1,6 @@
+{
+  "bos_token": "<BOS>",
+  "eos_token": "<EOS>",
+  "pad_token": "<EOS>",
+  "unk_token": "<EOS>"
+}

tokenization_binaryllm.py

@@ -0,0 +1,243 @@
+#!/usr/bin/env python3
+# tokenization_binaryllm.py
+# ============================================================
+# BinaryLLMTokenizer (AutoTokenizer compatible) — EXACTEMENT la même
+# tokenisation/decodage que llmTalk (mode base=65536) + infer_tagged12/11:
+#
+# - Base: 65536
+# - IDs radix: 0..65535
+# - BOS: 65536
+# - EOS: 65537
+# - UNK: alias EOS (65537) (pas de nouveau token dans la base)
+# - Encodage: UTF-8 bytes -> digits base65536 BIG-ENDIAN (chunks 2 bytes)
+#   * si longueur impaire: dernier byte encodé en valeur 0..255 (1 digit)
+# - Décodage: digits -> bytes BIG-ENDIAN -> UTF-8 (errors="replace")
+#
+# Important:
+# - build_inputs_with_special_tokens: [BOS] + seq + [EOS] (comme HF classique)
+# - encode(..., add_special_tokens=False) renvoie UNIQUEMENT les digits base65536
+# - encode(..., add_special_tokens=True) ajoute BOS/EOS via build_inputs...
+#
+# Ce fichier suffit pour `trust_remote_code=True` côté repo HF.
+# ============================================================
+
+from __future__ import annotations
+
+import json
+import os
+import re
+from typing import Dict, List, Optional, Tuple, Any
+
+from transformers import PreTrainedTokenizer
+
+
+class BinaryLLMTokenizer(PreTrainedTokenizer):
+    model_input_names = ["input_ids", "attention_mask"]
+
+    TOKEN_RE = re.compile(r"^<U([0-9A-Fa-f]{4})>$")
+
+    def __init__(
+        self,
+        bos_token: str = "<BOS>",
+        eos_token: str = "<EOS>",
+        unk_token: str = "<UNK>",
+        pad_token: Optional[str] = None,
+        **kwargs: Any,
+    ):
+        # radix strict
+        self._base_vocab_size = 65536
+
+        # specials strict: base + 0/1
+        self._bos_id = 65536
+        self._eos_id = 65537
+
+        # UNK alias EOS (pas de token additionnel)
+        self._unk_id = self._eos_id
+
+        self._bos_str = bos_token
+        self._eos_str = eos_token
+        self._unk_str = unk_token
+        self._pad_str = pad_token
+
+        super().__init__(
+            bos_token=bos_token,
+            eos_token=eos_token,
+            unk_token=unk_token,
+            pad_token=pad_token,
+            **kwargs,
+        )
+
+    # ---------- vocab / ids ----------
+
+    @property
+    def vocab_size(self) -> int:
+        # 65536 + BOS + EOS
+        return 65538
+
+    def get_vocab(self) -> Dict[str, int]:
+        # IMPORTANT: ne jamais appeler self.unk_token_id ici (boucle)
+        v = {
+            self._bos_str: self._bos_id,
+            self._eos_str: self._eos_id,
+            self._unk_str: self._unk_id,
+        }
+        if self.pad_token is not None:
+            v[self.pad_token] = self._convert_token_to_id(self.pad_token)
+        return v
+
+    def _id_to_token_base(self, i: int) -> str:
+        return f"<U{i:04X}>"
+
+    # ---------- core encode/decode (même logique que infer_tagged / llmTalk base) ----------
+
+    def _encode_to_base65536_big_endian(self, text: str) -> List[int]:
+        b = bytearray(text.encode("utf-8", errors="strict"))
+        if len(b) == 0:
+            return [0]
+
+        out: List[int] = []
+        i = 0
+        n = len(b)
+
+        while i + 1 < n:
+            # 2 bytes -> 1 digit base65536 big-endian
+            out.append((b[i] << 8) | b[i + 1])
+            i += 2
+
+        if i < n:
+            # dernier byte seul -> digit 0..255
+            out.append(int(b[i]))
+
+        return out
+
+    def _decode_from_base65536_big_endian(self, ids: List[int]) -> str:
+        bb = bytearray()
+        for x in ids:
+            xi = int(x) & 0xFFFFFFFF
+            if 0 <= xi <= 255:
+                bb.append(xi)
+            else:
+                bb.append((xi >> 8) & 0xFF)
+                bb.append(xi & 0xFF)
+        return bytes(bb).decode("utf-8", errors="replace")
+
+    # ---------- HF tokenizer API overrides ----------
+
+    def _tokenize(self, text: str) -> List[str]:
+        ids = self._encode_to_base65536_big_endian(text)
+        return [self._id_to_token_base(i) for i in ids]
+
+    def _convert_token_to_id(self, token: str) -> int:
+        if token == self._bos_str:
+            return self._bos_id
+        if token == self._eos_str:
+            return self._eos_id
+        if token == self._unk_str:
+            return self._unk_id
+
+        if self.pad_token is not None and token == self.pad_token:
+            # pas de PAD dédié => alias EOS (compatible avec ton cadre)
+            if self.pad_token == self._eos_str:
+                return self._eos_id
+            return self._eos_id
+
+        m = self.TOKEN_RE.match(token)
+        if m:
+            return int(m.group(1), 16)
+
+        return self._unk_id
+
+    def _convert_id_to_token(self, index: int) -> str:
+        if index == self._bos_id:
+            return self._bos_str
+        if index == self._eos_id:
+            return self._eos_str
+        if index == self._unk_id:
+            return self._unk_str
+
+        if self.pad_token is not None and index == self.pad_token_id:
+            return self.pad_token
+
+        if 0 <= index < self._base_vocab_size:
+            return self._id_to_token_base(index)
+
+        return self._unk_str
+
+    def convert_tokens_to_string(self, tokens: List[str]) -> str:
+        ids: List[int] = []
+        for t in tokens:
+            if t in (self._bos_str, self._eos_str, self._unk_str):
+                continue
+            if self.pad_token is not None and t == self.pad_token:
+                continue
+            m = self.TOKEN_RE.match(t)
+            if m:
+                ids.append(int(m.group(1), 16))
+        return self._decode_from_base65536_big_endian(ids)
+
+    def build_inputs_with_special_tokens(
+        self,
+        token_ids_0: List[int],
+        token_ids_1: Optional[List[int]] = None,
+    ) -> List[int]:
+        # HF-style (simple): [BOS] seq [EOS]
+        # Pair: [BOS] seq0 [EOS] seq1 [EOS]
+        if token_ids_1 is None:
+            return [self._bos_id] + token_ids_0 + [self._eos_id]
+        return [self._bos_id] + token_ids_0 + [self._eos_id] + token_ids_1 + [self._eos_id]
+
+    def get_special_tokens_mask(
+        self,
+        token_ids_0: List[int],
+        token_ids_1: Optional[List[int]] = None,
+        already_has_special_tokens: bool = False,
+    ) -> List[int]:
+        pad_id = self.pad_token_id if self.pad_token is not None else -1
+
+        if already_has_special_tokens:
+            return [
+                1 if t in (self._bos_id, self._eos_id, self._unk_id, pad_id) else 0
+                for t in token_ids_0
+            ]
+
+        if token_ids_1 is None:
+            return [1] + [0] * len(token_ids_0) + [1]
+        return [1] + [0] * len(token_ids_0) + [1] + [0] * len(token_ids_1) + [1]
+
+    def create_token_type_ids_from_sequences(
+        self,
+        token_ids_0: List[int],
+        token_ids_1: Optional[List[int]] = None,
+    ) -> List[int]:
+        if token_ids_1 is None:
+            return [0] * (len(token_ids_0) + 2)
+        return [0] * (len(token_ids_0) + len(token_ids_1) + 3)
+
+    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
+        if not os.path.isdir(save_directory):
+            os.makedirs(save_directory, exist_ok=True)
+
+        name = (filename_prefix + "-" if filename_prefix else "") + "binaryllm_vocab.json"
+        path = os.path.join(save_directory, name)
+
+        data = {
+            "base_vocab_size": 65536,
+            "vocab_size": 65538,
+            "bos_token": self._bos_str,
+            "bos_token_id": self._bos_id,
+            "eos_token": self._eos_str,
+            "eos_token_id": self._eos_id,
+            "unk_token": self._unk_str,
+            "unk_token_id": self._unk_id,
+            "pad_token": self.pad_token,
+            "pad_token_id": self.pad_token_id,
+            "encoding": "utf-8",
+            "radix": 65536,
+            "endianness": "big",
+            "odd_length_rule": "last_byte_as_single_digit_0_255",
+        }
+
+        with open(path, "w", encoding="utf-8") as f:
+            json.dump(data, f, ensure_ascii=False, indent=2)
+
+        return (path,)

tokenizer_config.json

@@ -0,0 +1,34 @@
+{
+  "added_tokens_decoder": {
+    "65536": {
+      "content": "<BOS>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65537": {
+      "content": "<EOS>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<BOS>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<EOS>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<EOS>",
+  "tokenizer_class": "BinaryLLMTokenizer",
+  "unk_token": "<EOS>",
+  "auto_map": {
+    "AutoTokenizer": [
+      "tokenization_binaryllm.BinaryLLMTokenizer",
+      null
+    ]
+  }
+}