Patenty-0.1

README.md

@@ -1,3 +1,8 @@
----
-license: apache-2.0
----
+# BinaryLLM (HF export)
+
+Tokenizer-free / base-N model export.
+
+## Load
+```python
+from transformers import AutoModelForCausalLM
+m = AutoModelForCausalLM.from_pretrained("./hf_binaryllm_repo", trust_remote_code=True)

SAVE_tokenizer_config_SAVE.json

@@ -0,0 +1,31 @@
+{
+  "added_tokens_decoder": {
+    "65536": {
+      "content": "<BOS>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65537": {
+      "content": "<UNK>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<BOS>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<EOS>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<EOS>",
+  "tokenizer_class": "tokenization_binaryllm.BinaryLLMTokenizer",
+  "unk_token": "<UNK>",
+  "auto_map": {
+    "AutoTokenizer": "tokenization_binaryllm.BinaryLLMTokenizer"
+  }
+}

__init__.py

@@ -0,0 +1,3 @@
+from .configuration_binaryllm import BinaryLLMConfig
+from .modeling_binaryllm import BinaryLLMForCausalLM
+from .tokenization_binaryllm import BinaryLLMTokenizer

binaryllm_vocab.json

@@ -0,0 +1,12 @@
+{
+  "base_vocab_size": 65536,
+  "vocab_size": 65538,
+  "bos_token": "<BOS>",
+  "bos_token_id": 65536,
+  "eos_token": "<EOS>",
+  "eos_token_id": 65537,
+  "unk_token": "<EOS>",
+  "unk_token_id": 65537,
+  "pad_token": "<EOS>",
+  "pad_token_id": 65537
+}

config.json

@@ -0,0 +1,21 @@
+{
+  "model_type": "binaryllm",
+  "architectures": ["BinaryLLMForCausalLM"],
+  "auto_map": {
+    "AutoConfig": "configuration_binaryllm.BinaryLLMConfig",
+    "AutoModelForCausalLM": "modeling_binaryllm.BinaryLLMForCausalLM",
+    "AutoTokenizer": "tokenization_binaryllm.BinaryLLMTokenizer"
+  },
+  "vocab_size": 65538,
+  "bos_token_id": 65536,
+  "eos_token_id": 65537,
+  "pad_token_id": 65537,
+  "hidden_size": 512,
+  "num_hidden_layers": 4,
+  "num_attention_heads": 4,
+  "intermediate_size": 2048,
+  "max_position_embeddings": 2048,
+  "dropout": 0.1,
+  "activation": "gelu",
+  "torch_dtype": "float32"
+}

configuration_binaryllm.py

@@ -0,0 +1,35 @@
+from transformers import PretrainedConfig
+
+
+class BinaryLLMConfig(PretrainedConfig):
+    model_type = "binaryllm"
+
+    def __init__(
+        self,
+        vocab_size: int = 65538,
+        hidden_size: int = 512,
+        num_hidden_layers: int = 4,
+        num_attention_heads: int = 4,
+        intermediate_size: int = 2048,
+        max_position_embeddings: int = 2048,
+        dropout: float = 0.1,
+        activation: str = "gelu",
+        bos_token_id: int = 65536,
+        eos_token_id: int = 65537,
+        pad_token_id: int = 65537,
+        **kwargs,
+    ):
+        self.vocab_size = int(vocab_size)
+        self.hidden_size = int(hidden_size)
+        self.num_hidden_layers = int(num_hidden_layers)
+        self.num_attention_heads = int(num_attention_heads)
+        self.intermediate_size = int(intermediate_size)
+        self.max_position_embeddings = int(max_position_embeddings)
+        self.dropout = float(dropout)
+        self.activation = str(activation)
+
+        self.bos_token_id = int(bos_token_id)
+        self.eos_token_id = int(eos_token_id)
+        self.pad_token_id = int(pad_token_id)
+
+        super().__init__(**kwargs)

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b2f7855de68552610bfb59cb6d013c08380ed3072f18b129ed2e8124701108d
+size 318891440

modeling_binaryllm.py

@@ -0,0 +1,162 @@
+import math
+from dataclasses import dataclass
+from typing import Optional
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from transformers import PreTrainedModel
+from transformers.modeling_outputs import CausalLMOutput
+
+from .configuration_binaryllm import BinaryLLMConfig
+
+
+class PositionalEncoding(nn.Module):
+    """
+    Sinusoidal positional encoding, stocké en fp32,
+    puis casté au dtype de x à chaque forward.
+    """
+
+    def __init__(self, d_model: int, max_len: int) -> None:
+        super().__init__()
+        pe = torch.zeros(max_len, d_model, dtype=torch.float32)
+        position = torch.arange(0, max_len, dtype=torch.float32).unsqueeze(1)
+        div_term = torch.exp(
+            torch.arange(0, d_model, 2, dtype=torch.float32) * (-torch.log(torch.tensor(10000.0)) / d_model)
+        )
+        pe[:, 0::2] = torch.sin(position * div_term)
+        pe[:, 1::2] = torch.cos(position * div_term)
+        pe = pe.unsqueeze(0)  # (1, max_len, d_model)
+        self.register_buffer("pe", pe, persistent=False)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        t = x.size(1)
+        pe = self.pe[:, :t, :]
+        pe = pe.to(device=x.device, dtype=x.dtype)
+        return x + pe
+
+
+@dataclass
+class _InnerCfg:
+    block_size: int
+    embed_dim: int
+    vocab_size: int
+    num_heads: int
+    num_layers: int
+    ff_hidden_dim: int
+    dropout: float
+    layernorm_dim: Optional[int] = None
+    head_dim: Optional[int] = None
+
+
+class TinyTransformerLM(nn.Module):
+    def __init__(self, cfg: _InnerCfg) -> None:
+        super().__init__()
+        self.cfg = cfg
+
+        vocab_size = cfg.vocab_size
+        self.tok_embed = nn.Embedding(vocab_size, cfg.embed_dim)
+        self.pos_encoding = PositionalEncoding(cfg.embed_dim, cfg.block_size)
+
+        encoder_layer = nn.TransformerEncoderLayer(
+            d_model=cfg.embed_dim,
+            nhead=cfg.num_heads,
+            dim_feedforward=cfg.ff_hidden_dim,
+            dropout=cfg.dropout,
+            activation="gelu",
+            batch_first=True,
+        )
+        self.encoder = nn.TransformerEncoder(encoder_layer, num_layers=cfg.num_layers)
+
+        ln_dim = cfg.layernorm_dim or cfg.embed_dim
+        head_dim = cfg.head_dim or ln_dim
+
+        self.pre_ln_proj: Optional[nn.Linear] = None
+        if ln_dim != cfg.embed_dim:
+            self.pre_ln_proj = nn.Linear(cfg.embed_dim, ln_dim)
+
+        self.ln = nn.LayerNorm(ln_dim)
+
+        self.head_pre: Optional[nn.Linear] = None
+        if head_dim != ln_dim:
+            self.head_pre = nn.Linear(ln_dim, head_dim)
+
+        self.head = nn.Linear(head_dim, vocab_size, bias=False)
+
+        # weight tying seulement si parfait alignement
+        if self.pre_ln_proj is None and self.head_pre is None and head_dim == cfg.embed_dim:
+            self.head.weight = self.tok_embed.weight
+
+        causal = torch.triu(torch.ones(cfg.block_size, cfg.block_size, dtype=torch.bool), diagonal=1)
+        self.register_buffer("causal_mask", causal, persistent=False)
+
+    def forward(self, tokens: torch.Tensor, padding_mask: Optional[torch.Tensor] = None) -> torch.Tensor:
+        x = self.tok_embed(tokens)
+        x = self.pos_encoding(x)
+
+        seq_len = tokens.size(1)
+        attn_mask = self.causal_mask[:seq_len, :seq_len].to(device=tokens.device)
+
+        if padding_mask is not None:
+            padding_mask = padding_mask[:, :seq_len].to(device=tokens.device, dtype=torch.bool)
+
+        x = self.encoder(x, mask=attn_mask, src_key_padding_mask=padding_mask)
+
+        if self.pre_ln_proj is not None:
+            x = self.pre_ln_proj(x)
+
+        x = self.ln(x)
+
+        if self.head_pre is not None:
+            x = self.head_pre(x)
+
+        return self.head(x)
+
+
+class BinaryLLMForCausalLM(PreTrainedModel):
+    config_class = BinaryLLMConfig
+    main_input_name = "input_ids"
+
+    def __init__(self, config: BinaryLLMConfig):
+        super().__init__(config)
+
+        inner = _InnerCfg(
+            block_size=int(config.max_position_embeddings),
+            embed_dim=int(config.hidden_size),
+            vocab_size=int(config.vocab_size),
+            num_heads=int(config.num_attention_heads),
+            num_layers=int(config.num_hidden_layers),
+            ff_hidden_dim=int(config.intermediate_size),
+            dropout=float(getattr(config, "dropout", 0.0)),
+            layernorm_dim=None,
+            head_dim=None,
+        )
+        self.model = TinyTransformerLM(inner)
+
+        self.post_init()
+
+    def forward(
+        self,
+        input_ids: torch.LongTensor,
+        attention_mask: Optional[torch.Tensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+        **kwargs,
+    ) -> CausalLMOutput:
+        padding_mask = None
+        if attention_mask is not None:
+            padding_mask = ~attention_mask.to(torch.bool)  # True = ignore
+
+        logits = self.model(input_ids, padding_mask=padding_mask)
+
+        loss = None
+        if labels is not None:
+            shift_logits = logits[:, :-1, :].contiguous()
+            shift_labels = labels[:, 1:].contiguous()
+            loss = F.cross_entropy(
+                shift_logits.view(-1, self.config.vocab_size),
+                shift_labels.view(-1),
+                ignore_index=-100,
+            )
+
+        return CausalLMOutput(loss=loss, logits=logits)

special_tokens_map.json

@@ -0,0 +1,6 @@
+{
+  "bos_token": "<BOS>",
+  "eos_token": "<EOS>",
+  "pad_token": "<EOS>",
+  "unk_token": "<EOS>"
+}

tokenization_binaryllm.py

@@ -0,0 +1,196 @@
+from __future__ import annotations
+
+import json
+import os
+import re
+from typing import Dict, List, Optional, Tuple
+
+from transformers import PreTrainedTokenizer
+
+
+class BinaryLLMTokenizer(PreTrainedTokenizer):
+    model_input_names = ["input_ids", "attention_mask"]
+
+    TOKEN_RE = re.compile(r"^<U([0-9A-Fa-f]{4})>$")
+
+    def __init__(
+        self,
+        bos_token: str = "<BOS>",
+        eos_token: str = "<EOS>",
+        unk_token: str = "<UNK>",
+        pad_token: Optional[str] = None,
+        **kwargs,
+    ):
+        # base ids 0..65535 reserved for radix tokens (strict)
+        self._base_vocab_size = 65536
+
+        # reserve ids
+        self._bos_id = 65536
+        self._eos_id = 65537
+
+        # UNK is an alias to EOS to preserve radix purity (no new base id)
+        self._unk_id = self._eos_id
+
+        # special token strings
+        self._bos_str = bos_token
+        self._eos_str = eos_token
+        self._unk_str = unk_token
+        self._pad_str = pad_token
+
+        super().__init__(
+            bos_token=bos_token,
+            eos_token=eos_token,
+            unk_token=unk_token,
+            pad_token=pad_token,
+            **kwargs,
+        )
+
+    @property
+    def vocab_size(self) -> int:
+        return 65538
+
+    def get_vocab(self) -> Dict[str, int]:
+        # IMPORTANT: never call self.unk_token_id here (it triggers recursion)
+        v = {
+            self._bos_str: self._bos_id,
+            self._eos_str: self._eos_id,
+            self._unk_str: self._unk_id,
+        }
+        if self.pad_token is not None:
+            # if pad_token == "<EOS>", it will map to eos id via _convert_token_to_id()
+            v[self.pad_token] = self._convert_token_to_id(self.pad_token)
+        return v
+
+    def _tokenize(self, text: str) -> List[str]:
+        ids = self._encode_to_uint16(text)
+        return [self._id_to_token_base(i) for i in ids]
+
+    def _convert_token_to_id(self, token: str) -> int:
+        if token == self._bos_str:
+            return self._bos_id
+        if token == self._eos_str:
+            return self._eos_id
+        if token == self._unk_str:
+            return self._unk_id
+
+        if self.pad_token is not None and token == self.pad_token:
+            # common case: pad_token is "<EOS>"
+            if self.pad_token == self._eos_str:
+                return self._eos_id
+            # otherwise: no dedicated PAD id in this vocab, alias to EOS
+            return self._eos_id
+
+        m = self.TOKEN_RE.match(token)
+        if m:
+            return int(m.group(1), 16)
+
+        return self._unk_id
+
+    def _convert_id_to_token(self, index: int) -> str:
+        if index == self._bos_id:
+            return self._bos_str
+        if index == self._eos_id:
+            return self._eos_str
+        if index == self._unk_id:
+            return self._unk_str
+        if self.pad_token is not None and index == self.pad_token_id:
+            return self.pad_token
+
+        if 0 <= index < self._base_vocab_size:
+            return self._id_to_token_base(index)
+
+        return self._unk_str
+
+    def convert_tokens_to_string(self, tokens: List[str]) -> str:
+        ids: List[int] = []
+        for t in tokens:
+            if t in (self._bos_str, self._eos_str, self._unk_str):
+                continue
+            if self.pad_token is not None and t == self.pad_token:
+                continue
+            m = self.TOKEN_RE.match(t)
+            if m:
+                ids.append(int(m.group(1), 16))
+        return self._decode_from_uint16(ids)
+
+    def build_inputs_with_special_tokens(
+        self,
+        token_ids_0: List[int],
+        token_ids_1: Optional[List[int]] = None,
+    ) -> List[int]:
+        if token_ids_1 is None:
+            return [self._bos_id] + token_ids_0 + [self._eos_id]
+        return [self._bos_id] + token_ids_0 + [self._eos_id] + token_ids_1 + [self._eos_id]
+
+    def get_special_tokens_mask(
+        self,
+        token_ids_0: List[int],
+        token_ids_1: Optional[List[int]] = None,
+        already_has_special_tokens: bool = False,
+    ) -> List[int]:
+        pad_id = self.pad_token_id if self.pad_token is not None else -1
+
+        if already_has_special_tokens:
+            return [
+                1 if t in (self._bos_id, self._eos_id, self._unk_id, pad_id) else 0
+                for t in token_ids_0
+            ]
+
+        if token_ids_1 is None:
+            return [1] + [0] * len(token_ids_0) + [1]
+        return [1] + [0] * len(token_ids_0) + [1] + [0] * len(token_ids_1) + [1]
+
+    def create_token_type_ids_from_sequences(
+        self,
+        token_ids_0: List[int],
+        token_ids_1: Optional[List[int]] = None,
+    ) -> List[int]:
+        if token_ids_1 is None:
+            return [0] * (len(token_ids_0) + 2)
+        return [0] * (len(token_ids_0) + len(token_ids_1) + 3)
+
+    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
+        if not os.path.isdir(save_directory):
+            os.makedirs(save_directory, exist_ok=True)
+
+        name = (filename_prefix + "-" if filename_prefix else "") + "binaryllm_vocab.json"
+        path = os.path.join(save_directory, name)
+
+        data = {
+            "base_vocab_size": 65536,
+            "vocab_size": 65538,
+            "bos_token": self._bos_str,
+            "bos_token_id": self._bos_id,
+            "eos_token": self._eos_str,
+            "eos_token_id": self._eos_id,
+            "unk_token": self._unk_str,
+            "unk_token_id": self._unk_id,
+            "pad_token": self.pad_token,
+            "pad_token_id": self.pad_token_id,
+        }
+        with open(path, "w", encoding="utf-8") as f:
+            json.dump(data, f, ensure_ascii=False, indent=2)
+
+        return (path,)
+
+    def _id_to_token_base(self, i: int) -> str:
+        return f"<U{i:04X}>"
+
+    def _encode_to_uint16(self, text: str) -> List[int]:
+        b = text.encode("utf-8", errors="strict")
+        if len(b) % 2 == 1:
+            b += b"\x00"
+        out: List[int] = []
+        for k in range(0, len(b), 2):
+            out.append(b[k] | (b[k + 1] << 8))
+        return out
+
+    def _decode_from_uint16(self, ids: List[int]) -> str:
+        bb = bytearray()
+        for x in ids:
+            x &= 0xFFFF
+            bb.append(x & 0xFF)
+            bb.append((x >> 8) & 0xFF)
+        if len(bb) and bb[-1] == 0:
+            bb = bb[:-1]
+        return bb.decode("utf-8", errors="replace")

tokenizer_config.json

@@ -0,0 +1,34 @@
+{
+  "added_tokens_decoder": {
+    "65536": {
+      "content": "<BOS>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65537": {
+      "content": "<EOS>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<BOS>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<EOS>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<EOS>",
+  "tokenizer_class": "BinaryLLMTokenizer",
+  "unk_token": "<EOS>",
+  "auto_map": {
+    "AutoTokenizer": [
+      "tokenization_binaryllm.BinaryLLMTokenizer",
+      null
+    ]
+  }
+}