Duplicate from PhysiQuanty/Patent-Test-Radix-65536-AutoTokenizer_FineTune

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,12 @@
+---
+base_model:
+- PhysiQuanty/Patent-Test-Radix-65536-AutoTokenizer
+---
+# BinaryLLM (HF export)
+
+Tokenizer-free / base-N model export.
+
+## Load
+```python
+from transformers import AutoModelForCausalLM
+m = AutoModelForCausalLM.from_pretrained("./hf_binaryllm_repo", trust_remote_code=True)

SAVE_tokenizer_config_SAVE.json

@@ -0,0 +1,31 @@
+{
+  "added_tokens_decoder": {
+    "65536": {
+      "content": "<BOS>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65537": {
+      "content": "<UNK>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<BOS>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<EOS>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<EOS>",
+  "tokenizer_class": "tokenization_binaryllm.BinaryLLMTokenizer",
+  "unk_token": "<UNK>",
+  "auto_map": {
+    "AutoTokenizer": "tokenization_binaryllm.BinaryLLMTokenizer"
+  }
+}

__init__.py

@@ -0,0 +1,3 @@
+from .configuration_binaryllm import BinaryLLMConfig
+from .modeling_binaryllm import BinaryLLMForCausalLM
+from .tokenization_binaryllm import BinaryLLMTokenizer

binaryllm_vocab.json

@@ -0,0 +1,12 @@
+{
+  "base_vocab_size": 65536,
+  "vocab_size": 65538,
+  "bos_token": "<BOS>",
+  "bos_token_id": 65536,
+  "eos_token": "<EOS>",
+  "eos_token_id": 65537,
+  "unk_token": "<EOS>",
+  "unk_token_id": 65537,
+  "pad_token": "<EOS>",
+  "pad_token_id": 65537
+}

config.json

@@ -0,0 +1,21 @@
+{
+  "model_type": "binaryllm",
+  "architectures": ["BinaryLLMForCausalLM"],
+  "auto_map": {
+    "AutoConfig": "configuration_binaryllm.BinaryLLMConfig",
+    "AutoModelForCausalLM": "modeling_binaryllm.BinaryLLMForCausalLM",
+    "AutoTokenizer": "tokenization_binaryllm.BinaryLLMTokenizer"
+  },
+  "vocab_size": 65538,
+  "bos_token_id": 65536,
+  "eos_token_id": 65537,
+  "pad_token_id": 65537,
+  "hidden_size": 512,
+  "num_hidden_layers": 4,
+  "num_attention_heads": 4,
+  "intermediate_size": 2048,
+  "max_position_embeddings": 2048,
+  "dropout": 0.1,
+  "activation": "gelu",
+  "torch_dtype": "float32"
+}

configuration_binaryllm.py

@@ -0,0 +1,35 @@
+from transformers import PretrainedConfig
+
+
+class BinaryLLMConfig(PretrainedConfig):
+    model_type = "binaryllm"
+
+    def __init__(
+        self,
+        vocab_size: int = 65538,
+        hidden_size: int = 512,
+        num_hidden_layers: int = 4,
+        num_attention_heads: int = 4,
+        intermediate_size: int = 2048,
+        max_position_embeddings: int = 2048,
+        dropout: float = 0.1,
+        activation: str = "gelu",
+        bos_token_id: int = 65536,
+        eos_token_id: int = 65537,
+        pad_token_id: int = 65537,
+        **kwargs,
+    ):
+        self.vocab_size = int(vocab_size)
+        self.hidden_size = int(hidden_size)
+        self.num_hidden_layers = int(num_hidden_layers)
+        self.num_attention_heads = int(num_attention_heads)
+        self.intermediate_size = int(intermediate_size)
+        self.max_position_embeddings = int(max_position_embeddings)
+        self.dropout = float(dropout)
+        self.activation = str(activation)
+
+        self.bos_token_id = int(bos_token_id)
+        self.eos_token_id = int(eos_token_id)
+        self.pad_token_id = int(pad_token_id)
+
+        super().__init__(**kwargs)

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c038b333e996b5f06e4d916ea9fc675353931743cb6f018a1403f80d0f467ae
+size 318891472

modeling_binaryllm.py

@@ -0,0 +1,162 @@
+import math
+from dataclasses import dataclass
+from typing import Optional
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from transformers import PreTrainedModel
+from transformers.modeling_outputs import CausalLMOutput
+
+from .configuration_binaryllm import BinaryLLMConfig
+
+
+class PositionalEncoding(nn.Module):
+    """
+    Sinusoidal positional encoding, stocké en fp32,
+    puis casté au dtype de x à chaque forward.
+    """
+
+    def __init__(self, d_model: int, max_len: int) -> None:
+        super().__init__()
+        pe = torch.zeros(max_len, d_model, dtype=torch.float32)
+        position = torch.arange(0, max_len, dtype=torch.float32).unsqueeze(1)
+        div_term = torch.exp(
+            torch.arange(0, d_model, 2, dtype=torch.float32) * (-torch.log(torch.tensor(10000.0)) / d_model)
+        )
+        pe[:, 0::2] = torch.sin(position * div_term)
+        pe[:, 1::2] = torch.cos(position * div_term)
+        pe = pe.unsqueeze(0)  # (1, max_len, d_model)
+        self.register_buffer("pe", pe, persistent=False)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        t = x.size(1)
+        pe = self.pe[:, :t, :]
+        pe = pe.to(device=x.device, dtype=x.dtype)
+        return x + pe
+
+
+@dataclass
+class _InnerCfg:
+    block_size: int
+    embed_dim: int
+    vocab_size: int
+    num_heads: int
+    num_layers: int
+    ff_hidden_dim: int
+    dropout: float
+    layernorm_dim: Optional[int] = None
+    head_dim: Optional[int] = None
+
+
+class TinyTransformerLM(nn.Module):
+    def __init__(self, cfg: _InnerCfg) -> None:
+        super().__init__()
+        self.cfg = cfg
+
+        vocab_size = cfg.vocab_size
+        self.tok_embed = nn.Embedding(vocab_size, cfg.embed_dim)
+        self.pos_encoding = PositionalEncoding(cfg.embed_dim, cfg.block_size)
+
+        encoder_layer = nn.TransformerEncoderLayer(
+            d_model=cfg.embed_dim,
+            nhead=cfg.num_heads,
+            dim_feedforward=cfg.ff_hidden_dim,
+            dropout=cfg.dropout,
+            activation="gelu",
+            batch_first=True,
+        )
+        self.encoder = nn.TransformerEncoder(encoder_layer, num_layers=cfg.num_layers)
+
+        ln_dim = cfg.layernorm_dim or cfg.embed_dim
+        head_dim = cfg.head_dim or ln_dim
+
+        self.pre_ln_proj: Optional[nn.Linear] = None
+        if ln_dim != cfg.embed_dim:
+            self.pre_ln_proj = nn.Linear(cfg.embed_dim, ln_dim)
+
+        self.ln = nn.LayerNorm(ln_dim)
+
+        self.head_pre: Optional[nn.Linear] = None
+        if head_dim != ln_dim:
+            self.head_pre = nn.Linear(ln_dim, head_dim)
+
+        self.head = nn.Linear(head_dim, vocab_size, bias=False)
+
+        # weight tying seulement si parfait alignement
+        if self.pre_ln_proj is None and self.head_pre is None and head_dim == cfg.embed_dim:
+            self.head.weight = self.tok_embed.weight
+
+        causal = torch.triu(torch.ones(cfg.block_size, cfg.block_size, dtype=torch.bool), diagonal=1)
+        self.register_buffer("causal_mask", causal, persistent=False)
+
+    def forward(self, tokens: torch.Tensor, padding_mask: Optional[torch.Tensor] = None) -> torch.Tensor:
+        x = self.tok_embed(tokens)
+        x = self.pos_encoding(x)
+
+        seq_len = tokens.size(1)
+        attn_mask = self.causal_mask[:seq_len, :seq_len].to(device=tokens.device)
+
+        if padding_mask is not None:
+            padding_mask = padding_mask[:, :seq_len].to(device=tokens.device, dtype=torch.bool)
+
+        x = self.encoder(x, mask=attn_mask, src_key_padding_mask=padding_mask)
+
+        if self.pre_ln_proj is not None:
+            x = self.pre_ln_proj(x)
+
+        x = self.ln(x)
+
+        if self.head_pre is not None:
+            x = self.head_pre(x)
+
+        return self.head(x)
+
+
+class BinaryLLMForCausalLM(PreTrainedModel):
+    config_class = BinaryLLMConfig
+    main_input_name = "input_ids"
+
+    def __init__(self, config: BinaryLLMConfig):
+        super().__init__(config)
+
+        inner = _InnerCfg(
+            block_size=int(config.max_position_embeddings),
+            embed_dim=int(config.hidden_size),
+            vocab_size=int(config.vocab_size),
+            num_heads=int(config.num_attention_heads),
+            num_layers=int(config.num_hidden_layers),
+            ff_hidden_dim=int(config.intermediate_size),
+            dropout=float(getattr(config, "dropout", 0.0)),
+            layernorm_dim=None,
+            head_dim=None,
+        )
+        self.model = TinyTransformerLM(inner)
+
+        self.post_init()
+
+    def forward(
+        self,
+        input_ids: torch.LongTensor,
+        attention_mask: Optional[torch.Tensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+        **kwargs,
+    ) -> CausalLMOutput:
+        padding_mask = None
+        if attention_mask is not None:
+            padding_mask = ~attention_mask.to(torch.bool)  # True = ignore
+
+        logits = self.model(input_ids, padding_mask=padding_mask)
+
+        loss = None
+        if labels is not None:
+            shift_logits = logits[:, :-1, :].contiguous()
+            shift_labels = labels[:, 1:].contiguous()
+            loss = F.cross_entropy(
+                shift_logits.view(-1, self.config.vocab_size),
+                shift_labels.view(-1),
+                ignore_index=-100,
+            )
+
+        return CausalLMOutput(loss=loss, logits=logits)

special_tokens_map.json

@@ -0,0 +1,6 @@
+{
+  "bos_token": "<BOS>",
+  "eos_token": "<EOS>",
+  "pad_token": "<EOS>",
+  "unk_token": "<EOS>"
+}

tokenization_binaryllm.py

@@ -0,0 +1,243 @@
+#!/usr/bin/env python3
+# tokenization_binaryllm.py
+# ============================================================
+# BinaryLLMTokenizer (AutoTokenizer compatible) — EXACTEMENT la même
+# tokenisation/decodage que llmTalk (mode base=65536) + infer_tagged12/11:
+#
+# - Base: 65536
+# - IDs radix: 0..65535
+# - BOS: 65536
+# - EOS: 65537
+# - UNK: alias EOS (65537) (pas de nouveau token dans la base)
+# - Encodage: UTF-8 bytes -> digits base65536 BIG-ENDIAN (chunks 2 bytes)
+#   * si longueur impaire: dernier byte encodé en valeur 0..255 (1 digit)
+# - Décodage: digits -> bytes BIG-ENDIAN -> UTF-8 (errors="replace")
+#
+# Important:
+# - build_inputs_with_special_tokens: [BOS] + seq + [EOS] (comme HF classique)
+# - encode(..., add_special_tokens=False) renvoie UNIQUEMENT les digits base65536
+# - encode(..., add_special_tokens=True) ajoute BOS/EOS via build_inputs...
+#
+# Ce fichier suffit pour `trust_remote_code=True` côté repo HF.
+# ============================================================
+
+from __future__ import annotations
+
+import json
+import os
+import re
+from typing import Dict, List, Optional, Tuple, Any
+
+from transformers import PreTrainedTokenizer
+
+
+class BinaryLLMTokenizer(PreTrainedTokenizer):
+    model_input_names = ["input_ids", "attention_mask"]
+
+    TOKEN_RE = re.compile(r"^<U([0-9A-Fa-f]{4})>$")
+
+    def __init__(
+        self,
+        bos_token: str = "<BOS>",
+        eos_token: str = "<EOS>",
+        unk_token: str = "<UNK>",
+        pad_token: Optional[str] = None,
+        **kwargs: Any,
+    ):
+        # radix strict
+        self._base_vocab_size = 65536
+
+        # specials strict: base + 0/1
+        self._bos_id = 65536
+        self._eos_id = 65537
+
+        # UNK alias EOS (pas de token additionnel)
+        self._unk_id = self._eos_id
+
+        self._bos_str = bos_token
+        self._eos_str = eos_token
+        self._unk_str = unk_token
+        self._pad_str = pad_token
+
+        super().__init__(
+            bos_token=bos_token,
+            eos_token=eos_token,
+            unk_token=unk_token,
+            pad_token=pad_token,
+            **kwargs,
+        )
+
+    # ---------- vocab / ids ----------
+
+    @property
+    def vocab_size(self) -> int:
+        # 65536 + BOS + EOS
+        return 65538
+
+    def get_vocab(self) -> Dict[str, int]:
+        # IMPORTANT: ne jamais appeler self.unk_token_id ici (boucle)
+        v = {
+            self._bos_str: self._bos_id,
+            self._eos_str: self._eos_id,
+            self._unk_str: self._unk_id,
+        }
+        if self.pad_token is not None:
+            v[self.pad_token] = self._convert_token_to_id(self.pad_token)
+        return v
+
+    def _id_to_token_base(self, i: int) -> str:
+        return f"<U{i:04X}>"
+
+    # ---------- core encode/decode (même logique que infer_tagged / llmTalk base) ----------
+
+    def _encode_to_base65536_big_endian(self, text: str) -> List[int]:
+        b = bytearray(text.encode("utf-8", errors="strict"))
+        if len(b) == 0:
+            return [0]
+
+        out: List[int] = []
+        i = 0
+        n = len(b)
+
+        while i + 1 < n:
+            # 2 bytes -> 1 digit base65536 big-endian
+            out.append((b[i] << 8) | b[i + 1])
+            i += 2
+
+        if i < n:
+            # dernier byte seul -> digit 0..255
+            out.append(int(b[i]))
+
+        return out
+
+    def _decode_from_base65536_big_endian(self, ids: List[int]) -> str:
+        bb = bytearray()
+        for x in ids:
+            xi = int(x) & 0xFFFFFFFF
+            if 0 <= xi <= 255:
+                bb.append(xi)
+            else:
+                bb.append((xi >> 8) & 0xFF)
+                bb.append(xi & 0xFF)
+        return bytes(bb).decode("utf-8", errors="replace")
+
+    # ---------- HF tokenizer API overrides ----------
+
+    def _tokenize(self, text: str) -> List[str]:
+        ids = self._encode_to_base65536_big_endian(text)
+        return [self._id_to_token_base(i) for i in ids]
+
+    def _convert_token_to_id(self, token: str) -> int:
+        if token == self._bos_str:
+            return self._bos_id
+        if token == self._eos_str:
+            return self._eos_id
+        if token == self._unk_str:
+            return self._unk_id
+
+        if self.pad_token is not None and token == self.pad_token:
+            # pas de PAD dédié => alias EOS (compatible avec ton cadre)
+            if self.pad_token == self._eos_str:
+                return self._eos_id
+            return self._eos_id
+
+        m = self.TOKEN_RE.match(token)
+        if m:
+            return int(m.group(1), 16)
+
+        return self._unk_id
+
+    def _convert_id_to_token(self, index: int) -> str:
+        if index == self._bos_id:
+            return self._bos_str
+        if index == self._eos_id:
+            return self._eos_str
+        if index == self._unk_id:
+            return self._unk_str
+
+        if self.pad_token is not None and index == self.pad_token_id:
+            return self.pad_token
+
+        if 0 <= index < self._base_vocab_size:
+            return self._id_to_token_base(index)
+
+        return self._unk_str
+
+    def convert_tokens_to_string(self, tokens: List[str]) -> str:
+        ids: List[int] = []
+        for t in tokens:
+            if t in (self._bos_str, self._eos_str, self._unk_str):
+                continue
+            if self.pad_token is not None and t == self.pad_token:
+                continue
+            m = self.TOKEN_RE.match(t)
+            if m:
+                ids.append(int(m.group(1), 16))
+        return self._decode_from_base65536_big_endian(ids)
+
+    def build_inputs_with_special_tokens(
+        self,
+        token_ids_0: List[int],
+        token_ids_1: Optional[List[int]] = None,
+    ) -> List[int]:
+        # HF-style (simple): [BOS] seq [EOS]
+        # Pair: [BOS] seq0 [EOS] seq1 [EOS]
+        if token_ids_1 is None:
+            return [self._bos_id] + token_ids_0 + [self._eos_id]
+        return [self._bos_id] + token_ids_0 + [self._eos_id] + token_ids_1 + [self._eos_id]
+
+    def get_special_tokens_mask(
+        self,
+        token_ids_0: List[int],
+        token_ids_1: Optional[List[int]] = None,
+        already_has_special_tokens: bool = False,
+    ) -> List[int]:
+        pad_id = self.pad_token_id if self.pad_token is not None else -1
+
+        if already_has_special_tokens:
+            return [
+                1 if t in (self._bos_id, self._eos_id, self._unk_id, pad_id) else 0
+                for t in token_ids_0
+            ]
+
+        if token_ids_1 is None:
+            return [1] + [0] * len(token_ids_0) + [1]
+        return [1] + [0] * len(token_ids_0) + [1] + [0] * len(token_ids_1) + [1]
+
+    def create_token_type_ids_from_sequences(
+        self,
+        token_ids_0: List[int],
+        token_ids_1: Optional[List[int]] = None,
+    ) -> List[int]:
+        if token_ids_1 is None:
+            return [0] * (len(token_ids_0) + 2)
+        return [0] * (len(token_ids_0) + len(token_ids_1) + 3)
+
+    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
+        if not os.path.isdir(save_directory):
+            os.makedirs(save_directory, exist_ok=True)
+
+        name = (filename_prefix + "-" if filename_prefix else "") + "binaryllm_vocab.json"
+        path = os.path.join(save_directory, name)
+
+        data = {
+            "base_vocab_size": 65536,
+            "vocab_size": 65538,
+            "bos_token": self._bos_str,
+            "bos_token_id": self._bos_id,
+            "eos_token": self._eos_str,
+            "eos_token_id": self._eos_id,
+            "unk_token": self._unk_str,
+            "unk_token_id": self._unk_id,
+            "pad_token": self.pad_token,
+            "pad_token_id": self.pad_token_id,
+            "encoding": "utf-8",
+            "radix": 65536,
+            "endianness": "big",
+            "odd_length_rule": "last_byte_as_single_digit_0_255",
+        }
+
+        with open(path, "w", encoding="utf-8") as f:
+            json.dump(data, f, ensure_ascii=False, indent=2)
+
+        return (path,)

tokenizer_config.json

@@ -0,0 +1,34 @@
+{
+  "added_tokens_decoder": {
+    "65536": {
+      "content": "<BOS>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65537": {
+      "content": "<EOS>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<BOS>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<EOS>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<EOS>",
+  "tokenizer_class": "BinaryLLMTokenizer",
+  "unk_token": "<EOS>",
+  "auto_map": {
+    "AutoTokenizer": [
+      "tokenization_binaryllm.BinaryLLMTokenizer",
+      null
+    ]
+  }
+}