Super-squash branch 'main' using huggingface_hub

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,117 @@
+---
+library_name: transformers
+license: apache-2.0
+datasets:
+- pints-ai/Expository-Prose-V1
+language:
+- en
+---
+
+# bpe tokenizer w byte-fallback: 24k vocab
+
+BPE tokenizer for encoders/MLM objective with byte-pair fallback:
+
+- Trained on `pints-ai/Expository-Prose-V1`; this tokenizer is primarily for English and code.
+- this tokenizer is cased: "HELLO WORLD" **is different** than "hello world"
+- `model_max_length` is set to 1e9 to not cause hidden issues. **Set `tokenizer.model_max_length` to your model's max position embeddings** when training.
+
+
+## visualize
+
+
+### code
+
+```py
+from typing import Any, Callable, Optional, Union
+
+from tokenizers import Tokenizer as RustTokenizer
+from tokenizers.tools import EncodingVisualizer
+from transformers import AutoTokenizer, PreTrainedTokenizerBase
+
+SAMPLE_TEXT = """class DyT(nn.Module):
+    def __init__(self, num_features, alpha_init_value=0.5):
+        super().__init__()
+        self.alpha = nn.Parameter(torch.ones(1) * alpha_init_value)
+        self.weight = nn.Parameter(torch.ones(num_features))
+        self.bias = nn.Parameter(torch.zeros(num_features))
+
+    def forward(self, x):
+        x = torch.tanh(self.alpha * x)
+        return x * self.weight + self.bias"""
+
+
+def tokenizer_report_and_visualize(
+    tk: Union[PreTrainedTokenizerBase, RustTokenizer],
+    sample_text: str = SAMPLE_TEXT,
+    *,
+    default_to_notebook: bool = True,
+    annotation_converter: Optional[Callable[[Any], Any]] = None,
+    add_special_tokens: bool = True,
+    n_first_tokens: int = 15,
+):
+    """
+    Count tokens and ALWAYS launch tokenization visualization for the given tokenizer.
+
+    tk:
+      - Either a HuggingFace *fast* tokenizer (PreTrainedTokenizerFast subclass)
+        or a low-level `tokenizers.Tokenizer` (RustTokenizer).
+    """
+
+    # Resolve a Rust-backed tokenizer for the visualizer and do the encoding.
+    if isinstance(tk, PreTrainedTokenizerBase):
+        name = getattr(tk, "name_or_path", tk.__class__.__name__)
+        backend = getattr(tk, "backend_tokenizer", None)
+        if backend is None:
+            raise ValueError(
+                "EncodingVisualizer requires a *fast* tokenizer. "
+                "Reload with `use_fast=True` or pass a `tokenizers.Tokenizer`."
+            )
+        # mirror your original behavior that used `.encode(...)`
+        input_ids = tk.encode(
+            sample_text,
+            add_special_tokens=add_special_tokens,
+            padding=False,
+            truncation=False,
+        )
+        tokens = tk.convert_ids_to_tokens(input_ids, skip_special_tokens=False)
+        rust_tok = backend  # this is a tokenizers.Tokenizer
+    elif isinstance(tk, RustTokenizer):
+        name = "tokenizers.Tokenizer"
+        enc = tk.encode(sample_text)
+        input_ids = enc.ids
+        tokens = enc.tokens
+        rust_tok = tk
+    else:
+        raise TypeError(
+            "`tk` must be a HF *fast* tokenizer or a `tokenizers.Tokenizer`."
+        )
+
+    num_tokens = len(input_ids)
+    print(f"tokenizer ({name}): {num_tokens} tokens")
+    print(
+        "first tokens:",
+        tokens[:n_first_tokens] + (["..."] if len(tokens) > n_first_tokens else []),
+    )
+
+    # EncodingVisualizer is REQUIRED (not optional).
+    viz = EncodingVisualizer(
+        tokenizer=rust_tok,
+        default_to_notebook=default_to_notebook,
+        annotation_converter=annotation_converter,
+    )
+    viz(sample_text)
+
+    return {
+        "tokenizer_name": name,
+        "num_tokens": num_tokens,
+        "input_ids": input_ids,
+        "tokens": tokens,
+        "text": sample_text,
+    }
+
+
+# --- example usage ---
+repo_id = "pszemraj/bytebpe-tokenizer-24k-en_code-mlm"
+tk = AutoTokenizer.from_pretrained(repo_id)
+report = tokenizer_report_and_visualize(tk)
+```

special_tokens_map.json

@@ -0,0 +1,51 @@
+{
+  "bos_token": {
+    "content": "[CLS]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "content": "[CLS]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "[SEP]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "[MASK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "[SEP]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "[UNK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,55 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "[CLS]",
+  "clean_up_tokenization_spaces": false,
+  "cls_token": "[CLS]",
+  "eos_token": "[SEP]",
+  "extra_special_tokens": {},
+  "mask_token": "[MASK]",
+  "model_max_length": 1000000000.0,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "unk_token": "[UNK]"
+}