Upload folder using huggingface_hub

README.md

@@ -0,0 +1,42 @@
+---
+language:
+- en
+license: mit
+tags:
+- text-generation
+library_name: transformers
+---
+
+# Nanochat
+
+Nanochat is a small language model from Andrej Karpathy, converted to HuggingFace format.
+
+## Model Details
+
+- **Architecture**: GPT-style transformer with RoPE, QK normalization, ReLU², and logits softcap
+- **Parameters**: ~393M
+- **Hidden Size**: 1280
+- **Layers**: 20
+- **Attention Heads**: 10
+- **Vocabulary**: 65536 tokens
+- **Context Length**: 2048 tokens
+
+## Usage
+
+This model uses a custom tokenizer. You must use `trust_remote_code=True` when loading.
+
+```python
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+model = AutoModelForCausalLM.from_pretrained("<model-path>", trust_remote_code=True)
+tokenizer = AutoTokenizer.from_pretrained("<model-path>", trust_remote_code=True)
+
+prompt = "Once upon a time"
+inputs = tokenizer(prompt, return_tensors="pt")
+outputs = model.generate(**inputs, max_new_tokens=50)
+print(tokenizer.decode(outputs[0]))
+```
+
+## Citation
+
+Original model by Andrej Karpathy: https://github.com/karpathy/nanochat

config.json

@@ -0,0 +1,16 @@
+{
+  "architectures": [
+    "NanoChatForCausalLM"
+  ],
+  "model_type": "nanochat",
+  "hidden_size": 1280,
+  "num_hidden_layers": 20,
+  "num_attention_heads": 10,
+  "num_key_value_heads": 10,
+  "vocab_size": 65536,
+  "max_position_embeddings": 2048,
+  "intermediate_size": 5120,
+  "rope_theta": 10000.0,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.0.0"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8799c63247e4a94c2b13c5550bc3615c77a395574a23157f627f02e171e8f80d
+size 2076193816

nanochat_tokenizer.py

@@ -0,0 +1,96 @@
+# Copyright © 2023-2024 Apple Inc.
+
+import pickle
+from pathlib import Path
+from typing import List, Optional, Union
+
+from transformers import PreTrainedTokenizer
+
+
+class NanochatTokenizer(PreTrainedTokenizer):
+    """HuggingFace-compatible wrapper for nanochat's tiktoken tokenizer."""
+
+    def __init__(self, tokenizer_path: Optional[Union[str, Path]] = None, **kwargs):
+        # Set default special tokens if not provided
+        kwargs.setdefault("unk_token", "<|endoftext|>")
+        kwargs.setdefault("bos_token", "<|endoftext|>")
+        kwargs.setdefault("eos_token", "<|endoftext|>")
+
+        # Initialize parent first
+        super().__init__(**kwargs)
+
+        # Load the tiktoken tokenizer
+        if tokenizer_path is None:
+            # When loaded via from_pretrained, use name_or_path
+            tokenizer_path = getattr(self, "name_or_path", ".")
+
+        tokenizer_path = Path(tokenizer_path)
+        tokenizer_file = tokenizer_path / "tokenizer.pkl"
+
+        if not tokenizer_file.exists():
+            # Try current directory
+            tokenizer_file = Path("tokenizer.pkl")
+
+        with open(tokenizer_file, "rb") as f:
+            self._tokenizer = pickle.load(f)
+
+        # Try to get special tokens
+        try:
+            self._eos_token_id = self._tokenizer.encode("<|endoftext|>")[0]
+        except (KeyError, IndexError):
+            self._eos_token_id = 0
+
+    @property
+    def vocab_size(self) -> int:
+        return self._tokenizer.n_vocab
+
+    def get_vocab(self):
+        # Return a minimal vocab dict - tiktoken doesn't expose full vocab
+        return {}
+
+    def _tokenize(self, text: str, **kwargs) -> List[str]:
+        # Not used by PreTrainedTokenizer's main interface
+        tokens = self._tokenizer.encode(text, allowed_special="all")
+        return [str(t) for t in tokens]
+
+    def _convert_token_to_id(self, token: str) -> int:
+        return int(token)
+
+    def _convert_id_to_token(self, index: int) -> str:
+        return str(index)
+
+    def convert_tokens_to_string(self, tokens: List[str]) -> str:
+        ids = [int(t) for t in tokens]
+        return self._tokenizer.decode(ids)
+
+    def encode(
+        self,
+        text: str,
+        add_special_tokens: bool = True,
+        **kwargs,
+    ) -> List[int]:
+        """Encode text to token IDs."""
+        return self._tokenizer.encode(text, allowed_special="all")
+
+    def decode(
+        self,
+        token_ids: List[int],
+        skip_special_tokens: bool = False,
+        **kwargs,
+    ) -> str:
+        """Decode token IDs to text."""
+        return self._tokenizer.decode(token_ids)
+
+    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> tuple:
+        """Save the tokenizer vocabulary."""
+        save_directory = Path(save_directory)
+        save_directory.mkdir(parents=True, exist_ok=True)
+
+        # Copy the tokenizer.pkl file
+        import shutil
+        src = self.name_or_path if hasattr(self, "name_or_path") else "."
+        src_file = Path(src) / "tokenizer.pkl"
+        if src_file.exists():
+            shutil.copy(src_file, save_directory / "tokenizer.pkl")
+
+        return (str(save_directory / "tokenizer.pkl"),)

special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<|bos|>",
+  "eos_token": "<|bos|>",
+  "unk_token": "<|bos|>"
+}

tokenizer.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c060565a46fe83b49d99005acba796f2a630daa7970eb49f7513b89f9fb40e0
+size 846208

tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "tokenizer_class": "NanochatTokenizer",
+  "auto_map": {
+    "AutoTokenizer": [
+      "nanochat_tokenizer.NanochatTokenizer",
+      null
+    ]
+  },
+  "clean_up_tokenization_spaces": false,
+  "bos_token": "<|bos|>",
+  "eos_token": "<|bos|>",
+  "unk_token": "<|bos|>"
+}