Update README & Add patch, script

README.md

@@ -1,12 +1,51 @@
 ---
 title: Exllamav2 Patch
-emoji: 🌖
-colorFrom: red
-colorTo: indigo
 sdk: static
-pinned: false
 license: mit
 short_description: Patch for ExLlamaV2 to support Unigram-based tokenizers
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Unigram Tokenizer Patch for ExLlamaV2
+
+This repository provides a patch to support **Unigram-based tokenizers** with `ExLlamaV2`.
+By default, using a Unigram tokenizer may result in an error during inference. This patch resolves that issue.
+
+---
+
+## Files
+
+- `hf.py.patch`
+  A patch file that modifies `hf.py` in the ExLlamaV2 repository to support Unigram tokenizers.
+
+- `apply_patch.sh`
+  A shell script to apply the patch.
+
+---
+
+## Usage
+
+You can apply the patch with the following command:
+
+```bash
+$ ./apply_patch.sh EXLLAMAV2_DIR
+```
+
+Replace EXLLAMAV2_DIR with the path to your local clone of ExLlamaV2.
+
+Example:
+```bash
+$ ./apply_patch.sh ~/repos/exllamav2
+```
+
+## Purpose
+
+The patch is intended to:
+- Prevent runtime errors when using tokenizers.models.Unigram.
+- Add fallback handling for missing unk_token attributes.
+- Improve compatibility with models trained using SentencePiece + Unigram tokenizers.
+
+## Notes
+
+- This patch is intended to be applied to the hf.py file in the ExLlamaV2 repository.
+- Tested Versions: **v0.2.6** through **v0.2.8**
+- If the patch fails, please ensure your copy of hf.py matches the original before applying.

apply_patch.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+THIS_DIR="$(cd $(dirname ${BASH_SOURCE}); pwd)"
+EXLLAMAV2_DIR="$1"
+PATCH_FILE="${THIS_DIR}/hf.py.patch"
+
+main() {
+    if [[ -z "${EXLLAMAV2_DIR}" ]]; then
+        echo "[Usage] $0 EXLLAMAV2_DIR"
+        exit 1
+    fi
+
+    if [[ ! -d "${EXLLAMAV2_DIR}" ]]; then
+        echo "[ERROR] EXLLAMAV2_DIR does not exist. (${EXLLAMAV2_DIR})" >&2
+        exit 1
+    fi
+
+    if [[ ! -e "${PATCH_FILE}" ]]; then
+        echo "[ERROR] Patch file not found. (${PATCH_FILE})" >&2
+        exit 1
+    fi
+
+    if ! (cd "${EXLLAMAV2_DIR}" && patch -p1 < "${PATCH_FILE}"); then
+        echo "[ERROR] Failed to apply patch (${PATCH_FILE})" >&2
+        exit 1
+    fi
+}
+main "$@"

hf.py.patch

@@ -0,0 +1,56 @@
+diff --git a/exllamav2/tokenizer/hf.py b/exllamav2/tokenizer/hf.py
+index 56134d0..9fde261 100644
+--- a/exllamav2/tokenizer/hf.py
++++ b/exllamav2/tokenizer/hf.py
+@@ -1,4 +1,5 @@
+ from __future__ import annotations
++import json
+ from typing import List, Union
+ from exllamav2.tokenizer.base import ExLlamaV2TokenizerBase
+ from tokenizers import Tokenizer
+@@ -10,6 +11,7 @@ class ExLlamaV2TokenizerHF(ExLlamaV2TokenizerBase):
+ 
+     space_char_: str
+     newline_char_: str
++    unk_token_: str | None
+     vocab: list[str] | None
+ 
+     def __init__(self, tokenizer_json: str) -> None:
+@@ -18,6 +20,7 @@ class ExLlamaV2TokenizerHF(ExLlamaV2TokenizerBase):
+         self.vocab = None
+         self.space_char_ = " "
+         self.newline_char_ = "\n"
++        self.unk_token_ = None
+ 
+         self.hf_tokenizer = Tokenizer.from_file(tokenizer_json)
+ 
+@@ -26,11 +29,18 @@ class ExLlamaV2TokenizerHF(ExLlamaV2TokenizerBase):
+             self.space_char_ = self.deduce_char_map(" ")  # "Ġ"
+             self.newline_char_ = self.deduce_char_map("\n")  # "Ċ"
+ 
++        if isinstance(m, models.Unigram):
++            unk_id = self._get_unk_id_from_tokenizer_json(tokenizer_json)
++            self.unk_token_ = self.id_to_piece(unk_id)
++        else:
++            self.unk_token_ = getattr(m, "unk_token", None)
++
+     def unk_id(self) -> int or None: return None if self.unk_token() is None else self.piece_to_id(self.unk_token())
+     def pad_id(self) -> int or None: return None
+     def bos_id(self) -> int or None: return None
+     def eos_id(self) -> int or None: return None
+-    def unk_token(self) -> str or None: return self.hf_tokenizer.model.unk_token
++    # def unk_token(self) -> str or None: return self.hf_tokenizer.model.unk_token
++    def unk_token(self) -> str or None: return self.unk_token_
+     def pad_token(self) -> str or None: return None
+     def bos_token(self) -> str or None: return None
+     def eos_token(self) -> str or None: return None
+@@ -84,3 +94,9 @@ class ExLlamaV2TokenizerHF(ExLlamaV2TokenizerBase):
+     def encode(self, text: list or str) -> list:
+         encoding = self.hf_tokenizer.encode(text, add_special_tokens = False)
+         return encoding.ids
++
++    @staticmethod
++    def _get_unk_id_from_tokenizer_json(tokenizer_json: str) -> str | None:
++        with open(tokenizer_json, "r", encoding="utf-8") as f:
++            tokenizer_json = json.load(f)
++        return tokenizer_json.get("model", {}).get("unk_id", None)

index.html

@@ -1,19 +0,0 @@
-<!doctype html>
-<html>
-	<head>
-		<meta charset="utf-8" />
-		<meta name="viewport" content="width=device-width" />
-		<title>My static Space</title>
-		<link rel="stylesheet" href="style.css" />
-	</head>
-	<body>
-		<div class="card">
-			<h1>Welcome to your static Space!</h1>
-			<p>You can modify this app directly by editing <i>index.html</i> in the Files and versions tab.</p>
-			<p>
-				Also don't forget to check the
-				<a href="https://huggingface.co/docs/hub/spaces" target="_blank">Spaces documentation</a>.
-			</p>
-		</div>
-	</body>
-</html>

style.css

@@ -1,28 +0,0 @@
-body {
-	padding: 2rem;
-	font-family: -apple-system, BlinkMacSystemFont, "Arial", sans-serif;
-}
-
-h1 {
-	font-size: 16px;
-	margin-top: 0;
-}
-
-p {
-	color: rgb(107, 114, 128);
-	font-size: 15px;
-	margin-bottom: 10px;
-	margin-top: 5px;
-}
-
-.card {
-	max-width: 620px;
-	margin: 0 auto;
-	padding: 16px;
-	border: 1px solid lightgray;
-	border-radius: 16px;
-}
-
-.card p:last-child {
-	margin-bottom: 0;
-}