Delete verify_tokenizer.py

verify_tokenizer.py

@@ -1,36 +0,0 @@
-from hashlib import sha256
-from pathlib import Path
-
-from tokenizers import Tokenizer
-
-
-ROOT = Path(__file__).resolve().parent
-TOKENIZER_PATH = ROOT / "tokenizer.json"
-EXPECTED_SHA256 = "7d9496a251b44688557a79fab59b5cf7a12f11a9559540b9a1e223657f35f613"
-
-actual_sha256 = sha256(TOKENIZER_PATH.read_bytes()).hexdigest()
-if actual_sha256 != EXPECTED_SHA256:
-    raise RuntimeError(
-        f"SHA-256 mismatch: expected {EXPECTED_SHA256}, got {actual_sha256}"
-    )
-
-tokenizer = Tokenizer.from_file(str(TOKENIZER_PATH))
-if tokenizer.get_vocab_size() != 65_536:
-    raise RuntimeError(f"Unexpected vocabulary size: {tokenizer.get_vocab_size()}")
-
-samples = [
-    "المملكة العربية السعودية",
-    "السلام عليكم ورحمة الله وبركاته",
-    "Artificial intelligence والذكاء الاصطناعي",
-    "ياهلا ومرحبا فيكم",
-]
-
-for text in samples:
-    encoding = tokenizer.encode(text, add_special_tokens=False)
-    decoded = tokenizer.decode(encoding.ids)
-    if decoded != text:
-        raise RuntimeError(f"Round-trip failure: {text!r} -> {decoded!r}")
-
-print("Tokenizer verification passed")
-print(f"Vocabulary size: {tokenizer.get_vocab_size():,}")
-print(f"SHA-256: {actual_sha256}")