Fix HF Space tokenizer startup crash

app.py

@@ -1,31 +1,84 @@
 import gradio as gr
 import os
+import json
+import logging
+import transformers
+import huggingface_hub
 from huggingface_hub import snapshot_download
 from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, RobertaTokenizer
 import torch
 
+try:
+    import tokenizers
+except Exception:  # pragma: no cover - diagnostics only
+    tokenizers = None
+
 # 加载 CodeT5+ 模型
 model_name = "Salesforce/codet5p-220m"
 
+logger = logging.getLogger(__name__)
+logging.basicConfig(level=logging.INFO)
+
+
+def log_runtime_versions() -> None:
+    """Log runtime package versions to simplify Space startup debugging."""
+    tokenizers_version = getattr(tokenizers, "__version__", "not-installed")
+    logger.info("transformers version: %s", transformers.__version__)
+    logger.info("huggingface_hub version: %s", huggingface_hub.__version__)
+    logger.info("tokenizers version: %s", tokenizers_version)
+
+
+def sanitize_added_tokens_file(added_tokens_file: str) -> None:
+    """Normalize malformed added_tokens.json to list format expected by tokenizers."""
+    if not os.path.exists(added_tokens_file):
+        return
+
+    try:
+        with open(added_tokens_file, "r", encoding="utf-8") as fp:
+            data = json.load(fp)
+    except Exception:
+        data = []
+
+    if isinstance(data, list):
+        sanitized = [item for item in data if isinstance(item, str)]
+    elif isinstance(data, dict):
+        # Some repos store empty/object payloads here; tokenizer expects a list.
+        sanitized = [key for key in data.keys() if isinstance(key, str)]
+    else:
+        sanitized = []
+
+    with open(added_tokens_file, "w", encoding="utf-8") as fp:
+        json.dump(sanitized, fp, ensure_ascii=True)
+
 
 def prepare_local_model(repo_id: str, local_dir: str = "./model_cache") -> str:
     snapshot_download(repo_id=repo_id, local_dir=local_dir)
 
-    # Work around a transformers/tokenizers incompatibility for this repo.
-    # Its added_tokens.json is an empty dict, which can crash tokenizer init in some versions.
+    # Work around tokenizer metadata incompatibility in some runtime combos.
     added_tokens_file = os.path.join(local_dir, "added_tokens.json")
-    if os.path.exists(added_tokens_file):
-        os.remove(added_tokens_file)
+    sanitize_added_tokens_file(added_tokens_file)
 
     return local_dir
 
 
+log_runtime_versions()
 local_model_dir = prepare_local_model(model_name)
+auto_error = None
 try:
     tokenizer = AutoTokenizer.from_pretrained(local_model_dir, use_fast=False, trust_remote_code=False)
-except Exception:
+    logger.info("Tokenizer loaded with AutoTokenizer (slow mode).")
+except Exception as exc:
+    auto_error = exc
+    logger.warning("AutoTokenizer load failed, trying RobertaTokenizer fallback: %s", exc)
     # Fallback to explicit slow tokenizer class to bypass tokenizers fast-path issues.
-    tokenizer = RobertaTokenizer.from_pretrained(local_model_dir, trust_remote_code=False)
+    try:
+        tokenizer = RobertaTokenizer.from_pretrained(local_model_dir, trust_remote_code=False)
+        logger.info("Tokenizer loaded with RobertaTokenizer fallback.")
+    except Exception as fallback_exc:
+        raise RuntimeError(
+            "Tokenizer initialization failed for both AutoTokenizer and RobertaTokenizer. "
+            f"AutoTokenizer error: {auto_error}; RobertaTokenizer error: {fallback_exc}"
+        ) from fallback_exc
 
 model = AutoModelForSeq2SeqLM.from_pretrained(local_model_dir, trust_remote_code=False)
 

requirements.txt

@@ -1,6 +1,7 @@
-transformers>=4.40.0
-huggingface_hub>=0.23.0
-torch>=2.0.0
+transformers==4.40.2
+huggingface_hub==0.23.2
+tokenizers==0.13.3
+torch==2.1.2
 sentencepiece>=0.1.96
 accelerate>=0.20.0
 datasets>=2.0.0