Fix tokenizer compatibility issue with fallback to slow tokenizer

app.py

@@ -50,7 +50,12 @@ class DesklibAIDetectionModel(PreTrainedModel):
 
 
 def load_model():
-    tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, cache_dir=HF_CACHE_DIR)
+    # Try fast tokenizer first, fall back to slow tokenizer if there's a compatibility issue
+    try:
+        tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, cache_dir=HF_CACHE_DIR, use_fast=True)
+    except Exception as e:
+        print(f"Warning: Fast tokenizer failed ({e}), falling back to slow tokenizer")
+        tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, cache_dir=HF_CACHE_DIR, use_fast=False)
     model = DesklibAIDetectionModel.from_pretrained(MODEL_ID, cache_dir=HF_CACHE_DIR)
     model.to(DEVICE)
     model.eval()