Update src/prediction_compile.py

src/prediction_compile.py

@@ -5,6 +5,8 @@ import pickle
 import joblib
 import nltk
 import os
+import sys
+import types
 import numpy as np
 import pandas as pd
 from tensorflow.keras.preprocessing.sequence import pad_sequences
@@ -33,8 +35,19 @@ st.markdown(
 def patch_transformers_pickle_compat() -> None:
     try:
         from transformers.models.bert import modeling_bert
+        from transformers import BertTokenizerFast
+
+        # Fix old attention symbol
         if not hasattr(modeling_bert, "BertSdpaSelfAttention"):
             modeling_bert.BertSdpaSelfAttention = modeling_bert.BertSelfAttention
+
+        # Fix old tokenizer module path
+        module_name = "transformers.models.bert.tokenization_bert_fast"
+        if module_name not in sys.modules:
+            shim = types.ModuleType(module_name)
+            shim.BertTokenizerFast = BertTokenizerFast
+            sys.modules[module_name] = shim
+
     except Exception:
         pass
 
@@ -56,7 +69,7 @@ def load_tokenizer_params():
 
 @st.cache_resource
 def load_topic_models():
-    patch_transformers_pickle_compat()  # must run before joblib.load
+    patch_transformers_pickle_compat()
     neg_path = "./src/fastopic_negative_model_10.pkl"
     pos_path = "./src/fastopic_positive_model_10.pkl"
     neg_model = joblib.load(neg_path)