Upload inference.py with huggingface_hub

inference.py

@@ -54,8 +54,15 @@ def load_ensemble(repo_id: str = None, local_dir: str = None):
         config = json.load(f)
 
     device = "cuda" if torch.cuda.is_available() else "cpu"
-    tokenizer1 = AutoTokenizer.from_pretrained(config["base_models"][0], use_fast=True)
-    tokenizer2 = AutoTokenizer.from_pretrained(config["base_models"][1], use_fast=False)
+    def _load_tok(mid, use_fast=True):
+        try:
+            return AutoTokenizer.from_pretrained(mid, use_fast=use_fast)
+        except Exception as e:
+            if "sentencepiece" in str(e).lower() and use_fast:
+                return AutoTokenizer.from_pretrained(mid, use_fast=False)
+            raise
+    tokenizer1 = _load_tok(config["base_models"][0])
+    tokenizer2 = AutoTokenizer.from_pretrained(config["base_models"][1], use_fast=False)  # PhoBERT cần use_fast=False
     model1 = AutoModelForQuestionAnswering.from_pretrained(config["base_models"][0]).to(device)
     model2 = AutoModelForQuestionAnswering.from_pretrained(config["base_models"][1]).to(device)
 
@@ -112,8 +119,18 @@ def predict(question: str, context: str, ensemble: dict, max_answer_len: int = 3
     inp1 = {k: v.to(dev) for k, v in enc1.items()}
     inp2 = {k: v.to(dev) for k, v in enc2.items()}
 
-    seq_ids = enc1.sequence_ids(0)
-    ctx_idx = [i for i, s in enumerate(seq_ids) if s == 1]
+    try:
+        seq_ids = enc1.sequence_ids(0)
+    except Exception:
+        # Slow tokenizer: RoBERTa layout [CLS] q [SEP] ctx [SEP], sep=2
+        sep_id = t1.convert_tokens_to_ids(t1.sep_token or "</s>")
+        ids = enc1["input_ids"][0].tolist()
+        sep_pos = [i for i, x in enumerate(ids) if x == sep_id]
+        if len(sep_pos) < 2:
+            return "", 1.0
+        ctx_idx = list(range(sep_pos[0] + 1, sep_pos[1]))
+    else:
+        ctx_idx = [i for i, s in enumerate(seq_ids) if s == 1]
     if not ctx_idx:
         return "", 1.0