Fix tokenizer issue by using slow tokenizer

app.py

@@ -1,5 +1,5 @@
 import gradio as gr
-from transformers import AutoModelForSeq2SeqLM, AutoModelForCausalLM, AutoTokenizer
+from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, AutoModelForCausalLM
 from PyPDF2 import PdfReader
 
 # Models and tokenizers setup
@@ -10,11 +10,11 @@ models = {
     },
     "PDF Summarizer (T5)": {
         "model": AutoModelForSeq2SeqLM.from_pretrained("t5-small"),
-        "tokenizer": AutoTokenizer.from_pretrained("t5-small"),
+        "tokenizer": AutoTokenizer.from_pretrained("t5-small", use_fast=False),  # Use the slow tokenizer
     },
     "Broken Answer (T0pp)": {
         "model": AutoModelForSeq2SeqLM.from_pretrained("bigscience/T0pp"),
-        "tokenizer": AutoTokenizer.from_pretrained("bigscience/T0pp"),
+        "tokenizer": AutoTokenizer.from_pretrained("bigscience/T0pp", use_fast=False),  # Use the slow tokenizer
     },
 }
 

requirements.txt

@@ -2,3 +2,4 @@ torch
 gradio
 transformers
 PyPDF2
+sentencepiece