Fixing tokenizer class

Dockerfile

@@ -5,6 +5,7 @@ WORKDIR /app
 
 # Install system dependencies
 RUN apt-get update && apt-get install -y \
+    git \
     tesseract-ocr \
     tesseract-ocr-eng \
     poppler-utils \

requirements.txt

@@ -1,8 +1,9 @@
 fastapi==0.109.0
 uvicorn[standard]==0.27.0
 python-multipart==0.0.6
-transformers>=4.45,<5.0
+transformers @ git+https://github.com/huggingface/transformers.git@main
 accelerate>=0.30
+tiktoken
 torch==2.2.2
 pypdf==4.0.1
 pdf2image==1.17.0