DEBUG: nltk

app.py

@@ -2,7 +2,7 @@ import io
 import tempfile
 from ipaddress import ip_address
 from typing import Optional
-
+import nltk
 import jwt
 import base64
 import json
@@ -21,6 +21,9 @@ from collections import Counter, defaultdict
 from datetime import datetime, timedelta
 from dateutil.parser import isoparse
 
+
+nltk.download("punkt_tab")
+
 app = FastAPI(title="ConversAI", root_path="/api/v1")
 
 app.add_middleware(
@@ -263,6 +266,7 @@ async def loadPDF(vectorstore: str, pdf: UploadFile = File(...)):
         "output": text,
         "source": source
     }
+    numTokens = len(" ".join([text[x] for x in text]).translate(str.maketrans('', '', string.punctuation)).split(" "))
     dct = json.dumps(dct, indent=1).encode("utf-8")
     fileName = createDataSourceName(sourceName=source)
     response = supabase.storage.from_("ConversAI").upload(file=dct, path=f"{fileName}_data.json")
@@ -271,6 +275,7 @@ async def loadPDF(vectorstore: str, pdf: UploadFile = File(...)):
         .insert({"username": username,
                  "chatbotName": chatbotName,
                  "dataSourceName": fileName,
+                 "numTokens": numTokens,
                  "sourceEndpoint": "/loadPDF",
                  "sourceContentURL": os.path.join(os.environ["SUPABASE_PUBLIC_BASE_URL"], f"{fileName}_data.json")})
         .execute()

functions.py

@@ -7,7 +7,6 @@ from langchain_qdrant import QdrantVectorStore
 from langchain_qdrant import RetrievalMode
 from langchain_core.prompts.chat import ChatPromptTemplate
 from uuid import uuid4
-import nltk
 from langchain_core.output_parsers import StrOutputParser
 from langchain.retrievers import ParentDocumentRetriever
 from langchain_core.runnables.history import RunnableWithMessageHistory
@@ -35,8 +34,6 @@ import base64
 import time
 import requests
 
-nltk.download('punkt_tab')
-
 
 load_dotenv("secrets.env")
 client = create_client(os.environ["SUPABASE_URL"], os.environ["SUPABASE_KEY"])