DEBUG: base64 -> plain text

app.py

@@ -2,9 +2,8 @@ import io
 import tempfile
 from ipaddress import ip_address
 from typing import Optional
-import nltk
-import jwt
 import base64
+import jwt
 import json
 from click import option
 from jwt import ExpiredSignatureError, InvalidTokenError
@@ -265,7 +264,7 @@ async def loadPDF(vectorstore: str, pdf: UploadFile = File(...)):
         "output": text,
         "source": source
     }
-    numTokens = len(" ".join([base64.b64decode(text[x].encode("utf-8")).decode("utf-8") for x in text]).translate(str.maketrans('', '', string.punctuation)).split(" "))
+    numTokens = len(" ".join([text[x] for x in text]).translate(str.maketrans('', '', string.punctuation)).split(" "))
     dct = json.dumps(dct, indent=1).encode("utf-8")
     fileName = createDataSourceName(sourceName=source)
     response = supabase.storage.from_("ConversAI").upload(file=dct, path=f"{fileName}_data.json")
@@ -297,7 +296,7 @@ async def loadImagePDF(vectorstore: str, pdf: UploadFile = File(...)):
     }
     dct = json.dumps(dct, indent=1).encode("utf-8")
     fileName = createDataSourceName(sourceName=source)
-    numTokens = len(" ".join([base64.b64decode(text[x].encode("utf-8")).decode("utf-8") for x in text]).translate(str.maketrans('', '', string.punctuation)).split(" "))
+    numTokens = len(" ".join([text[x] for x in text]).translate(str.maketrans('', '', string.punctuation)).split(" "))
     response = supabase.storage.from_("ConversAI").upload(file=dct, path=f"{fileName}_data.json")
     response = (
         supabase.table("ConversAI_ChatbotDataSources")
@@ -321,8 +320,8 @@ class AddText(BaseModel):
 
 @app.post("/loadText")
 async def loadText(addTextConfig: AddText):
-    trackUsage(vectorstore=vectorstore, endpoint="/loadText")
     vectorstore, text = addTextConfig.vectorstore, addTextConfig.text
+    trackUsage(vectorstore=vectorstore, endpoint="/loadText")
     username, chatbotName = vectorstore.split("$")[1], vectorstore.split("$")[2]
     text = cleanText(text = text)
     dct = {
@@ -389,7 +388,7 @@ async def loadWebURLs(loadWebsite: LoadWebsite):
         "output": text,
         "source": source
     }
-    numTokens = len(" ".join([base64.b64decode(text[x].encode("utf-8")).decode("utf-8") for x in text]).translate(str.maketrans('', '', string.punctuation)).split(" "))
+    numTokens = len(" ".join([text[x] for x in text]).translate(str.maketrans('', '', string.punctuation)).split(" "))
     dct = json.dumps(dct, indent=1).encode("utf-8")
     fileName = createDataSourceName(sourceName=source)
     response = supabase.storage.from_("ConversAI").upload(file=dct, path=f"{fileName}_data.json")
@@ -472,7 +471,7 @@ async def loadYoutubeTranscript(ytTranscript: YtTranscript):
         "output": text,
         "source": "www.youtube.com"
     }
-    numTokens = len(" ".join([base64.b64decode(text[x].encode("utf-8")).decode("utf-8") for x in text]).translate(str.maketrans('', '', string.punctuation)).split(" "))
+    numTokens = len(" ".join([text[x] for x in text]).translate(str.maketrans('', '', string.punctuation)).split(" "))
     dct = json.dumps(dct, indent=1).encode("utf-8")
     fileName = createDataSourceName(sourceName="youtube")
     response = supabase.storage.from_("ConversAI").upload(file=dct, path=f"{fileName}_data.json")
@@ -531,6 +530,13 @@ async def listChatbotSources(vectorstore: str):
     return result
 
 
+@app.post("/getDataSource")
+async def getDataSource(vectorstore: str, sourceUrl: str):
+    trackUsage(vectorstore=vectorstore, endpoint="/getDataSource")
+    r = requests.get(sourceUrl)
+    return encodeToBase64(eval(r.content.decode("utf-8")))
+
+
 @app.post("/deleteChatbotSource")
 async def deleteChatbotSource(vectorstore: str, dataSourceName: str):
     trackUsage(vectorstore=vectorstore, endpoint="/deleteChatbotSource")
@@ -552,7 +558,8 @@ class LoadEditedJson(BaseModel):
 async def loadEditedJson(loadEditedJsonConfig: LoadEditedJson):
     username, chatbotName = loadEditedJsonConfig.vectorstore.split("$")[1], loadEditedJsonConfig.vectorstore.split("$")[2]
     trackUsage(vectorstore=loadEditedJsonConfig.vectorstore, endpoint="/loadEditedJson")
-    jsonData = json.dumps(loadEditedJsonConfig.jsonData, indent = 1).encode("utf-8")
+    jsonData = decodeBase64(loadEditedJsonConfig.jsonData)
+    jsonData = json.dumps(jsonData, indent = 1).encode("utf-8")
     fileName = createDataSourceName(loadEditedJsonConfig.dataSourceName)
     response = supabase.storage.from_("ConversAI").upload(file=jsonData, path=f"{fileName}_data.json")
     response = (
@@ -608,7 +615,7 @@ async def trainChatbot(trainChatbotConfig: TrainChatbot):
             content = file["output"]
             fileSource = file["source"]
             texts.append(".".join(
-                [base64.b64decode(content[key].encode("utf-8")).decode("utf-8") for key in content.keys()]).replace(
+                [content[key] for key in content.keys()]).replace(
                 "\n", " "))
             sources.append(fileSource)
         elif fileType == "/loadText":
@@ -624,7 +631,7 @@ async def trainChatbot(trainChatbotConfig: TrainChatbot):
             content = file["output"]
             fileSource = file["source"]
             texts.append(".".join(
-                [base64.b64decode(content[key].encode("utf-8")).decode("utf-8") for key in content.keys()]).replace(
+                [content[key] for key in content.keys()]).replace(
                 "\n", " "))
             sources.append(fileSource)
         else:

functions.py

@@ -356,6 +356,24 @@ def extractTextFromUrlList(urls):
     return {x: y for x, y in zip(urls, texts)}
 
 
+def encodeToBase64(dct: dict):
+    for key in dct:
+        if type(dct[key]) == str:
+            dct[key] = base64.b64encode(dct[key].encode("utf-8")).decode("utf-8")
+        elif type(dct[key]) == dict:
+            dct[key] = encodeToBase64(dct[key])
+    return dct
+
+
+def decodeBase64(dct: dict):
+    for key in dct:
+        if type(dct[key]) == str:
+            dct[key] = base64.b64decode(dct[key].encode("utf-8")).decode("utf-8")
+        elif type(dct[key]) == dict:
+            dct[key] = decodeBase64(dct[key])
+    return dct
+
+
 def createDataSourceName(sourceName):
   sources = [x["dataSourceName"] for x in client.table("ConversAI_ChatbotDataSources").select("dataSourceName").execute().data]
   if sourceName not in sources: