DEBUG: WebURLs

app.py

@@ -294,7 +294,7 @@ async def loadImagePDF(vectorstore: str, pdf: UploadFile = File(...)):
         "output": text,
         "source": source
     }
-    dct = json.dumps(dct, indent=1).encode("utf-8")
+    dct = json.dumps(dct, indent=1).encode("utf-8", errors = "replace")
     fileName = createDataSourceName(sourceName=source)
     numTokens = len(" ".join([text[x] for x in text]).translate(str.maketrans('', '', string.punctuation)).split(" "))
     response = supabase.storage.from_("ConversAI").upload(file=dct, path=f"{fileName}_data.json")
@@ -329,7 +329,7 @@ async def loadText(addTextConfig: AddText):
         "source": "Text"
     }
     numTokens = len(text.translate(str.maketrans('', '', string.punctuation)).split(" "))
-    dct = json.dumps(dct, indent=1).encode("utf-8")
+    dct = json.dumps(dct, indent=1).encode("utf-8", errors = "replace")
     fileName = createDataSourceName(sourceName="Text")
     response = supabase.storage.from_("ConversAI").upload(file=dct, path=f"{fileName}_data.json")
     response = (
@@ -389,7 +389,7 @@ async def loadWebURLs(loadWebsite: LoadWebsite):
         "source": source
     }
     numTokens = len(" ".join([text[x] for x in text]).translate(str.maketrans('', '', string.punctuation)).split(" "))
-    dct = json.dumps(dct, indent=1).encode("utf-8")
+    dct = json.dumps(dct, indent=1).encode("utf-8", errors = "replace")
     fileName = createDataSourceName(sourceName=source)
     response = supabase.storage.from_("ConversAI").upload(file=dct, path=f"{fileName}_data.json")
     response = (
@@ -472,7 +472,7 @@ async def loadYoutubeTranscript(ytTranscript: YtTranscript):
         "source": "www.youtube.com"
     }
     numTokens = len(" ".join([text[x] for x in text]).translate(str.maketrans('', '', string.punctuation)).split(" "))
-    dct = json.dumps(dct, indent=1).encode("utf-8")
+    dct = json.dumps(dct, indent=1).encode("utf-8", errors = "replace")
     fileName = createDataSourceName(sourceName="youtube")
     response = supabase.storage.from_("ConversAI").upload(file=dct, path=f"{fileName}_data.json")
     response = (
@@ -534,7 +534,7 @@ async def listChatbotSources(vectorstore: str):
 async def getDataSource(vectorstore: str, sourceUrl: str):
     trackUsage(vectorstore=vectorstore, endpoint="/getDataSource")
     r = requests.get(sourceUrl)
-    return encodeToBase64(eval(r.content.decode("utf-8")))
+    return encodeToBase64(eval(r.content.decode("utf-8", errors = "replace")))
 
 
 @app.post("/deleteChatbotSource")
@@ -624,7 +624,7 @@ async def trainChatbot(trainChatbotConfig: TrainChatbot):
     for source, fileType in zip(UrlSources, fileTypes):
         if ((fileType == "/loadPDF") | (fileType == "/loadImagePDF")):
             r = requests.get(source)
-            file = eval(r.content.decode("utf-8"))
+            file = eval(r.content.decode("utf-8", errors = "replace"))
             content = file["output"]
             fileSource = file["source"]
             texts.append(".".join(
@@ -633,14 +633,14 @@ async def trainChatbot(trainChatbotConfig: TrainChatbot):
             sources.append(fileSource)
         elif fileType == "/loadText":
             r = requests.get(source)
-            file = eval(r.content.decode("utf-8"))
+            file = eval(r.content.decode("utf-8", errors = "replace"))
             content = file["output"]
             fileSource = file["source"]
             texts.append(content.replace("\n", " "))
             sources.append(fileSource)
         elif ((fileType == "/loadWebURLs") | (fileType == "/loadYoutubeTranscript")):
             r = requests.get(source)
-            file = eval(r.content.decode("utf-8"))
+            file = eval(r.content.decode("utf-8", errors = "replace"))
             content = file["output"]
             fileSource = file["source"]
             texts.append(".".join(

functions.py

@@ -348,7 +348,7 @@ def analyzeData(query, dataframe):
     response = df.chat(query)
     if os.path.isfile(response):
         with open(response, "rb") as file:
-            b64string = base64.b64encode(file.read()).decode("utf-8")
+            b64string = base64.b64encode(file.read()).decode("utf-8", errors = "replace")
         return f"data:image/png;base64,{b64string}"
     else:
         return response
@@ -384,7 +384,7 @@ def extractTextFromUrlList(urls):
 def encodeToBase64(dct: dict):
     for key in dct:
         if type(dct[key]) == str:
-            dct[key] = base64.b64encode(dct[key].encode("utf-8")).decode("utf-8", errors = "replace")
+            dct[key] = base64.b64encode(dct[key].encode("utf-8", errors = "replace")).decode("utf-8", errors = "replace")
         elif type(dct[key]) == dict:
             dct[key] = encodeToBase64(dct[key])
     return dct
@@ -392,10 +392,10 @@ def encodeToBase64(dct: dict):
 
 def decodeBase64(dct: dict):
     if type(dct["output"]) == str:
-        dct["output"] = base64.b64decode(dct["output"].encode("utf-8")).decode("utf-8", errors = "replace")
+        dct["output"] = base64.b64decode(dct["output"].encode("utf-8", errors = "replace")).decode("utf-8", errors = "replace")
     else:
         for key in dct["output"]:
-            dct["output"][key] = base64.b64decode(dct["output"][key].encode("utf-8")).decode("utf-8", errors = "replace")
+            dct["output"][key] = base64.b64decode(dct["output"][key].encode("utf-8", errors = "replace")).decode("utf-8", errors = "replace")
     return dct