DEBUG: base64 -> plain text

app.py

@@ -329,7 +329,7 @@ async def loadText(addTextConfig: AddText):
         "output": text,
         "source": "Text"
     }
-    numTokens = len(" ".join([base64.b64decode(text[x].encode("utf-8")).decode("utf-8") for x in text]).translate(str.maketrans('', '', string.punctuation)).split(" "))
+    numTokens = len(" ".join([text[x] for x in text]).translate(str.maketrans('', '', string.punctuation)).split(" "))
     dct = json.dumps(dct, indent=1).encode("utf-8")
     fileName = createDataSourceName(sourceName="Text")
     response = supabase.storage.from_("ConversAI").upload(file=dct, path=f"{fileName}_data.json")

functions.py

@@ -297,7 +297,7 @@ def getTextFromImagePDF(pdfBytes):
         return cleanText(text = text)
 
     allImages = convert_from_bytes(pdfBytes)
-    texts = [base64.b64encode(getText(image).encode("utf-8")).decode("utf-8") for image in allImages]
+    texts = [getText(image) for image in allImages]
     return {x + 1: y for x, y in enumerate(texts)}
 
 
@@ -313,7 +313,6 @@ def getTranscript(urls: str):
         except:
             doc = ""
             texts.append(doc)
-    texts = [base64.b64encode(text.encode("utf-8")).decode("utf-8") for text in texts]
     return {x: y for x, y in zip(urls, texts)}
 
 
@@ -331,8 +330,7 @@ def analyzeData(query, dataframe):
 
 
 def extractTextFromPage(page):
-    text = cleanText(text = page.get_text())
-    return base64.b64encode(text.encode("utf-8")).decode("utf-8")
+    return cleanText(text = page.get_text())
 
 
 def extractTextFromPdf(pdf_path):
@@ -349,8 +347,7 @@ def extractTextFromUrl(url):
     response.raise_for_status()
     html = response.text
     soup = BeautifulSoup(html, 'lxml')
-    text = cleanText(text = soup.get_text(separator=' ', strip=True))
-    return base64.b64encode(text.encode("utf-8")).decode("utf-8")
+    return cleanText(text = soup.get_text(separator=' ', strip=True))
 
 
 def extractTextFromUrlList(urls):