UPDATE: trainChatbot

app.py

@@ -320,7 +320,7 @@ async def loadText(addTextConfig: AddText):
     vectorstore, text = addTextConfig.vectorstore, addTextConfig.text
     username, chatbotName = vectorstore.split("$")[1], vectorstore.split("$")[2]
     dct = {
-        "output": text,
+        "output": cleanText(text = text),
         "source": "Text"
     }
     dct = json.dumps(dct, indent=1).encode("utf-8")
@@ -544,13 +544,27 @@ async def loadEditedJson(loadEditedJsonConfig: LoadEditedJson):
                  "sourceContentURL": os.path.join(os.environ["SUPABASE_PUBLIC_BASE_URL"], f"{fileName}_data.json")})
         .execute()
     )
-
     return {
         "output": "SUCCESS"
     }
 
 
 
+@app.post("/publicOrPrivate")
+async def publicOrPrivate(vectorstore: str, mode: str = "public"):
+    username, chatbotName = vectorstore.split("$")[1], vectorstore.split("$")[2]
+    response = (
+        supabase.table("ConversAI_ChatbotInfo")
+        .update({"public/private": mode})
+        .eq("user_id", username)
+        .eq("chatbotname", chatbotName)
+        .execute()
+    )
+    return {
+        "output": "SUCCESS"
+    }
+
+
 
 class TrainChatbot(BaseModel):
     vectorstore: str

functions.py

@@ -7,6 +7,7 @@ from langchain_qdrant import QdrantVectorStore
 from langchain_qdrant import RetrievalMode
 from langchain_core.prompts.chat import ChatPromptTemplate
 from uuid import uuid4
+import nltk
 from langchain_core.output_parsers import StrOutputParser
 from langchain.retrievers import ParentDocumentRetriever
 from langchain_core.runnables.history import RunnableWithMessageHistory
@@ -120,6 +121,10 @@ def createTable(tablename: str):
         "output": "SUCCESS"
     }
 
+def cleanText(text: str):
+    text = text.replace("\n", " ")
+    text = text.translate(str.maketrans('', '', string.punctuation.replace(".", "")))
+    return text
 
 def addDocuments(texts: list[tuple[str]], vectorstore: str):
     global vectorEmbeddings
@@ -288,7 +293,8 @@ def getLinks(url: str, timeout=30):
 def getTextFromImagePDF(pdfBytes):
     def getText(image):
         global reader
-        return "\n".join([text[1] for text in reader.readtext(np.array(image), paragraph=True)])
+        text = "\n".join([text[1] for text in reader.readtext(np.array(image), paragraph=True)])
+        return cleanText(text = text)
 
     allImages = convert_from_bytes(pdfBytes)
     texts = [base64.b64encode(getText(image).encode("utf-8")).decode("utf-8") for image in allImages]
@@ -303,7 +309,7 @@ def getTranscript(urls: str):
                 url, add_video_info=False
             )
             doc = " ".join([x.page_content for x in loader.load()])
-            texts.append(doc)
+            texts.append(cleanText(text = doc))
         except:
             doc = ""
             texts.append(doc)
@@ -325,7 +331,7 @@ def analyzeData(query, dataframe):
 
 
 def extractTextFromPage(page):
-    text = page.get_text()
+    text = cleanText(text = page.get_text())
     return base64.b64encode(text.encode("utf-8")).decode("utf-8")
 
 
@@ -343,7 +349,7 @@ def extractTextFromUrl(url):
     response.raise_for_status()
     html = response.text
     soup = BeautifulSoup(html, 'lxml')
-    text = soup.get_text(separator=' ', strip=True)
+    text = cleanText(text = soup.get_text(separator=' ', strip=True))
     return base64.b64encode(text.encode("utf-8")).decode("utf-8")
 
 
@@ -361,4 +367,5 @@ def createDataSourceName(sourceName):
     i = 1
     while True:
       sourceName = sourceName + "-" + str(i)
-      return createDataSourceName(sourceName)
+      return createDataSourceName(sourceName)
+