UPDATE: web crawler

Dockerfile

@@ -12,7 +12,7 @@ RUN apt-get update && apt-get install -y \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
     
-RUN pip install -r requirements.txt
+RUN pip install --no-cache-dir -r requirements.txt
 
 EXPOSE 7860
 

app.py

@@ -50,7 +50,7 @@ async def addText(vectorstore: str, text: str):
 
 @app.post("/addWebsite")
 async def addWebsite(vectorstore: str, websiteUrl: str):
-    urls = getLinks("https://www.youtube.com/watch?v=dQw4w9WgXcQ")
+    urls = getLinks(websiteUrl)
     loader = UnstructuredURLLoader(urls=urls)
     docs = loader.load()
     text = "\n\n\n\n".join([f"Metadata:\n{docs[doc].metadata} \nPage Content:\n {docs[doc].page_content}" for doc in range(len(docs))])

secrets.env

@@ -3,4 +3,5 @@ SUPABASE_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZi
 GROQ_API_KEY=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
 QDRANT_URL=https://baeef19e-8f9f-4b14-b95f-45946d6fe1e6.us-east4-0.gcp.cloud.qdrant.io:6333
 QDRANT_API_KEY=k0V8kKNulQdRLukhYy03kJcncctoDImbiPHgmvaEEsup8MwTjqgT0w
-COHERE_API_KEY=lCu3rZEjcUPAt0RsdQpQlGtgYp1uKAmuNIBdjFKq
+COHERE_API_KEY=lCu3rZEjcUPAt0RsdQpQlGtgYp1uKAmuNIBdjFKq
+NLTK_DATA=/app/nltk_data