Update functions.py

functions.py

@@ -88,8 +88,119 @@ def create_retriever_from_chroma(vectorstore_path="./docs/chroma/", search_type=
         
     else:
         st.write("Vector store doesnt exist and will be created now")
-        loader = DirectoryLoader('./data/', glob="./*.txt", loader_cls=TextLoader)
-        docs = loader.load()
+        urls = [
+ 
+"https://github.com/zedr/clean-code-python",
+"https://tenthousandmeters.com/blog/python-behind-the-scenes-10-how-python-dictionaries-work/",
+"https://realpython.com/python-testing/",
+"https://docs.python-guide.org/writing/license/",
+    "https://blogs.nvidia.com/blog/what-is-a-transformer-model/",
+    "https://research.google/blog/transformer-a-novel-neural-network-architecture-for-language-understanding/",
+"https://realpython.com/python-pep8/",
+"https://towardsdatascience.com/ideal-python-environment-setup-for-data-science-cdb03a447de8",
+"https://realpython.com/python3-object-oriented-programming/",
+"https://realpython.com/python-functional-programming/",
+"https://fivethirtyeight.com/features/science-isnt-broken/",
+"https://github.com/renatofillinich/ab_test_guide_in_python/blob/master/AB%20testing%20with%20Python.ipynb",
+"https://towardsdatascience.com/why-is-data-science-failing-to-solve-the-right-problems-7b5b6121e3b4",
+"https://medium.com/@srowen/common-probability-distributions-347e6b945ce4",
+"https://github.com/renatofillinich/ab_test_guide_in_python/blob/master/AB%20testing%20with%20Python.ipynb",
+"https://scikit-learn.org/stable/modules/compose.html",
+"https://machinelearningmastery.com/light-gradient-boosted-machine-lightgbm-ensemble/",
+"https://neptune.ai/blog/xgboost-vs-lightgbm",
+"https://towardsdatascience.com/interpretable-machine-learning-with-xgboost-9ec80d148d27",
+"https://www.cio.com/article/247005/what-are-containers-and-why-do-you-need-them.html",
+"https://mitsloan.mit.edu/ideas-made-to-matter/machine-learning-explained",
+"https://towardsdatascience.com/making-friends-with-machine-learning-5e28d5205a29",
+"https://towardsdatascience.com/handling-imbalanced-datasets-in-machine-learning-7a0e84220f28",
+"https://machinelearningmastery.com/multi-class-imbalanced-classification/",
+"https://imbalanced-learn.org/stable/auto_examples/applications/plot_impact_imbalanced_classes.html",
+"https://docs.ray.io/en/master/tune/examples/tune-sklearn.html",
+"https://www.kaggle.com/code/ldfreeman3/a-data-science-framework-to-achieve-99-accuracy",
+"https://cs231n.github.io/optimization-2/",
+"https://alexander-schiendorfer.github.io/2020/02/24/a-worked-example-of-backprop.html",
+"https://www.analyticsvidhya.com/blog/2020/01/fundamentals-deep-learning-activation-functions-when-to-use-them/",
+"https://ml-cheatsheet.readthedocs.io/en/latest/activation_functions.html",
+"https://d2l.ai/chapter_multilayer-perceptrons/mlp.html",
+"https://d2l.ai/chapter_linear-classification/softmax-regression.html#loss-function",
+"https://d2l.ai/chapter_optimization/",
+    "https://www.investopedia.com/terms/s/statistical-significance.asp",
+"https://d2l.ai/chapter_linear-classification/softmax-regression.html#loss-function",
+"https://d2l.ai/chapter_convolutional-neural-networks/why-conv.html",
+"https://d2l.ai/chapter_convolutional-modern/alexnet.html",
+"https://d2l.ai/chapter_convolutional-modern/vgg.html",
+"https://d2l.ai/chapter_convolutional-modern/nin.html",
+"https://d2l.ai/chapter_convolutional-modern/googlenet.html",
+    'https://python.langchain.com/v0.1/docs/guides/productionization/evaluation/',
+    'https://python.langchain.com/v0.1/docs/guides/productionization/evaluation/string/',
+    'https://python.langchain.com/v0.1/docs/guides/productionization/evaluation/comparison/',
+    'https://python.langchain.com/v0.1/docs/guides/productionization/evaluation/trajectory/',
+    "https://langchain-ai.github.io/langgraph/concepts/high_level/#why-langgraph",
+    'https://langchain-ai.github.io/langgraph/concepts/low_level/#only-stream-tokens-from-specific-nodesllms',
+    "https://langchain-ai.github.io/langgraph/concepts/agentic_concepts/#reflection",
+    "https://langchain-ai.github.io/langgraph/concepts/faq/",
+    "https://www.geeksforgeeks.org/python-oops-concepts/",
+    "https://www.mckinsey.com/featured-insights/mckinsey-explainers/what-is-fintech",
+    "https://datascientest.com/en/adversarial-attack-definition-and-protection-against-this-threat",
+    "https://datascientest.com/en/all-about-dspy",
+    "https://datascientest.com/en/arithmetic-and-data-science",
+    "https://datascientest.com/en/all-about-machine-learning-metrics",
+    "https://datascientest.com/en/all-about-procedural-programming",
+    "https://datascientest.com/en/all-about-cryptography",
+   "https://datascientest.com/en/all-about-predictive-coding",
+    "https://datascientest.com/en/all-about-network-convergence",
+    "https://datascientest.com/en/all-about-forensic-analysis",
+    "https://datascientest.com/en/all-about-chatgpt-jailbreak",
+    "https://datascientest.com/en/all-about-pentest",
+    "https://datascientest.com/en/all-about-embedded-systems",
+    "https://datascientest.com/en/all-about-network-operating-system",
+    "https://datascientest.com/en/all-about-ai-and-cybersecurity",
+    "https://datascientest.com/en/all-about-cybernetics",
+    "https://datascientest.com/en/all-about-seo",
+    "https://datascientest.com/en/all-about-expert-system",
+    "https://datascientest.com/en/all-about-telecommunications",
+    "https://datascientest.com/en/all-about-smart-cities",
+    "https://datascientest.com/en/all-about-artificial-intelligence-and-finance-sector",
+    "https://datascientest.com/en/all-about-generated-pre-trained-transformers",
+    "https://datascientest.com/en/all-about-iso-27001",
+    "https://datascientest.com/en/all-about-smart-sensors",
+    "https://datascientest.com/en/all-about-virtual-networks",
+    "https://datascientest.com/en/all-about-ethical-ai",
+    "https://datascientest.com/en/all-about-saio",
+    "https://datascientest.com/en/all-about-recommendation-algorithm",
+    "https://www.geeksforgeeks.org/activation-functions-neural-networks/",
+    "https://www.geeksforgeeks.org/activation-functions-in-neural-networks-set2/?ref=oin_asr1",
+    "https://www.geeksforgeeks.org/choosing-the-right-activation-function-for-your-neural-network/?ref=oin_asr3",
+    "https://www.geeksforgeeks.org/difference-between-feed-forward-neural-networks-and-recurrent-neural-networks/?ref=oin_asr2",
+    "https://www.geeksforgeeks.org/recurrent-neural-networks-explanation/?ref=oin_asr11",
+    "https://www.geeksforgeeks.org/deeppose-human-pose-estimation-via-deep-neural-networks/?ref=oin_asr13",
+    "https://www.geeksforgeeks.org/auto-associative-neural-networks/?ref=oin_asr18",
+    "https://www.geeksforgeeks.org/what-are-graph-neural-networks/?ref=oin_asr30",
+    "https://hdsr.mitpress.mit.edu/pub/la3vitqm/release/2",
+    "https://datasciencedojo.com/blog/a-guide-to-large-language-models/",
+    "https://datasciencedojo.com/blog/bootstrap-sampling/",
+    "https://datasciencedojo.com/blog/top-statistical-concepts/",
+    "https://datasciencedojo.com/blog/probability-for-data-science/",
+    "https://datasciencedojo.com/blog/top-statistical-techniques/",
+    "https://datasciencedojo.com/blog/statistical-distributions/",
+    "https://datasciencedojo.com/blog/data-science-in-finance/",
+    "https://datasciencedojo.com/blog/random-forest-algorithm/",
+    "https://datasciencedojo.com/blog/gini-index-and-entropy/",
+    "https://datasciencedojo.com/blog/boosting-algorithms-in-machine-learning/",
+    "https://datasciencedojo.com/blog/ensemble-methods-in-machine-learning/",
+    "https://datasciencedojo.com/blog/langgraph-tutorial/",
+    "https://datasciencedojo.com/blog/data-driven-marketing-in-2024/",
+    "https://datasciencedojo.com/blog/on-device-ai/",
+    
+    def extract_sentences_from_web(links, chunk_size=500, chunk_overlap=30):
+        data = []
+        for link in links:
+            loader = NewsURLLoader(urls=[link])
+            data += loader.load()
+        return data    
+
+
+    docs = extract_sentences_from_web(links=urls)
         
         
         text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(