add tool calls and implemented knowledge base

.env.example

@@ -1,4 +1,15 @@
 LOG_FILE=
 PINECONE_API_KEY=
 PINECONE_INDEX_NAME=
-GROQ_API_KEY=
+GROQ_API_KEY=
+RECIPIENT_EMAIL=
+SENDER_EMAIL=
+SMTP_HOST=
+SMTP_PORT=
+SMTP_USERNAME=
+SMTP_PASSWORD=
+AZURE_TENANT_ID=
+AZURE_CLIENT_ID=
+AZURE_CLIENT_SECRET=
+AZURE_SCOPE=https://graph.microsoft.com/.default
+AZURE_DRIVE_ID=

poetry.lock

@@ -827,6 +827,160 @@ win32-setctime = {version = ">=1.0.0", markers = "sys_platform == \"win32\""}
 [package.extras]
 dev = ["Sphinx (==7.2.5)", "colorama (==0.4.5)", "colorama (==0.4.6)", "exceptiongroup (==1.1.3)", "freezegun (==1.1.0)", "freezegun (==1.2.2)", "mypy (==v0.910)", "mypy (==v0.971)", "mypy (==v1.4.1)", "mypy (==v1.5.1)", "pre-commit (==3.4.0)", "pytest (==6.1.2)", "pytest (==7.4.0)", "pytest-cov (==2.12.1)", "pytest-cov (==4.1.0)", "pytest-mypy-plugins (==1.9.3)", "pytest-mypy-plugins (==3.0.0)", "sphinx-autobuild (==2021.3.14)", "sphinx-rtd-theme (==1.3.0)", "tox (==3.27.1)", "tox (==4.11.0)"]
 
+[[package]]
+name = "lxml"
+version = "5.3.0"
+description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API."
+optional = false
+python-versions = ">=3.6"
+files = [
+    {file = "lxml-5.3.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:dd36439be765e2dde7660212b5275641edbc813e7b24668831a5c8ac91180656"},
+    {file = "lxml-5.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ae5fe5c4b525aa82b8076c1a59d642c17b6e8739ecf852522c6321852178119d"},
+    {file = "lxml-5.3.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:501d0d7e26b4d261fca8132854d845e4988097611ba2531408ec91cf3fd9d20a"},
+    {file = "lxml-5.3.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb66442c2546446944437df74379e9cf9e9db353e61301d1a0e26482f43f0dd8"},
+    {file = "lxml-5.3.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9e41506fec7a7f9405b14aa2d5c8abbb4dbbd09d88f9496958b6d00cb4d45330"},
+    {file = "lxml-5.3.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f7d4a670107d75dfe5ad080bed6c341d18c4442f9378c9f58e5851e86eb79965"},
+    {file = "lxml-5.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:41ce1f1e2c7755abfc7e759dc34d7d05fd221723ff822947132dc934d122fe22"},
+    {file = "lxml-5.3.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:44264ecae91b30e5633013fb66f6ddd05c006d3e0e884f75ce0b4755b3e3847b"},
+    {file = "lxml-5.3.0-cp310-cp310-manylinux_2_28_ppc64le.whl", hash = "sha256:3c174dc350d3ec52deb77f2faf05c439331d6ed5e702fc247ccb4e6b62d884b7"},
+    {file = "lxml-5.3.0-cp310-cp310-manylinux_2_28_s390x.whl", hash = "sha256:2dfab5fa6a28a0b60a20638dc48e6343c02ea9933e3279ccb132f555a62323d8"},
+    {file = "lxml-5.3.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:b1c8c20847b9f34e98080da785bb2336ea982e7f913eed5809e5a3c872900f32"},
+    {file = "lxml-5.3.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:2c86bf781b12ba417f64f3422cfc302523ac9cd1d8ae8c0f92a1c66e56ef2e86"},
+    {file = "lxml-5.3.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:c162b216070f280fa7da844531169be0baf9ccb17263cf5a8bf876fcd3117fa5"},
+    {file = "lxml-5.3.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:36aef61a1678cb778097b4a6eeae96a69875d51d1e8f4d4b491ab3cfb54b5a03"},
+    {file = "lxml-5.3.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f65e5120863c2b266dbcc927b306c5b78e502c71edf3295dfcb9501ec96e5fc7"},
+    {file = "lxml-5.3.0-cp310-cp310-win32.whl", hash = "sha256:ef0c1fe22171dd7c7c27147f2e9c3e86f8bdf473fed75f16b0c2e84a5030ce80"},
+    {file = "lxml-5.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:052d99051e77a4f3e8482c65014cf6372e61b0a6f4fe9edb98503bb5364cfee3"},
+    {file = "lxml-5.3.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:74bcb423462233bc5d6066e4e98b0264e7c1bed7541fff2f4e34fe6b21563c8b"},
+    {file = "lxml-5.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a3d819eb6f9b8677f57f9664265d0a10dd6551d227afb4af2b9cd7bdc2ccbf18"},
+    {file = "lxml-5.3.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5b8f5db71b28b8c404956ddf79575ea77aa8b1538e8b2ef9ec877945b3f46442"},
+    {file = "lxml-5.3.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c3406b63232fc7e9b8783ab0b765d7c59e7c59ff96759d8ef9632fca27c7ee4"},
+    {file = "lxml-5.3.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2ecdd78ab768f844c7a1d4a03595038c166b609f6395e25af9b0f3f26ae1230f"},
+    {file = "lxml-5.3.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:168f2dfcfdedf611eb285efac1516c8454c8c99caf271dccda8943576b67552e"},
+    {file = "lxml-5.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa617107a410245b8660028a7483b68e7914304a6d4882b5ff3d2d3eb5948d8c"},
+    {file = "lxml-5.3.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:69959bd3167b993e6e710b99051265654133a98f20cec1d9b493b931942e9c16"},
+    {file = "lxml-5.3.0-cp311-cp311-manylinux_2_28_ppc64le.whl", hash = "sha256:bd96517ef76c8654446fc3db9242d019a1bb5fe8b751ba414765d59f99210b79"},
+    {file = "lxml-5.3.0-cp311-cp311-manylinux_2_28_s390x.whl", hash = "sha256:ab6dd83b970dc97c2d10bc71aa925b84788c7c05de30241b9e96f9b6d9ea3080"},
+    {file = "lxml-5.3.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:eec1bb8cdbba2925bedc887bc0609a80e599c75b12d87ae42ac23fd199445654"},
+    {file = "lxml-5.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6a7095eeec6f89111d03dabfe5883a1fd54da319c94e0fb104ee8f23616b572d"},
+    {file = "lxml-5.3.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6f651ebd0b21ec65dfca93aa629610a0dbc13dbc13554f19b0113da2e61a4763"},
+    {file = "lxml-5.3.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:f422a209d2455c56849442ae42f25dbaaba1c6c3f501d58761c619c7836642ec"},
+    {file = "lxml-5.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:62f7fdb0d1ed2065451f086519865b4c90aa19aed51081979ecd05a21eb4d1be"},
+    {file = "lxml-5.3.0-cp311-cp311-win32.whl", hash = "sha256:c6379f35350b655fd817cd0d6cbeef7f265f3ae5fedb1caae2eb442bbeae9ab9"},
+    {file = "lxml-5.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:9c52100e2c2dbb0649b90467935c4b0de5528833c76a35ea1a2691ec9f1ee7a1"},
+    {file = "lxml-5.3.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:e99f5507401436fdcc85036a2e7dc2e28d962550afe1cbfc07c40e454256a859"},
+    {file = "lxml-5.3.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:384aacddf2e5813a36495233b64cb96b1949da72bef933918ba5c84e06af8f0e"},
+    {file = "lxml-5.3.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:874a216bf6afaf97c263b56371434e47e2c652d215788396f60477540298218f"},
+    {file = "lxml-5.3.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:65ab5685d56914b9a2a34d67dd5488b83213d680b0c5d10b47f81da5a16b0b0e"},
+    {file = "lxml-5.3.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aac0bbd3e8dd2d9c45ceb82249e8bdd3ac99131a32b4d35c8af3cc9db1657179"},
+    {file = "lxml-5.3.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b369d3db3c22ed14c75ccd5af429086f166a19627e84a8fdade3f8f31426e52a"},
+    {file = "lxml-5.3.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c24037349665434f375645fa9d1f5304800cec574d0310f618490c871fd902b3"},
+    {file = "lxml-5.3.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:62d172f358f33a26d6b41b28c170c63886742f5b6772a42b59b4f0fa10526cb1"},
+    {file = "lxml-5.3.0-cp312-cp312-manylinux_2_28_ppc64le.whl", hash = "sha256:c1f794c02903c2824fccce5b20c339a1a14b114e83b306ff11b597c5f71a1c8d"},
+    {file = "lxml-5.3.0-cp312-cp312-manylinux_2_28_s390x.whl", hash = "sha256:5d6a6972b93c426ace71e0be9a6f4b2cfae9b1baed2eed2006076a746692288c"},
+    {file = "lxml-5.3.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:3879cc6ce938ff4eb4900d901ed63555c778731a96365e53fadb36437a131a99"},
+    {file = "lxml-5.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:74068c601baff6ff021c70f0935b0c7bc528baa8ea210c202e03757c68c5a4ff"},
+    {file = "lxml-5.3.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:ecd4ad8453ac17bc7ba3868371bffb46f628161ad0eefbd0a855d2c8c32dd81a"},
+    {file = "lxml-5.3.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:7e2f58095acc211eb9d8b5771bf04df9ff37d6b87618d1cbf85f92399c98dae8"},
+    {file = "lxml-5.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e63601ad5cd8f860aa99d109889b5ac34de571c7ee902d6812d5d9ddcc77fa7d"},
+    {file = "lxml-5.3.0-cp312-cp312-win32.whl", hash = "sha256:17e8d968d04a37c50ad9c456a286b525d78c4a1c15dd53aa46c1d8e06bf6fa30"},
+    {file = "lxml-5.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:c1a69e58a6bb2de65902051d57fde951febad631a20a64572677a1052690482f"},
+    {file = "lxml-5.3.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8c72e9563347c7395910de6a3100a4840a75a6f60e05af5e58566868d5eb2d6a"},
+    {file = "lxml-5.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e92ce66cd919d18d14b3856906a61d3f6b6a8500e0794142338da644260595cd"},
+    {file = "lxml-5.3.0-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1d04f064bebdfef9240478f7a779e8c5dc32b8b7b0b2fc6a62e39b928d428e51"},
+    {file = "lxml-5.3.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c2fb570d7823c2bbaf8b419ba6e5662137f8166e364a8b2b91051a1fb40ab8b"},
+    {file = "lxml-5.3.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0c120f43553ec759f8de1fee2f4794452b0946773299d44c36bfe18e83caf002"},
+    {file = "lxml-5.3.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:562e7494778a69086f0312ec9689f6b6ac1c6b65670ed7d0267e49f57ffa08c4"},
+    {file = "lxml-5.3.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:423b121f7e6fa514ba0c7918e56955a1d4470ed35faa03e3d9f0e3baa4c7e492"},
+    {file = "lxml-5.3.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:c00f323cc00576df6165cc9d21a4c21285fa6b9989c5c39830c3903dc4303ef3"},
+    {file = "lxml-5.3.0-cp313-cp313-manylinux_2_28_ppc64le.whl", hash = "sha256:1fdc9fae8dd4c763e8a31e7630afef517eab9f5d5d31a278df087f307bf601f4"},
+    {file = "lxml-5.3.0-cp313-cp313-manylinux_2_28_s390x.whl", hash = "sha256:658f2aa69d31e09699705949b5fc4719cbecbd4a97f9656a232e7d6c7be1a367"},
+    {file = "lxml-5.3.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:1473427aff3d66a3fa2199004c3e601e6c4500ab86696edffdbc84954c72d832"},
+    {file = "lxml-5.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a87de7dd873bf9a792bf1e58b1c3887b9264036629a5bf2d2e6579fe8e73edff"},
+    {file = "lxml-5.3.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:0d7b36afa46c97875303a94e8f3ad932bf78bace9e18e603f2085b652422edcd"},
+    {file = "lxml-5.3.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:cf120cce539453ae086eacc0130a324e7026113510efa83ab42ef3fcfccac7fb"},
+    {file = "lxml-5.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:df5c7333167b9674aa8ae1d4008fa4bc17a313cc490b2cca27838bbdcc6bb15b"},
+    {file = "lxml-5.3.0-cp313-cp313-win32.whl", hash = "sha256:c802e1c2ed9f0c06a65bc4ed0189d000ada8049312cfeab6ca635e39c9608957"},
+    {file = "lxml-5.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:406246b96d552e0503e17a1006fd27edac678b3fcc9f1be71a2f94b4ff61528d"},
+    {file = "lxml-5.3.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:8f0de2d390af441fe8b2c12626d103540b5d850d585b18fcada58d972b74a74e"},
+    {file = "lxml-5.3.0-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1afe0a8c353746e610bd9031a630a95bcfb1a720684c3f2b36c4710a0a96528f"},
+    {file = "lxml-5.3.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:56b9861a71575f5795bde89256e7467ece3d339c9b43141dbdd54544566b3b94"},
+    {file = "lxml-5.3.0-cp36-cp36m-manylinux_2_28_x86_64.whl", hash = "sha256:9fb81d2824dff4f2e297a276297e9031f46d2682cafc484f49de182aa5e5df99"},
+    {file = "lxml-5.3.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:2c226a06ecb8cdef28845ae976da407917542c5e6e75dcac7cc33eb04aaeb237"},
+    {file = "lxml-5.3.0-cp36-cp36m-musllinux_1_2_x86_64.whl", hash = "sha256:7d3d1ca42870cdb6d0d29939630dbe48fa511c203724820fc0fd507b2fb46577"},
+    {file = "lxml-5.3.0-cp36-cp36m-win32.whl", hash = "sha256:094cb601ba9f55296774c2d57ad68730daa0b13dc260e1f941b4d13678239e70"},
+    {file = "lxml-5.3.0-cp36-cp36m-win_amd64.whl", hash = "sha256:eafa2c8658f4e560b098fe9fc54539f86528651f61849b22111a9b107d18910c"},
+    {file = "lxml-5.3.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:cb83f8a875b3d9b458cada4f880fa498646874ba4011dc974e071a0a84a1b033"},
+    {file = "lxml-5.3.0-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:25f1b69d41656b05885aa185f5fdf822cb01a586d1b32739633679699f220391"},
+    {file = "lxml-5.3.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23e0553b8055600b3bf4a00b255ec5c92e1e4aebf8c2c09334f8368e8bd174d6"},
+    {file = "lxml-5.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ada35dd21dc6c039259596b358caab6b13f4db4d4a7f8665764d616daf9cc1d"},
+    {file = "lxml-5.3.0-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:81b4e48da4c69313192d8c8d4311e5d818b8be1afe68ee20f6385d0e96fc9512"},
+    {file = "lxml-5.3.0-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:2bc9fd5ca4729af796f9f59cd8ff160fe06a474da40aca03fcc79655ddee1a8b"},
+    {file = "lxml-5.3.0-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:07da23d7ee08577760f0a71d67a861019103e4812c87e2fab26b039054594cc5"},
+    {file = "lxml-5.3.0-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:ea2e2f6f801696ad7de8aec061044d6c8c0dd4037608c7cab38a9a4d316bfb11"},
+    {file = "lxml-5.3.0-cp37-cp37m-win32.whl", hash = "sha256:5c54afdcbb0182d06836cc3d1be921e540be3ebdf8b8a51ee3ef987537455f84"},
+    {file = "lxml-5.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:f2901429da1e645ce548bf9171784c0f74f0718c3f6150ce166be39e4dd66c3e"},
+    {file = "lxml-5.3.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c56a1d43b2f9ee4786e4658c7903f05da35b923fb53c11025712562d5cc02753"},
+    {file = "lxml-5.3.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ee8c39582d2652dcd516d1b879451500f8db3fe3607ce45d7c5957ab2596040"},
+    {file = "lxml-5.3.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0fdf3a3059611f7585a78ee10399a15566356116a4288380921a4b598d807a22"},
+    {file = "lxml-5.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:146173654d79eb1fc97498b4280c1d3e1e5d58c398fa530905c9ea50ea849b22"},
+    {file = "lxml-5.3.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:0a7056921edbdd7560746f4221dca89bb7a3fe457d3d74267995253f46343f15"},
+    {file = "lxml-5.3.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:9e4b47ac0f5e749cfc618efdf4726269441014ae1d5583e047b452a32e221920"},
+    {file = "lxml-5.3.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:f914c03e6a31deb632e2daa881fe198461f4d06e57ac3d0e05bbcab8eae01945"},
+    {file = "lxml-5.3.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:213261f168c5e1d9b7535a67e68b1f59f92398dd17a56d934550837143f79c42"},
+    {file = "lxml-5.3.0-cp38-cp38-win32.whl", hash = "sha256:218c1b2e17a710e363855594230f44060e2025b05c80d1f0661258142b2add2e"},
+    {file = "lxml-5.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:315f9542011b2c4e1d280e4a20ddcca1761993dda3afc7a73b01235f8641e903"},
+    {file = "lxml-5.3.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:1ffc23010330c2ab67fac02781df60998ca8fe759e8efde6f8b756a20599c5de"},
+    {file = "lxml-5.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2b3778cb38212f52fac9fe913017deea2fdf4eb1a4f8e4cfc6b009a13a6d3fcc"},
+    {file = "lxml-5.3.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b0c7a688944891086ba192e21c5229dea54382f4836a209ff8d0a660fac06be"},
+    {file = "lxml-5.3.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:747a3d3e98e24597981ca0be0fd922aebd471fa99d0043a3842d00cdcad7ad6a"},
+    {file = "lxml-5.3.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:86a6b24b19eaebc448dc56b87c4865527855145d851f9fc3891673ff97950540"},
+    {file = "lxml-5.3.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b11a5d918a6216e521c715b02749240fb07ae5a1fefd4b7bf12f833bc8b4fe70"},
+    {file = "lxml-5.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:68b87753c784d6acb8a25b05cb526c3406913c9d988d51f80adecc2b0775d6aa"},
+    {file = "lxml-5.3.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:109fa6fede314cc50eed29e6e56c540075e63d922455346f11e4d7a036d2b8cf"},
+    {file = "lxml-5.3.0-cp39-cp39-manylinux_2_28_ppc64le.whl", hash = "sha256:02ced472497b8362c8e902ade23e3300479f4f43e45f4105c85ef43b8db85229"},
+    {file = "lxml-5.3.0-cp39-cp39-manylinux_2_28_s390x.whl", hash = "sha256:6b038cc86b285e4f9fea2ba5ee76e89f21ed1ea898e287dc277a25884f3a7dfe"},
+    {file = "lxml-5.3.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:7437237c6a66b7ca341e868cda48be24b8701862757426852c9b3186de1da8a2"},
+    {file = "lxml-5.3.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:7f41026c1d64043a36fda21d64c5026762d53a77043e73e94b71f0521939cc71"},
+    {file = "lxml-5.3.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:482c2f67761868f0108b1743098640fbb2a28a8e15bf3f47ada9fa59d9fe08c3"},
+    {file = "lxml-5.3.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:1483fd3358963cc5c1c9b122c80606a3a79ee0875bcac0204149fa09d6ff2727"},
+    {file = "lxml-5.3.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:2dec2d1130a9cda5b904696cec33b2cfb451304ba9081eeda7f90f724097300a"},
+    {file = "lxml-5.3.0-cp39-cp39-win32.whl", hash = "sha256:a0eabd0a81625049c5df745209dc7fcef6e2aea7793e5f003ba363610aa0a3ff"},
+    {file = "lxml-5.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:89e043f1d9d341c52bf2af6d02e6adde62e0a46e6755d5eb60dc6e4f0b8aeca2"},
+    {file = "lxml-5.3.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7b1cd427cb0d5f7393c31b7496419da594fe600e6fdc4b105a54f82405e6626c"},
+    {file = "lxml-5.3.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:51806cfe0279e06ed8500ce19479d757db42a30fd509940b1701be9c86a5ff9a"},
+    {file = "lxml-5.3.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ee70d08fd60c9565ba8190f41a46a54096afa0eeb8f76bd66f2c25d3b1b83005"},
+    {file = "lxml-5.3.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:8dc2c0395bea8254d8daebc76dcf8eb3a95ec2a46fa6fae5eaccee366bfe02ce"},
+    {file = "lxml-5.3.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:6ba0d3dcac281aad8a0e5b14c7ed6f9fa89c8612b47939fc94f80b16e2e9bc83"},
+    {file = "lxml-5.3.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:6e91cf736959057f7aac7adfc83481e03615a8e8dd5758aa1d95ea69e8931dba"},
+    {file = "lxml-5.3.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:94d6c3782907b5e40e21cadf94b13b0842ac421192f26b84c45f13f3c9d5dc27"},
+    {file = "lxml-5.3.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c300306673aa0f3ed5ed9372b21867690a17dba38c68c44b287437c362ce486b"},
+    {file = "lxml-5.3.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78d9b952e07aed35fe2e1a7ad26e929595412db48535921c5013edc8aa4a35ce"},
+    {file = "lxml-5.3.0-pp37-pypy37_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:01220dca0d066d1349bd6a1726856a78f7929f3878f7e2ee83c296c69495309e"},
+    {file = "lxml-5.3.0-pp37-pypy37_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:2d9b8d9177afaef80c53c0a9e30fa252ff3036fb1c6494d427c066a4ce6a282f"},
+    {file = "lxml-5.3.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:20094fc3f21ea0a8669dc4c61ed7fa8263bd37d97d93b90f28fc613371e7a875"},
+    {file = "lxml-5.3.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ace2c2326a319a0bb8a8b0e5b570c764962e95818de9f259ce814ee666603f19"},
+    {file = "lxml-5.3.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:92e67a0be1639c251d21e35fe74df6bcc40cba445c2cda7c4a967656733249e2"},
+    {file = "lxml-5.3.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd5350b55f9fecddc51385463a4f67a5da829bc741e38cf689f38ec9023f54ab"},
+    {file = "lxml-5.3.0-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:4c1fefd7e3d00921c44dc9ca80a775af49698bbfd92ea84498e56acffd4c5469"},
+    {file = "lxml-5.3.0-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:71a8dd38fbd2f2319136d4ae855a7078c69c9a38ae06e0c17c73fd70fc6caad8"},
+    {file = "lxml-5.3.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:97acf1e1fd66ab53dacd2c35b319d7e548380c2e9e8c54525c6e76d21b1ae3b1"},
+    {file = "lxml-5.3.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:68934b242c51eb02907c5b81d138cb977b2129a0a75a8f8b60b01cb8586c7b21"},
+    {file = "lxml-5.3.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b710bc2b8292966b23a6a0121f7a6c51d45d2347edcc75f016ac123b8054d3f2"},
+    {file = "lxml-5.3.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:18feb4b93302091b1541221196a2155aa296c363fd233814fa11e181adebc52f"},
+    {file = "lxml-5.3.0-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:3eb44520c4724c2e1a57c0af33a379eee41792595023f367ba3952a2d96c2aab"},
+    {file = "lxml-5.3.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:609251a0ca4770e5a8768ff902aa02bf636339c5a93f9349b48eb1f606f7f3e9"},
+    {file = "lxml-5.3.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:516f491c834eb320d6c843156440fe7fc0d50b33e44387fcec5b02f0bc118a4c"},
+    {file = "lxml-5.3.0.tar.gz", hash = "sha256:4e109ca30d1edec1ac60cdbe341905dc3b8f55b16855e03a54aaf59e51ec8c6f"},
+]
+
+[package.extras]
+cssselect = ["cssselect (>=0.7)"]
+html-clean = ["lxml-html-clean"]
+html5 = ["html5lib"]
+htmlsoup = ["BeautifulSoup4"]
+source = ["Cython (>=3.0.11)"]
+
 [[package]]
 name = "lz4"
 version = "4.3.3"
@@ -1233,6 +1387,37 @@ files = [
 [package.dependencies]
 typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0"
 
+[[package]]
+name = "pymupdf"
+version = "1.24.13"
+description = "A high performance Python library for data extraction, analysis, conversion & manipulation of PDF (and other) documents."
+optional = false
+python-versions = ">=3.9"
+files = [
+    {file = "PyMuPDF-1.24.13-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:c11bb9198af69d490b4b346421db827d875a28fbc760d239e691d4b3ed12b5ad"},
+    {file = "PyMuPDF-1.24.13-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:240d5c43daa9278db50d609162b48f673ab256d7e5c73eea67af517c1fc2d47c"},
+    {file = "PyMuPDF-1.24.13-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e4c8808e62afbbde0f7b9c4151c4b1a5735911c2d39c34332860df600dba76f8"},
+    {file = "PyMuPDF-1.24.13-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c830610e4fde237fcf0532f1f8c1381453f48c164a5eadd0c6e5fd0bea1ca8e3"},
+    {file = "PyMuPDF-1.24.13-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:4520558580ac6b5a7164fda29fbc14e39d3114fd803420721500edbf47d04872"},
+    {file = "PyMuPDF-1.24.13-cp39-abi3-win32.whl", hash = "sha256:ab22828d4fc205791ef1332a64893cbfc38cd9c331c5f46ae4537372ffee6fc1"},
+    {file = "PyMuPDF-1.24.13-cp39-abi3-win_amd64.whl", hash = "sha256:ec17914e4a560f4070212a2e84db5cc8b561d85d1ead193605a22f9561b03148"},
+    {file = "PyMuPDF-1.24.13.tar.gz", hash = "sha256:6ec3ab3c6d5cba60bfcf58daaa2d1a5b700b0366ce52be666445007351461fa4"},
+]
+
+[[package]]
+name = "pymupdf4llm"
+version = "0.0.17"
+description = "PyMuPDF Utilities for LLM/RAG"
+optional = false
+python-versions = "*"
+files = [
+    {file = "pymupdf4llm-0.0.17-py3-none-any.whl", hash = "sha256:26de9996945f15e3ca507908f80dc18a959f5b5214bb2e302c7f7034089665a0"},
+    {file = "pymupdf4llm-0.0.17.tar.gz", hash = "sha256:27287ef9fe0217cf37841a3ef2bcf70da2553c43d95ea39b664a6de6485678c3"},
+]
+
+[package.dependencies]
+pymupdf = ">=1.24.10"
+
 [[package]]
 name = "pysocks"
 version = "1.7.1"
@@ -1245,6 +1430,21 @@ files = [
     {file = "PySocks-1.7.1.tar.gz", hash = "sha256:3f8804571ebe159c380ac6de37643bb4685970655d3bba243530d6558b799aa0"},
 ]
 
+[[package]]
+name = "python-docx"
+version = "1.1.2"
+description = "Create, read, and update Microsoft Word .docx files."
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "python_docx-1.1.2-py3-none-any.whl", hash = "sha256:08c20d6058916fb19853fcf080f7f42b6270d89eac9fa5f8c15f691c0017fabe"},
+    {file = "python_docx-1.1.2.tar.gz", hash = "sha256:0cf1f22e95b9002addca7948e16f2cd7acdfd498047f1941ca5d293db7762efd"},
+]
+
+[package.dependencies]
+lxml = ">=3.1.0"
+typing-extensions = ">=4.9.0"
+
 [[package]]
 name = "python-dotenv"
 version = "1.0.1"
@@ -1259,6 +1459,17 @@ files = [
 [package.extras]
 cli = ["click (>=5.0)"]
 
+[[package]]
+name = "python-multipart"
+version = "0.0.17"
+description = "A streaming multipart parser for Python"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "python_multipart-0.0.17-py3-none-any.whl", hash = "sha256:15dc4f487e0a9476cc1201261188ee0940165cffc94429b6fc565c4d3045cb5d"},
+    {file = "python_multipart-0.0.17.tar.gz", hash = "sha256:41330d831cae6e2f22902704ead2826ea038d0419530eadff3ea80175aec5538"},
+]
+
 [[package]]
 name = "pyyaml"
 version = "6.0.2"
@@ -2040,4 +2251,4 @@ h11 = ">=0.9.0,<1"
 [metadata]
 lock-version = "2.0"
 python-versions = "3.11.*"
-content-hash = "e1ec6f9e7d685af47fa9850b8e5c7ce55242d4ca63b87ff417ddb09815290855"
+content-hash = "03a23ed4d0df7d827033ee798432d2a443968853c9002926bfde5556e0d5455c"

pyproject.toml

@@ -22,6 +22,10 @@ selenium = "^4.25.0"
 webdriver-manager = "^4.0.2"
 bs4 = "^0.0.2"
 tiktoken = "^0.8.0"
+python-multipart = "^0.0.17"
+pymupdf = "^1.24.13"
+pymupdf4llm = "^0.0.17"
+python-docx = "^1.1.2"
 
 
 [build-system]

src/app.py

@@ -5,7 +5,7 @@ from fastapi.middleware.cors import CORSMiddleware
 
 from src.utils import logger
 
-from src.controllers import ws_router, api_router
+from src.controllers import ws_router, api_router, file_router
 
 
 @asynccontextmanager
@@ -38,4 +38,5 @@ async def check_health():
     return {"response": "Service is healthy!"}
 
 app.include_router(ws_router)
-app.include_router(api_router, prefix="/api/v1")
+app.include_router(api_router, prefix="/api/v1")
+app.include_router(file_router, prefix="/api/v1")

src/controllers/__init__.py

@@ -1,12 +1,15 @@
 from ._chat_controller import ChatController
 from ._database_controller import DatabaseController
+from ._file_controller import FileController
 
 ws_router = ChatController().router
 api_router = DatabaseController().router
+file_router = FileController().router
 
 __all__ = [
     "ws_router",
     "api_router",
+    "file_router",
 ]
 __version__ = "0.1.0"
 __author__ = "Aryan Jain"

src/controllers/_database_controller.py

@@ -1,4 +1,5 @@
 from fastapi import APIRouter, HTTPException
+from fastapi.responses import JSONResponse
 
 from src.utils import logger
 
@@ -10,10 +11,14 @@ class DatabaseController:
         self.router = APIRouter(prefix="/update_database", tags=["database"])
         self.router.add_api_route("/", self.update_database, methods=["POST"])
 
-    async def update_database(self, url: str = "https://sifars.com/"):
+    async def update_database(self, urls: list[str] = ["https://sifars.com/"], knowledge_base: bool = False):
         try:
             async with self.database_service() as database_service:
-                await database_service._update_database(url)
+                response = await database_service._update_database(urls=urls, knowledge_base=knowledge_base)
+            return JSONResponse({
+                "status": "success",
+                "data": {"message": response}
+            })
         except Exception as e:
             logger.error(e)
             raise HTTPException(status_code=500, detail=str(e))

src/controllers/_file_controller.py

@@ -0,0 +1,33 @@
+import os
+from fastapi import APIRouter, HTTPException, UploadFile
+from fastapi.responses import JSONResponse
+
+from src.utils import logger
+
+from src.services import FileService
+
+import aiofiles
+
+class FileController:
+    def __init__(self):
+        self.file_service = FileService
+        self.router = APIRouter(prefix="/upload_file", tags=["file_upload"])
+        self.router.add_api_route("/", self.upload_file, methods=["POST"])
+
+    async def upload_file(self, file: UploadFile):
+        try:
+            async with aiofiles.tempfile.TemporaryDirectory() as temp_dir:
+                temp_file_path = os.path.join(temp_dir, file.filename)
+                async with aiofiles.open(temp_file_path, "wb") as f:
+                    await f.write(file.file.read())
+                async with self.file_service() as file_service:
+                    response = await file_service.upload_file(temp_file_path)
+            return JSONResponse(
+                {
+                    "status": "success",
+                    "data": {"web_url": response},
+                }
+            )
+        except Exception as e:
+            logger.error(e)
+            raise HTTPException(status_code=500, detail=str(e))

src/crawler/_database_updater.py

@@ -1,13 +1,19 @@
+import os
 import aiofiles
-from src.utils import PineconeClient
+import httpx
+from src.utils import PineconeClient, SharepointClient
 from src.crawler import WebCrawler
 import tiktoken
+import pymupdf4llm
+from docx import Document
+from src.utils import logger
 
 class DatabaseUpdater:
     def __init__(self):
         self.model_name = 'gpt-3.5-turbo'
         self.pinecone_client = PineconeClient
         self.web_crawler = WebCrawler
+        self.sharepoint_client = SharepointClient
 
     async def __aenter__(self):
         self.tokenizer = tiktoken.encoding_for_model(self.model_name)
@@ -38,11 +44,55 @@ class DatabaseUpdater:
             await pinecone_client._delete_index()
         return
     
-    async def _extract_new_data(self, url):
+    async def process_text_file(self, file_path):
+        async with aiofiles.open(file_path, 'r', encoding='utf-8') as f:
+            content = await f.read()
+        async with aiofiles.open("knowledge_base.txt", 'a+', encoding='utf-8') as f:
+            await f.write(content)
+            await f.write("\n")
+        return
+    
+    async def process_pdf_file(self, file_path):
+        docs = pymupdf4llm.to_markdown(file_path, page_chunks=True)
+        async with aiofiles.open("knowledge_base.txt", 'a+', encoding='utf-8') as f:
+            for doc in docs:
+                await f.write(doc.get("text"))
+                await f.write("\n")
+        return
+    
+    async def process_docx_file(self, file_path):
+        doc = Document(file_path)
+        async with aiofiles.open("knowledge_base.txt", 'a+', encoding='utf-8') as f:
+            for paragraph in doc.paragraphs:
+                await f.write(paragraph.text)
+                await f.write("\n")
+    
+    async def _extract_scraped_data(self, url):
         async with self.web_crawler(url) as crawler:
             await crawler.crawl(url)
         return
     
+    async def _extract_knowledge_base(self):
+        async with self.sharepoint_client() as sharepoint_client:
+            files = await sharepoint_client.get_files()
+        async with aiofiles.tempfile.TemporaryDirectory() as temp_dir:
+            for file in files["value"]:
+                file_path = os.path.join(temp_dir, file['name'])
+                async with httpx.AsyncClient() as client:
+                    response = await client.get(file['@microsoft.graph.downloadUrl'])
+                    async with aiofiles.open(file_path, 'wb') as f:
+                        await f.write(response.content)
+                if file["name"].endswith(".txt") or file["name"].endswith(".md"):
+                    await self.process_text_file(file_path)
+                elif file["name"].endswith(".pdf"):
+                    await self.process_pdf_file(file_path)
+                elif file["name"].endswith("docx"):
+                    await self.process_docx_file(file_path)
+                else:
+                    logger.error(f"Unsupported file format: {file['name']}")
+        return 
+
+    
     async def _update_database_from_file(self, file_paths):
         for file_path in file_paths:
             async with aiofiles.open(file_path, 'r', encoding='utf-8') as f:
@@ -51,7 +101,8 @@ class DatabaseUpdater:
             await self._update_database(sentences)
         return
     
-    async def _clear_old_scrapped_file(self):
-        async with aiofiles.open("scrapped.txt", 'w', encoding='utf-8') as f:
-            await f.write("")
+    async def _clear_old_files(self, file_paths):
+        for file_path in file_paths:
+            async with aiofiles.open(file_path, 'w', encoding='utf-8') as f:
+                await f.write("")
         return

src/services/__init__.py

@@ -1,11 +1,13 @@
 from ._connection_service import ConnectionService
 from ._chat_service import ChatService
 from ._database_service import DatabaseService
+from ._file_service import FileService
 
 __all__ = [
     "ConnectionService",
     "ChatService",
-    "DatabaseService"
+    "DatabaseService",
+    "FileService",
 ]
 __version__ = "0.1.0"
 __author__ = "Aryan Jain"

src/services/_chat_service.py

@@ -17,11 +17,6 @@ class ChatService:
         pass
 
     async def create_response_message(self, messages: list):
-        async with self.pinecone_client() as pinecone_client:
-            context = []
-            async for text in pinecone_client._get_similar_texts(messages[-1]["content"], top_k=2):
-                context.append(text)
-            context = " ".join(context)
         async with self.chat_client() as chat_client:
-            async for response in chat_client.chat(messages[1:][-11:], context):
+            async for response in chat_client.create_chat_completions(messages[1:][-11:]):
                 yield response

src/services/_database_service.py

@@ -5,7 +5,7 @@ from src.utils import logger
 class DatabaseService:
     def __init__(self):
         self.database_updater = DatabaseUpdater
-        self.file_paths = ["scrapped.txt", "sifars.md"]
+        self.file_paths = ["knowledge_base.txt", "scrapped.txt", "sifars.md"]
 
     async def __aenter__(self):
         return self
@@ -13,9 +13,18 @@ class DatabaseService:
     async def __aexit__(self, exc_type, exc_val, exc_tb):
         pass
 
-    async def _update_database(self, url: str = ""):
-        async with self.database_updater() as database_updater:
-            await database_updater._clear_old_scrapped_file()
-            await database_updater._extract_new_data(url)
-            await database_updater._delete_old_database()
-            await database_updater._update_database_from_file(file_paths=self.file_paths)
+    async def _update_database(self, urls: list[str], knowledge_base: bool):
+        if not urls and not knowledge_base:
+            return "Nothing to update"
+        if urls:
+            async with self.database_updater() as database_updater:
+                await database_updater._clear_old_files(file_paths=self.file_paths[1:2])
+                for url in urls:
+                    await database_updater._extract_scraped_data(url=url)
+        if knowledge_base:
+            async with self.database_updater() as database_updater:
+                await database_updater._clear_old_files(file_paths=self.file_paths[:1])
+                await database_updater._extract_knowledge_base()
+        await database_updater._delete_old_database()
+        await database_updater._update_database_from_file(file_paths=self.file_paths)
+        return "Database updated"

src/services/_file_service.py

@@ -0,0 +1,17 @@
+from src.utils import logger, SharepointClient
+
+class FileService:
+    def __init__(self):
+        pass
+
+    async def __aenter__(self):
+        return self
+    
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        pass
+
+    async def upload_file(self, file_path: str):
+        async with SharepointClient() as sharepoint_client:
+            web_url = await sharepoint_client.upload_file(file_path)
+        logger.info(f"File uploaded to Sharepoint. Web URL: {web_url}")
+        return web_url

src/utils/__init__.py

@@ -1,11 +1,17 @@
 from ._config import logger
 from ._chat_client import ChatClient
 from ._pinecone_client import PineconeClient
+from ._email_client import EmailClient
+from ._tool_call import ToolCall
+from ._sharepoint_client import SharepointClient
 
 __all__ = [
     "logger",
     "ChatClient",
     "PineconeClient",
+    "EmailClient",
+    "ToolCall",
+    "SharepointClient"
 ]
 __version__ = "0.1.0"
 __author__ = "Aryan Jain"

src/utils/_chat_client.py

@@ -1,19 +1,37 @@
 import os
 from groq import AsyncGroq
 
+import json
+
+from ._tool_call import ToolCall
+
+from ._config import logger
+
+
+AVAILABLE_FUNCTIONS = {
+    "contact_us": ToolCall.send_email.__name__,
+    "get_context_for_user_query": ToolCall.get_context_for_user_query.__name__
+}
+
 INSTRUCTIONS="""
+# Objective
 You are an AI assistant designed by Sifars, a web development company, who gives answers to queries regarding Sifars, to the best of your ability.
-You will be provided a context along with each user query which you will use to answer the queries.
-Match the question with the context and according to that answer the user query.
+You can get additional context to answer the user query by calling the tool 'get_context_for_user_query', which you can then use to answer their query. Be smart about when to use the tool to get the context for the query.
 
 # Here is a bit of information about Sifars:
-Sifars, a pioneering web service provider, emerged onto the tech landscape in 2018 with a vision to revolutionize the digital sphere. Founded by visionary entrepreneurs Jatin Sethi, Munish Kumar, and Sukhwinder Singh, Sifars set its sights on empowering businesses worldwide with cutting-edge technology solutions. With its global headquarters nestled in the vibrant city of Patiala, Punjab, India, Sifars quickly garnered recognition as a leading application development company, committed to propelling businesses towards success in the ever-evolving tech landscape.
+Sifars, a pioneering web service provider, emerged onto the tech landscape in 2018 with a vision to revolutionize the digital sphere. Founded by visionary entrepreneurs Jatin Sethi, Munish Kumar, and Sukhwinder Singh, Sifars set its sights on empowering businesses worldwide with cutting-edge technology solutions. With its global headquarters nestled in the vibrant city of Patiala, Punjab, India, Sifars quickly garnered recognition as a leading application development company, committed to propelling businesses towards success in the ever-evolving tech landscape. The company leverages a diverse stack of technologies, including Python, JavaScript, React, Node.js, and AWS, ensuring robust and scalable solutions. Sifars also fosters a culture of innovation and work-life balance, enabling its team to thrive professionally and personally.
 
 Email: [contact@sifars.com](mailto:contact@sifars.com)
 Address: SCO 6, First Floor, Phulkian Enclave, Near Mini Secretariat, Patiala, Punjab 147001, India
+Phone: [+91 8106 455 950](tel:+918106455950), [+91 8008 296 463](tel:+918008296463), [+91 8896 720 000](tel:+918896720000)
+
+---
 
-<RULES>
+# Rules
+- If you need context about the query then use tool call 'get_context_for_user_query'. Do not ask from the user.
 - It is mandatory for you to keep your responses extremely brief and to the point. Avoid repeating the same information multiple times.
+- It is mandatory when calling the 'contact_us' tool, to not assume tool call parameters by yourself. You must ask the user to drop the neccessary details in the chat and use that information. You must use the reason for contact that the user provides to dynamically construct the subject of the email. It is mandatory for you do not mention the subject of the email and how you are creating it in the response. So first ask tool call parameters and then call the tool. Do not pass empty values in the tool call.
+- Do not assume any parameter of 'contact_us' tool by yourself. All parameters must be provided by user. When user wants to connect us for any query then ask the required parameters and then call the tool "contact us". If required variables are not there then collect required parameters from user and call the tool "contact us". Only after asking all the required parameteres then call the tool "contact us".
 - It is mandatory to use only one or two sentences in response. These sentences should be short and concise, and should use simple and non-repititive words.
 - It is mandatory to split the responses into extremely short paragraphs when using more than two short sentences to increase readability and engagement. 
 - All links should be properly formatted in markdown. When listing items in your response use markdown bullet points instead to ensure readability.
@@ -24,17 +42,63 @@ Address: SCO 6, First Floor, Phulkian Enclave, Near Mini Secretariat, Patiala, P
 - It is mandatory for you to not mention the context provided, or the contents of the context, for any reason in your message.
 - Avoid including extra information not asked for in the user query. 
 - Use plural first person pronouns when talking about sifars.
-- Try to answer the query from the context provided. Even if the context does not directly answer the question, try to relate it to the question and formulate an answer.
-- If someone asks to contact us, give them our contact information and link of contact page [here](https://www.sifars.com/en/contact/).
-- If someone asks about sifars, give them only the necessary details and then ask them to visit our about page [here](https://www.sifars.com/en/about/).
-- If someone asks about the services we provide, give them a bit of idea according to the interest of their query and then direct them to our services page [here](https://www.sifars.com/en/services/).
-- If someone asks about our projects or portfolio, provide them details of any three projects with minimal overview of those projects, according to the interest of their query and then direct them to our portfolio page [here](https://www.sifars.com/en/portfolio/).
-- If someone asks about our technology or tech stack that we use, give them a bit of idea according to the interest of their query and then direct them to our technology page [here](https://www.sifars.com/en/technology/).
-- If someone asks about career opportunities, direct them to our career page [here](https://www.sifars.com/en/career/).
-- It is mandatory for you to not make up any links on your own. Only use the links provided above.
-</RULES>
+- Try to answer the query from the context you get from the tool 'get_context_for_user_query'. Even if the context does not directly answer the question, try to relate it to the question and formulate an answer.
+- If someone asks about career opportunities and there is no information regarding it in the context provided by the tool 'get_context_for_user_query', direct them to our career page [here](https://www.sifars.com/en/career/). Only provided the link if it does not already exist in the chat history.
 """
 
+CONTACT_TOOL={
+    "type": "function",
+    "function": {
+        "name": "contact_us",
+        "description": "Collect the information dropped by user in the chat to contact the sifars team",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "name": {
+                    "type": "string",
+                    "description": "This field is the name of the user which you will collect from the user"
+                },
+                "email": {
+                    "type": "string",
+                    "description": "This field is the email of the user which you will collect from the user"
+                },
+                "phone_number": {
+                    "type": "string",
+                    "description": "This field is the phone number of the user which you will collect from the user"
+                },
+                "reason_for_contact": {
+                    "type": "string",
+                    "description": "This field is the reason for contact of the user which you will collect. This is seperate from the subject of the email. Keep it same that user has given in the chat. Do not change it."
+                },
+                "subject": {
+                    "type": "string",
+                    "description": "This is the subject of the email which will be dynamically contructed by you using the reason for contact provided. It is mandatory that you do not mention this field to the user."
+                }
+            },
+            "required": ["name", "email", "phone_number","reason_for_contact", "subject"]
+        }
+    }
+}
+
+GET_CONTEXT_FOR_USER_QUERY_TOOL={
+    "type": "function",
+    "function": {
+        "name": "get_context_for_user_query",
+        "description": "This function will give the similar texts from the database based on the user query",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "query": {
+                    "type": "string",
+                    "description": "This is the user query. Do not reformulate the query. Pass the query as it is."
+                }
+            },
+            "required": ["query"]
+        }
+    }
+}
+
+
 class ChatClient:
     def __init__(
         self,
@@ -57,21 +121,73 @@ class ChatClient:
     async def __aexit__(self, exc_type, exc, traceback):
         pass
 
-    async def chat(
+    async def create_chat_completions(
         self,
-        messages: list,
-        context: str
+        messages: list
     ):
-        async for message in await self.client.chat.completions.create(
+        response = await self.client.chat.completions.create(
             messages=[
                 {"role": "system", "content": self.system_message},
-                *messages[:-1],
-                {"role": "user", "content": f"Context is {context} and User query is {messages[-1]['content']}"}
+                *messages
             ],
             model=self.model,
             max_tokens=self.max_tokens,
             stream=self.stream,
-            temperature=0.7
-        ):
-            if message.choices[0].delta.content:
-                yield message.choices[0].delta.content
+            temperature=0.7,
+            tools=[CONTACT_TOOL, GET_CONTEXT_FOR_USER_QUERY_TOOL],
+            tool_choice="auto",
+        )
+        tool_call_response = None
+        async for chunk in response:
+            delta = chunk.choices[0].delta
+            if delta and delta.content:
+                yield delta.content
+            elif delta:
+                if not tool_call_response:
+                    tool_call_response = delta
+                if not tool_call_response.tool_calls and delta.tool_calls:
+                    tool_call_response.tool_calls = delta.tool_calls
+                if (
+                    tool_call_response
+                    and delta.tool_calls
+                    and len(tool_call_response.tool_calls)
+                    < delta.tool_calls[0].index + 1
+                ):
+                    tool_call_response.tool_calls.append(delta.tool_calls[0])
+        if tool_call_response and tool_call_response.tool_calls:
+            tool_calls_output = await self._handle_required_action(
+                tool_calls=tool_call_response.tool_calls,
+            )
+            messages.append(tool_call_response)
+            messages.extend(tool_calls_output)
+            async for chunk in self.create_chat_completions(messages=messages):
+                yield chunk
+
+    async def _handle_required_action(self, tool_calls: list[dict]):
+        tool_calls_output = []
+        for tool in tool_calls:
+            if tool.type == "function":
+                try:
+                    function_to_call = AVAILABLE_FUNCTIONS[tool.function.name]
+                    function_arguments = json.loads(str(tool.function.arguments)) if tool.function.arguments else {}
+                    async with ToolCall() as tool_call:
+                        function_response = await getattr(tool_call, function_to_call)(
+                            function_arguments
+                        )
+                except Exception as e:
+                    logger.error(e)
+                    function_response = "Unable to call the tool."
+                tool_calls_output.append(
+                    {
+                        "role": "tool",
+                        "tool_call_id": tool.id,
+                        "name": tool.function.name,
+                        "content": (
+                            str(function_response)
+                            if function_response
+                            else "No results found."
+                        ),
+                    }
+                )
+        return tool_calls_output
+

src/utils/_email_client.py

@@ -0,0 +1,54 @@
+import os
+from email.message import EmailMessage
+
+import aiosmtplib
+
+from ._config import logger
+
+
+class EmailClient:
+    def __init__(self):
+        self.message = EmailMessage()
+        self.message["From"] = os.getenv("SENDER_EMAIL")
+        self.message["To"] = os.getenv("RECIPIENT_EMAIL")
+        self.smtp_host = os.getenv("SMTP_HOST")
+        self.smtp_port = int(os.getenv("SMTP_PORT"))
+        self.smtp_username = os.getenv("SMTP_USERNAME")
+        self.smtp_password = os.getenv("SMTP_PASSWORD")
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        pass
+
+    async def send_email(self, details: dict, attachment_paths=[]):
+        subject = details.get("subject")
+        name = details.get("name")
+        email = details.get("email")
+        phone_number = details.get("phone_number")
+        reason_for_contact = details.get("reason_for_contact")
+        subject = details.get("subject")
+        self.message["Subject"] = subject
+        content = f"""Hello Team, \n\n{name} wants to connect with us regarding the following reason: \n{reason_for_contact}. \n\nContact Details:\nEmail: {email}\nPhone Number: {phone_number}\n\nThanks for your consideration."""
+        self.message.set_content(content)
+        if attachment_paths:
+            for attachment_path in attachment_paths:
+                with open(attachment_path, "rb") as f:
+                    file_data = f.read()
+                    file_name = os.path.basename(attachment_path)
+                    self.message.add_attachment(
+                        file_data,
+                        maintype="application",
+                        subtype="octet-stream",
+                        filename=file_name,
+                    )
+        response = await aiosmtplib.send(
+            self.message,
+            hostname=self.smtp_host,
+            port=self.smtp_port,
+            start_tls=True,
+            username=self.smtp_username,
+            password=self.smtp_password,
+        )
+        logger.info(response)

src/utils/_pinecone_client.py

@@ -82,7 +82,8 @@ class PineconeClient:
             )
         return
     
-    async def _get_similar_texts(self, query_text, top_k=5, index_name=None):
+    async def get_context_for_user_query(self, details: dict, top_k=2, index_name=None):
+        query_text = details['query']
         if not index_name:
             index_name = self.index_name
         embeddings = await self._create_embeddings([query_text])
@@ -92,8 +93,10 @@ class PineconeClient:
             top_k=top_k,
             include_metadata=True
         )
+        content = []
         for metadata in results['matches']:
-            yield metadata["metadata"]["text"]
+            content.append(metadata["metadata"]["text"])
+        return "\n".join(content)
 
     async def _delete_index(self, index_name=None):
         if not index_name:

src/utils/_sharepoint_client.py

@@ -0,0 +1,55 @@
+import os
+
+import aiofiles
+import httpx
+
+
+class SharepointClient:
+    def __init__(self):
+        self.access_token_url = f"https://login.microsoftonline.com/{os.getenv('AZURE_TENANT_ID')}/oauth2/v2.0/token"
+        self.access_token_data = {
+            "grant_type": "client_credentials",
+            "client_id": os.getenv("AZURE_CLIENT_ID"),
+            "client_secret": os.getenv("AZURE_CLIENT_SECRET"),
+            "scope": os.getenv("AZURE_SCOPE"),
+        }
+        self.token = None
+
+    async def __aenter__(self):
+        async with httpx.AsyncClient() as client:
+            token_response = await client.post(
+                self.access_token_url, data=self.access_token_data
+            )
+            self.token = token_response.json()["access_token"]
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        pass
+
+    async def upload_file(self, file_path: str):
+        async with httpx.AsyncClient() as client:
+            headers = {
+                "Authorization": f"Bearer {self.token}",
+                "Content-Type": "application/json",
+            }
+            file_name = os.path.basename(file_path)
+            async with aiofiles.open(file_path, "rb") as file:
+                file_data = await file.read()
+            upload_url = f"https://graph.microsoft.com/v1.0/drives/{os.getenv('AZURE_DRIVE_ID')}/root:/{file_name}:/content"
+            response = await client.put(
+                upload_url, headers=headers, content=file_data
+            )
+        uploaded_response = response.json()
+        web_url = uploaded_response["webUrl"]
+        return web_url
+    
+    async def get_files(self):
+        async with httpx.AsyncClient() as client:
+            headers = {
+                "Authorization": f"Bearer {self.token}",
+                "Content-Type": "application/json",
+            }
+            files_url = f"https://graph.microsoft.com/v1.0/drives/{os.getenv('AZURE_DRIVE_ID')}/root/children"
+            response = await client.get(files_url, headers=headers)
+        files = response.json()
+        return files

src/utils/_tool_call.py

@@ -0,0 +1,21 @@
+from ._email_client import EmailClient
+from ._pinecone_client import PineconeClient
+
+class ToolCall:
+    def __init__(self):
+        pass
+
+    async def __aenter__(self):
+        return self
+    
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        pass
+
+    async def send_email(self, details: dict):
+        async with EmailClient() as email_client:
+            await email_client.send_email(details)
+        return "Email sent successfully"
+    
+    async def get_context_for_user_query(self, details: dict):
+        async with PineconeClient() as pinecone_client:
+            return await pinecone_client.get_context_for_user_query(details)