add:updated

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.github/workflows/dockerhub.yaml

@@ -0,0 +1,39 @@
+name: Publish ConversAI Docker image
+
+on:
+  push:
+    branches: [ jan20 ]
+
+jobs:
+  push_to_registry:
+    name: Push Docker image to Docker Hub
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+      attestations: write
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v4
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        with:
+          images: techconsp/tcp_frwjeiqhpyl9ty53cyfg2jtpbhwwzl_co
+
+      - name: Build and push Docker image
+        id: push
+        uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
+        with:
+          context: .
+          file: ./Dockerfile
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.gitignore

@@ -0,0 +1,149 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# Machine Learning and Speech Libraries
+# TensorFlow
+*.ckpt*
+*.pbtxt
+*.tfevents*
+# PyTorch
+*.pt
+# Keras
+*.h5
+# Scikit-learn
+*.pkl
+# Speech Recognition
+*.wav
+*.mp3
+.idea/
+logs
+images
+resources
+experiments

Dockerfile

@@ -0,0 +1,32 @@
+FROM python:3.10-slim
+
+WORKDIR /app
+
+COPY . /app
+
+RUN chmod -R 777 /app
+
+RUN apt-get update && \
+    apt-get upgrade -y && \
+    apt-get install -y \
+        build-essential \
+        git \
+        cmake \
+        poppler-utils \
+        ffmpeg \
+        libsm6 \
+        libxext6 && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN pip install --no-cache-dir nltk && \
+    mkdir -p /app/nltk_data && \
+    chmod -R 777 /app/nltk_data && \
+    python -m nltk.downloader -d /app/nltk_data all
+
+RUN pip install --no-cache-dir --upgrade pip && \
+    pip install --no-cache-dir -r requirements.txt
+
+EXPOSE 8000
+
+CMD ["python", "app.py"]

README.md

@@ -0,0 +1,10 @@
+---
+title: J7RSYILECL
+emoji: 🚀
+colorFrom: purple
+colorTo: purple
+sdk: docker
+pinned: false
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,55 @@
+"""
+Created By: ishwor subedi
+Date: 2024-07-31
+"""
+import uvicorn
+from fastapi import FastAPI, Depends
+from fastapi.middleware.cors import CORSMiddleware
+from src import logging as logger
+from src.api.conversai_api import conversai_api_router
+from src.api.speech_api import speech_translator_router
+from src.api.user_management_api import user_management_api_router
+from src.utils.utils import load_ini_config
+from src.api.analytics_api import analytic_endpoints_router
+from src.api.jwt_bearer import access_check_bearer
+from src.api.jewel_mirror import jewel_api_router
+
+
+logger.info("---------------------------> Starting FastAPI Server <---------------------------")
+
+config = load_ini_config("config.ini")
+app = FastAPI(docs_url=config.get('fastapi_config', 'docs_url'), redoc_url=config.get('fastapi_config', 'redoc_url'),
+              openapi_url=config.get('fastapi_config', 'openapi_url')
+              )
+
+PROTECTED = [Depends(access_check_bearer)]
+
+logger.info("---------------------------> FastAPI Server Started <---------------------------")
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],  # Allow access from all sources
+    allow_credentials=True,
+    allow_methods=["*"],  # Allow all HTTP methods
+    allow_headers=["*"],  # Allow all request headers
+)
+app.include_router(user_management_api_router, prefix="/conversai")
+logger.info("---------------------------> User Management API Started <---------------------------")
+
+app.include_router(conversai_api_router, prefix="/conversai")
+logger.info("---------------------------> ConversAI API Started <---------------------------")
+
+app.include_router(speech_translator_router, prefix="/conversai")
+logger.info("---------------------------> Speech Translator API Started <---------------------------")
+
+app.include_router(analytic_endpoints_router, prefix="/conversai",dependencies=PROTECTED)
+logger.info("---------------------------> Analytics API Started <---------------------------")
+
+#for goldpricing
+app.include_router(jewel_api_router,prefix="/conversai")
+logger.info("---------------------------> Goldpricing API Started <---------------------------")
+
+
+
+if __name__ == '__main__':
+    uvicorn.run(app, port=int(config.get('fastapi_config', 'port')), host=config.get('fastapi_config', 'host'),
+                timeout_keep_alive=300, timeout_graceful_shutdown=600)

config.ini

@@ -0,0 +1,39 @@
+[default]
+chatbot_name = ConversAI
+chatbot_prefix = convai
+
+[data_analyzer]
+groq_llm_name = llama-3.1-8b-instant
+additional_query = .In case, you are to plot a chart, make sure the x-axis labels are 90 degree rotated.
+verbose = False
+
+; [easy_ocr]
+; model_path = resources/easyocr_model
+; language = en
+; gpu = True
+
+[speech_to_text]
+model_id = openai/whisper-large-v3
+max_new_tokens = 128
+chunks_length_s = 30
+batch_size = 16
+
+[supabase_chatbot_management]
+user_config_table = ConversAI_UserConfig
+chat_bot_table = ConversAI_ChatbotInfo
+
+; [all_mini_l6_v2_vector_embedding]
+; device = cuda
+; normalize_embeddings = True
+
+
+[fastapi_config]
+host = 0.0.0.0
+port = 8000
+docs_url = /docs
+redoc_url = /redoc
+openapi_url = /openapi.json
+
+
+[oauth]
+redirect_to : https://convers-ai-test.vercel.app/home/

examples/easy_ocr_example.py

@@ -0,0 +1,12 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""
+import cv2
+
+from src.services.ocr.easy_ocr.easy_ocr_ import EasyOCR_
+
+if __name__ == '__main__':
+    image = cv2.imread("images/img.png")
+    ocr = EasyOCR_()
+    print(ocr.read_text(image))

examples/url_text_extraction_example.py

@@ -0,0 +1,10 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""
+from src.services.website_url.text_extraction_urls import extract_text_from_url
+
+if __name__ == '__main__':
+    website = "https://huggingface.co/BAAI/bge-m3"
+    extracted_text = extract_text_from_url(website)
+    print(extracted_text)

logging_config.yaml

@@ -0,0 +1,27 @@
+version: 1
+disable_existing_loggers: False
+formatters:
+  standard:
+    format: '[%(asctime)s: %(levelname)s: %(module)s: %(message)s]'
+handlers:
+  console:
+    class: logging.StreamHandler
+    level: DEBUG
+    formatter: standard
+    stream: ext://sys.stdout
+  file:
+    class: logging.handlers.TimedRotatingFileHandler
+    level: INFO
+    formatter: standard
+    filename: logs/application.log
+    when: midnight
+    interval: 1
+    backupCount: 30
+loggers:
+  __main__:
+    level: DEBUG
+    handlers: [console, file]
+    propagate: no
+root:
+  level: DEBUG
+  handlers: [console, file]

requirements.txt

@@ -0,0 +1,100 @@
+accelerate==0.33.0
+annotated-types==0.7.0
+anyio==4.4.0
+certifi==2024.7.4
+charset-normalizer==3.3.2
+click==8.1.7
+dnspython==2.6.1
+email_validator==2.2.0
+exceptiongroup==1.2.2
+fastapi==0.111.1
+fastapi-cli==0.0.4
+filelock==3.15.4
+fsspec==2024.6.1
+gTTS==2.5.2
+h11==0.14.0
+httpcore==1.0.5
+httptools==0.6.1
+httpx==0.27.0
+huggingface-hub==0.24.5
+idna==3.7
+Jinja2==3.1.4
+markdown-it-py==3.0.0
+MarkupSafe==2.1.5
+mdurl==0.1.2
+mpmath==1.3.0
+networkx==3.3
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==9.1.0.70
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.6.20
+nvidia-nvtx-cu12==12.1.105
+packaging==24.1
+psutil==6.0.0
+pydantic==2.8.2
+pydantic_core==2.20.1
+Pygments==2.18.0
+python-dotenv==1.0.1
+python-multipart==0.0.9
+PyYAML==6.0.1
+regex==2024.7.24
+requests==2.32.3
+rich==13.7.1
+safetensors==0.4.3
+scikit-build==0.18.0
+shellingham==1.5.4
+sniffio==1.3.1
+starlette==0.37.2
+sympy==1.13.1
+tokenizers==0.19.1
+tomli==2.0.1
+torch==2.4.0
+tqdm==4.66.4
+transformers @ git+https://github.com/huggingface/transformers.git@85a1269e19af022e04bc2aad82572cd5a9e8cdd9
+triton==3.0.0
+typer==0.12.3
+typing_extensions==4.12.2
+urllib3==2.2.2
+uvicorn==0.30.4
+uvloop==0.19.0
+watchfiles==0.22.0
+websockets==12.0
+bs4
+huggingface-hub
+fastembed
+nest_asyncio
+beautifulsoup4
+flashrank
+flashrank[listwise]
+PyMuPDF
+langchain
+langchain-community
+langchain-cohere
+langchain-huggingface
+langchain-qdrant
+langchain-groq
+lxml
+python-dotenv
+pillow
+pandas
+sentence-transformers
+supabase
+unstructured
+urllib3
+langsmith
+pandasai
+easyocr
+youtube-transcript-api
+pdf2image
+PyPDF2
+PyJWT
+replicate
+langgraph

src/__init__.py

@@ -0,0 +1,19 @@
+"""
+Created By: ishwor subedi
+Date: 2024-07-31
+"""
+
+import logging.config
+import yaml
+import os
+
+if os.path.exists("logs"):
+    pass
+else:
+    os.makedirs("logs")
+
+log_config_path = os.path.join(os.getcwd(), "logging_config.yaml")
+with open(log_config_path, 'r') as file:
+    config = yaml.safe_load(file.read())
+
+logging.config.dictConfig(config)

src/api/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-07-31
+"""

src/api/analytics_api.py

@@ -0,0 +1,235 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-28
+"""
+from collections import Counter, defaultdict
+from datetime import datetime, timedelta
+from dateutil.parser import isoparse
+from fastapi.routing import APIRouter
+from src.pipeline.conversai_analytic_pipeline import ConversAIAnalyticPipeline
+from fastapi import Request
+from src.utils.error_handling import create_success_response, raise_http_exception, \
+    success_response_user_management
+from src.models.apis_models import FeedbackRequest, DailyActiveEndUserRequest, AverageSessionInteractionRequest, \
+    TokenUsageRequest, UserSatisfactionRateRequest
+from src import logging as logger
+
+analytic_endpoints_router = APIRouter(tags=["Analytics Endpoints"])
+
+conversai_analytic_pipeline = ConversAIAnalyticPipeline()
+
+
+@analytic_endpoints_router.post("/daily_chat_count")
+async def daily_chat_count(
+        request: DailyActiveEndUserRequest):
+    start_date, end_date, vectorstore = request.start_date, request.end_date, request.vectorstore
+    logger.info(f">>> daily_chat_count API Triggered by {vectorstore} <<<")
+    try:
+        if not start_date or not end_date:
+            end_date = datetime.now().astimezone().date()
+            start_date = end_date - timedelta(days=7)
+        else:
+            start_date = isoparse(start_date).date()
+            end_date = isoparse(end_date).date()
+
+        response = conversai_analytic_pipeline.chat_history_table_(vectorstore=vectorstore)
+
+        dates = [
+            isoparse(i["timestamp"]).date()
+            for i in response
+            if start_date <= isoparse(i["timestamp"]).date() <= end_date
+        ]
+
+        date_count = Counter(dates)
+
+        data = [{"date": date.isoformat(), "count": count} for date, count in date_count.items()]
+
+        response = create_success_response(code=200, data=dict(output=data))
+        logger.info(f">>> daily_chat_count API Response Success  for {vectorstore} <<<")
+
+        return response
+
+    except Exception as e:
+        logger.error(f">>> daily_chat_count API Response Failed for {vectorstore} {e}<<<")
+
+        raise_http_exception(500, "Internal Server Error")
+
+
+@analytic_endpoints_router.post("/daily_active_end_user")
+async def daily_active_end_user(
+        request: DailyActiveEndUserRequest
+):
+    start_date, end_date, vectorstore = request.start_date, request.end_date, request.vectorstore
+    logger.info(f">>> daily_active_end_user API Triggered by {vectorstore} <<<")
+    try:
+        if not start_date or not end_date:
+            end_date = datetime.now().astimezone().date()
+            start_date = end_date - timedelta(days=7)
+        else:
+            start_date = isoparse(start_date).date()
+            end_date = isoparse(end_date).date()
+
+        response = conversai_analytic_pipeline.chat_history_table_(vectorstore=vectorstore)
+
+        ip_by_date = defaultdict(set)
+
+        for i in response:
+            timestamp = isoparse(i["timestamp"])
+            ip_address = i["IpAddress"]
+            if start_date <= timestamp.date() <= end_date:
+                date = timestamp.date()
+                ip_by_date[date].add(ip_address)
+
+        data = [{"date": date.isoformat(), "terminal": len(ips)} for date, ips in ip_by_date.items() if len(ips) > 1]
+
+        response = create_success_response(code=200, data=dict(output=data))
+        logger.info(f">>> daily_active_end_user API Response Success for {vectorstore} <<<")
+
+        return response
+    except Exception as e:
+        logger.error(f">>> daily_active_end_user API Response Failed for {vectorstore} {e}<<<")
+
+        raise_http_exception(500, "Internal Server Error")
+
+
+@analytic_endpoints_router.post("/average_session_interaction")
+async def average_session_interaction(
+        request: AverageSessionInteractionRequest
+):
+    start_date, end_date, vectorstore = request.start_date, request.end_date, request.vectorstore
+    logger.info(f">>> average_session_interaction API Triggered by {vectorstore} <<<")
+    try:
+        if not start_date or not end_date:
+            end_date = datetime.now().astimezone().date()
+            start_date = end_date - timedelta(days=7)
+        else:
+            start_date = isoparse(start_date).date()
+            end_date = isoparse(end_date).date()
+
+        response = conversai_analytic_pipeline.chat_history_table_(vectorstore=vectorstore)
+
+        total_messages_by_date = defaultdict(int)
+        unique_ips_by_date = defaultdict(set)
+
+        for i in response:
+            timestamp = isoparse(i["timestamp"])
+            ip_address = i["IpAddress"]
+            if start_date <= timestamp.date() <= end_date:
+                date = timestamp.date()
+                total_messages_by_date[date] += 1
+                unique_ips_by_date[date].add(ip_address)
+
+        data = []
+        for date in sorted(total_messages_by_date.keys()):
+            total_messages = total_messages_by_date[date]
+            unique_ips = len(unique_ips_by_date[date])
+            average_interactions = total_messages / unique_ips if unique_ips > 0 else 0
+            data.append({"date": date.isoformat(), "interactions": average_interactions})
+
+        response = create_success_response(code=200, data=dict(data=data))
+        logger.info(f">>> average_session_interaction API Response Success for {vectorstore} <<<")
+
+        return response
+    except Exception as e:
+        logger.error(f">>> average_session_interaction API Response Failed for {vectorstore} {e}<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@analytic_endpoints_router.post("/token_usages")
+async def token_usages(request: TokenUsageRequest):
+    start_date, end_date, vectorstore = request.start_date, request.end_date, request.vectorstore
+    logger.info(f">>> token_usages API Triggered by {vectorstore} <<<")
+    try:
+        if not start_date or not end_date:
+            end_date = datetime.now().astimezone().date()
+            start_date = end_date - timedelta(days=7)
+        else:
+            start_date = isoparse(start_date).date()
+            end_date = isoparse(end_date).date()
+
+        response = conversai_analytic_pipeline.chat_history_table_(vectorstore=vectorstore)
+
+        token_usage_by_date = defaultdict(int)
+
+        for i in response:
+            timestamp = isoparse(i["timestamp"])
+            if start_date <= timestamp.date() <= end_date:
+                date = timestamp.date()
+                response_token_count = i.get("ResponseTokenCount")
+                if response_token_count is not None:
+                    token_usage_by_date[date] += response_token_count
+
+        data = [{"date": date.isoformat(), "total_tokens": total_tokens} for date, total_tokens in
+                token_usage_by_date.items()]
+
+        response = create_success_response(code=200, data=dict(output=data))
+        logger.info(f">>> token_usages API Response Success for {vectorstore} <<<")
+
+        return response
+    except Exception as e:
+        logger.error(f">>> token_usages API Response Failed for {vectorstore} {e}<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@analytic_endpoints_router.post("/add_feedback")
+async def add_feedback(req: Request, request: FeedbackRequest):
+    feedback, user_id, vectorstore = request.feedback, request.user_id, request.vectorstore
+    try:
+        logger.info(f">>> add_feedback API Triggered by {request.vectorstore} <<<")
+
+        client_ip = req.client.host
+        city = conversai_analytic_pipeline.get_ip_info(client_ip)
+
+        conversai_analytic_pipeline.add_feedback_(feedback, user_id, city, client_ip, vectorstore)
+
+        response = success_response_user_management(code=200, message="Add Feedback Sucess")
+        logger.info(f">>> add_feedback API Response Success for {vectorstore} <<<")
+
+        return response
+
+    except Exception as e:
+        logger.error(f">>> add_feedback API Response Failed for {vectorstore} {e}<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@analytic_endpoints_router.post("/user_satisfaction_rate")
+async def user_satisfaction_rate(
+        request: UserSatisfactionRateRequest
+):
+    start_date, end_date, vectorstore = request.start_date, request.end_date, request.vectorstore
+    logger.info(f">>> user_satisfaction_rate API Triggered by {vectorstore} <<<")
+    try:
+        if not start_date or not end_date:
+            end_date = datetime.now().astimezone().date()
+            start_date = end_date - timedelta(days=7)
+        else:
+            start_date = isoparse(start_date).date()
+            end_date = isoparse(end_date).date()
+
+        feedback_counts = defaultdict(lambda: {"like": 0, "dislike": 0})
+        response = conversai_analytic_pipeline.feedback_table_(vectorstore)
+        for i in response:
+            timestamp = isoparse(i["timestamp"])
+            if start_date <= timestamp.date() <= end_date:
+                date = timestamp.date()
+                feedback = i.get("feedback")
+                if feedback == "like":
+                    feedback_counts[date]["like"] += 1
+                elif feedback == "dislike":
+                    feedback_counts[date]["dislike"] += 1
+
+        data = []
+        for date in sorted(feedback_counts.keys()):
+            like_count = feedback_counts[date]["like"]
+            dislike_count = feedback_counts[date]["dislike"]
+            total_feedback = like_count + dislike_count
+            satisfaction_rate = (like_count / total_feedback * 100) if total_feedback > 0 else 0
+            data.append({"date": date.isoformat(), "rate": satisfaction_rate})
+
+        response = create_success_response(code=200, data=dict(output=data))
+        logger.info(f">>> user_satisfaction_rate API Response Success for {vectorstore} <<<")
+
+        return response
+    except Exception as e:
+        logger.info(f">>> user_satisfaction_rate API Response Failed for {vectorstore} {e}<<<")
+        raise_http_exception(500, "Internal Server Error")

src/api/conversai_api.py

@@ -0,0 +1,650 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-02
+"""
+import io
+import json
+import os
+import string
+import tempfile
+import requests
+import pandas as pd
+from src import logging as logger
+from supabase import create_client
+from urllib.parse import urlparse
+
+from src.api.jwt_bearer import access_check_bearer
+from src.models.apis_models import *
+from fastapi.requests import Request
+from fastapi.routing import APIRouter
+from fastapi import UploadFile, File, HTTPException, Form, Depends
+from src.pipeline.conversai_pipeline import ConversAIPipeline
+from src.api.user_management_api import user_management
+from src.services.supabase.analytics.analytic_tables import track_usage
+from src.services.supabase.user_management.token_limit import token_limit_check
+from src.utils.error_handling import create_error_response, create_success_response, raise_http_exception
+from src.api.user_management_api import user_management as user_management_pipeline
+from src.utils.utils import get_ip_info, encode_to_base64, clean_text, decode_base64
+
+conversai_api_router = APIRouter(tags=["ConversAI"])
+
+supabase_client = create_client(os.getenv("SUPABASE_URL"), os.getenv("SUPABASE_KEY"))
+supabase_client_ = supabase_client
+conversai_pipeline = ConversAIPipeline()
+
+
+@conversai_api_router.post("/add_text" )
+async def add_text(request: AddTextRequest):
+    logger.info(f">>>AddText API Triggered By {request.vectorstore}<<<")
+    try:
+        vectorstore, text = request.vectorstore, request.text
+        track_usage(vectorstore=vectorstore, endpoint="/add_text", supabase_client=supabase_client)
+        username, chat_bot_name = request.vectorstore.split("$")[1], request.vectorstore.split("$")[2]
+        cleaned_text = " ".join(text.split())
+        num_token = len(cleaned_text)
+        lim = token_limit_check(supabase_client=supabase_client, username=username, chatbot_name=chat_bot_name,
+                                len_text=num_token)
+        text = clean_text(text)
+        if lim:
+            dct = {
+                "output": {"text": text},
+                "source": "Text",
+            }
+            cleaned_text = " ".join(text.split()) #handles unnencessary spaces
+            # Count characters
+            num_token = len(cleaned_text)
+            logger.info(f"Number of token {num_token}")
+            dct = json.dumps(dct, indent=1).encode("utf-8", errors="replace")
+            file_name = user_management_pipeline.create_data_source_name(source_name="text", username=username)
+            supabase_client.storage.from_("ConversAI").upload(file=dct, path=f"{file_name}_data.json")
+
+            supa = supabase_client.table("ConversAI_ChatbotDataSources").insert(
+                {"username": username, "chatbotName": chat_bot_name, "dataSourceName": file_name,
+                 "numTokens": num_token, "sourceEndpoint": "/add_text",
+                 "sourceContentURL": os.path.join(os.environ["SUPABASE_PUBLIC_BASE_URL"],
+                                                  f"{file_name}_data.json")}).execute()
+
+            response = create_success_response(200, {"message": "Successfully added the text."})
+            logger.info(f">>>Text added successfully for {request.vectorstore}.<<<")
+
+            return response
+        else:
+            response = create_error_response(400,
+                                             "Exceeding limits, please try with a smaller chunks of information or subscribe to our premium plan.")
+            return response
+
+    except Exception as e:
+        logger.error(f">>>Error in add_text: {e} for {request.vectorstore}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.post("/answer_query")
+async def answer_query(request: AnswerQueryRequest, req: Request):
+    logger.info(f">>>answer_query API Triggered By {request.vectorstore}<<<")
+    try:
+        track_usage(supabase_client=supabase_client, vectorstore=request.vectorstore, endpoint="/answer_query")
+        username, chatbot_name = request.vectorstore.split("$")[1], request.vectorstore.split("$")[2]
+        ip_address = req.client.host
+        city = get_ip_info(ip_address)
+        output, followup_questions, source = conversai_pipeline.answer_query_(query=request.query,
+                                                                              vectorstore=request.vectorstore,
+                                                                              llm_model=request.llm_model)
+        supa = supabase_client.table("ConversAI_ChatHistory").insert(
+            {"username": username, "chatbotName": chatbot_name, "llmModel": request.llm_model,
+             "question": request.query, "response": output, "IpAddress": ip_address, "ResponseTokenCount": len(output),
+             "vectorstore": request.vectorstore, "City": city}).execute()
+
+        response = create_success_response(200, data={"output": output, "follow_up_questions": followup_questions,
+                                                      "source": source})
+        logger.info(f">>>Query answered successfully for {request.vectorstore}.<<<")
+        return response
+
+    except Exception as e:
+        logger.error(f">>>Error in answer_query: {e} for {request.vectorstore}.<<<")
+        # raise HTTPException(status_code=500, detail="Internal Server Error")
+        raise e
+
+@conversai_api_router.post("/data_analyzer" )
+async def data_analyzer(query: str = Form(...), file: UploadFile = File(...)):
+    logger.info(f">>>data_analyzer API Triggered By {query}<<<")
+    try:
+        extension = file.filename.split(".")[-1]
+        if extension in ["xls", "xlsx", "xlsm", "xlsb"]:
+            df = pd.read_excel(io.BytesIO(await file.read()))
+        elif extension == "csv":
+            df = pd.read_csv(io.BytesIO(await file.read()))
+        else:
+            return {"output": "INVALID FILE TYPE"}
+
+        response = conversai_pipeline.data_analyzer(query=query, dataframe=df)
+        response = create_success_response(200, {"output": response})
+        logger.info(f">>>Data analyzed successfully for {query}.<<<")
+        return response
+
+    except Exception as e:
+        logger.error(f">>>Error in data_analyzer: {e} for {query}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.post("/get_links" )
+async def get_links(request: GetLinksRequest):
+    logger.info(f">>>get_links API Triggered By {request.url}<<<")
+    try:
+        response = conversai_pipeline.get_links_(url=request.url, timeout=30)
+        response = create_success_response(200, {"urls": response, "source": urlparse(request.url).netloc})
+        logger.info(f">>>Links fetched successfully for {request.url}.<<<")
+        return response
+
+    except Exception as e:
+        logger.error(f">>>Error in get_links: {e} for {request.url}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.post("/image_pdf_text_extraction" )
+async def image_pdf_text_extraction(vectorstore: str = Form(...)
+                                    , pdf: UploadFile = File(...)):
+    logger.info(f">>>image_pdf_text_extraction API Triggered By {pdf.filename}<<<")
+    try:
+        track_usage(vectorstore=vectorstore, endpoint="/image_pdf_text_extraction", supabase_client=supabase_client)
+        username, chatbot_name = vectorstore.split("$")[1], vectorstore.split("$")[2]
+        source = pdf.filename
+        pdf_bytes = await pdf.read()
+        response = conversai_pipeline.image_pdf_text_extraction_(image_pdf=pdf_bytes)
+        num_tokens = len(" ".join([response[x] for x in response]))
+        lim = token_limit_check(supabase_client=supabase_client, username=username, chatbot_name=chatbot_name,
+                                len_text=num_tokens)
+        logger.info(f"this is the {lim}")
+        if lim:
+            dct = {
+                "output": response,
+                "source": source
+            }
+            dct = json.dumps(dct, indent=1).encode("utf-8", errors="replace")
+            file_name = user_management_pipeline.create_data_source_name(source_name=source, username=username)
+            num_tokens = len(" ".join([response[x] for x in response]))
+
+            response = supabase_client.storage.from_("ConversAI").upload(file=dct, path=f"{file_name}_data.json")
+            supa = supabase_client.table("ConversAI_ChatbotDataSources").insert(
+                {"username": username,
+                 "chatbotName": chatbot_name,
+                 "dataSourceName": file_name,
+                 "numTokens": num_tokens,
+                 "sourceEndpoint": "/image_pdf_text_extraction",
+                 "sourceContentURL": os.path.join(os.environ["SUPABASE_PUBLIC_BASE_URL"],
+                                                  f"{file_name}_data.json")}).execute()
+
+            response = create_success_response(200,
+                                               {"source": pdf.filename, "message": "Successfully extracted the text."})
+            logger.info(f">>>Text extracted successfully for {pdf.filename}.<<<")
+            return response
+        else:
+            response = create_error_response(402,
+                                             "Exceeding limits, please try with a smaller chunks of PDF or subscribe to our premium plan.")
+            return response
+
+    except Exception as e:
+        logger.error(f">>>Error in image_pdf_text_extraction: {e} for {pdf.filename}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.post("/text_pdf_extraction" )
+async def text_pdf_extraction(vectorstore: str = Form(...)
+                              , pdf: UploadFile = File(...)):
+    logger.info(f">>>text_pdf_extraction API Triggered By {pdf.filename}<<<")
+    try:
+        track_usage(vectorstore=vectorstore, endpoint="/text_pdf_extraction", supabase_client=supabase_client)
+        username, chatbot_name = vectorstore.split("$")[1], vectorstore.split("$")[2]
+        source = pdf.filename
+        pdf = await pdf.read()
+
+        with tempfile.NamedTemporaryFile(delete=False, suffix='.pdf') as temp_file:
+            temp_file.write(pdf)
+            temp_file_path = temp_file.name
+
+        response = conversai_pipeline.text_pdf_extraction_(pdf=temp_file_path)
+        numTokens = len(" ".join([response[x] for x in response]))
+        lim = token_limit_check(supabase_client=supabase_client, username=username, chatbot_name=chatbot_name,
+                                len_text=numTokens)
+        os.remove(temp_file_path)
+        if lim:
+            dct = {
+                "output": response,
+                "source": source
+            }
+            numTokens=len(" ".join([response[x] for x in response]))
+            logger.info(f"Num of tokens {numTokens} text_pdf_extraction")
+            dct = json.dumps(dct, indent=1).encode("utf-8", errors="replace")
+            file_name = user_management_pipeline.create_data_source_name(source_name=source, username=username)
+            response = supabase_client.storage.from_("ConversAI").upload(file=dct, path=f"{file_name}_data.json")
+            response = (
+                supabase_client.table("ConversAI_ChatbotDataSources")
+                .insert({"username": username,
+                         "chatbotName": chatbot_name,
+                         "dataSourceName": file_name,
+                         "numTokens": numTokens,
+                         "sourceEndpoint": "/text_pdf_extraction",
+                         "sourceContentURL": os.path.join(os.environ["SUPABASE_PUBLIC_BASE_URL"],
+                                                          f"{file_name}_data.json")})
+                .execute()
+            )
+            response = create_success_response(200, {"source": source, "message": "Successfully extracted the text."})
+            logger.info(f">>>Text extracted successfully for {source}.<<<")
+            return response
+
+        else:
+            response = create_error_response(402,
+                                             "Exceeding limits, please try with a smaller chunks of PDF or subscribe to our premium plan.")
+            return response
+
+    except Exception as e:
+        logger.error(f">>>Error in text_pdf_extraction: {e} for {vectorstore}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.post("/youtube_transcript" )
+async def youtube_transcript(request: YoutubeTranscriptRequest):
+    vectorstore, urls = request.vectorstore, request.urls
+    logger.info(f">>>youtube_transcript API Triggered By {urls}<<<")
+    try:
+        track_usage(supabase_client=supabase_client, vectorstore=vectorstore, endpoint="/youtube_transcript")
+        username, chatbot_name = vectorstore.split("$")[1], vectorstore.split("$")[2]
+
+        response = conversai_pipeline.youtube_transcript_(url=urls)
+        num_tokens = len(" ".join([response[x] for x in response]))
+        lim = token_limit_check(supabase_client=supabase_client, username=username, chatbot_name=chatbot_name,
+                                len_text=num_tokens)
+        if lim:
+
+            dct = {
+                "output": response,
+                "source": "www.youtube.com"
+            }
+
+
+            dct = json.dumps(dct, indent=1).encode("utf-8", errors="replace")
+            file_name = user_management_pipeline.create_data_source_name(source_name="youtube", username=username)
+            response = supabase_client.storage.from_("ConversAI").upload(file=dct, path=f"{file_name}_data.json")
+            response = (
+                supabase_client.table("ConversAI_ChatbotDataSources")
+                .insert({"username": username,
+                         "chatbotName": chatbot_name,
+                         "dataSourceName": file_name,
+                         "numTokens": num_tokens,
+                         "sourceEndpoint": "/youtube_transcript",
+                         "sourceContentURL": os.path.join(os.environ["SUPABASE_PUBLIC_BASE_URL"],
+                                                          f"{file_name}_data.json")})
+                .execute()
+            )
+            response = create_success_response(200, {"message": "Successfully fetched the youtube transcript."})
+            logger.info(f">>>Youtube transcript fetched successfully for {urls}.<<<")
+            return response
+        else:
+            response = create_error_response(402,
+                                             "Exceeding limits, please try with a smaller chunks of information or subscribe to our premium plan.")
+            return response
+
+    except Exception as e:
+        logger.error(f">>>Error in youtube_transcript: {e} for {urls}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.post("/website_url_text_extraction" )
+async def add_website(request: AddWebsiteRequest):
+    vectorstore, website_urls, source = request.vectorstore, request.website_urls, request.source
+
+    logger.info(f">>>website_url_text_extraction API Triggered By {request.website_urls}<<<")
+    try:
+        track_usage(supabase_client=supabase_client, vectorstore=vectorstore, endpoint="/fetch_text/urls")
+        username, chatbot_name = vectorstore.split("$")[1], vectorstore.split("$")[2]
+
+        text = conversai_pipeline.website_url_text_extraction_list_(urls=website_urls)
+        num_token = len(" ".join([text[x] for x in text]))
+
+        logger.info(f">>>website_url_text_extraction len{num_token}<<<")
+        logger.info(f">>>website_url_text_extraction {text}<<<")
+
+        lim = token_limit_check(supabase_client=supabase_client, username=username, chatbot_name=chatbot_name,
+                                len_text=num_token)
+        if not lim:
+
+            response = create_error_response(402,
+                                             "Exceeding limits, please try with a smaller chunks of information or subscribe to our premium plan.")
+            return response
+        else:
+            dct = {
+                "output": text,
+                "source": source
+            }
+
+            dct = json.dumps(dct, indent=1).encode("utf-8", errors="replace")
+            file_name = user_management_pipeline.create_data_source_name(source_name=urlparse(source).netloc,
+                                                                         username=username)
+            supabase_client.storage.from_("ConversAI").upload(file=dct, path=f"{file_name}_data.json")
+            (
+                supabase_client.table("ConversAI_ChatbotDataSources")
+                .insert({"username": username,
+                         "chatbotName": chatbot_name,
+                         "dataSourceName": file_name,
+                         "numTokens": num_token,
+                         "sourceEndpoint": "/fetch_text/urls",
+                         "sourceContentURL": os.path.join(os.environ["SUPABASE_PUBLIC_BASE_URL"],
+                                                          f"{file_name}_data.json")})
+                .execute()
+            )
+            response = create_success_response(200, {"message": "Successfully fetched the website text."})
+            logger.info(f">>>Website text extracted successfully for {request.website_urls}.<<<")
+            return response
+    except Exception as e:
+        logger.error(f">>>Error in website_url_text_extraction: {e} for {request.website_urls}.<<<")
+        raise e
+        # raise HTTPException(status_code=500, detail="Internal Server Error")
+
+
+
+
+
+@conversai_api_router.get("/get_current_count" )
+async def get_count(vectorstore: str):
+    logger.info(f">>>get_current_count API Triggered By {vectorstore}<<<")
+    try:
+        username, chatbot_name = vectorstore.split("$")[1], vectorstore.split("$")[2]
+        current_count = user_management_pipeline.get_current_count_(username)
+
+        response = create_success_response(200, {"current_count": current_count})
+        logger.info(f">>>Current count fetched successfully for {vectorstore}.<<<")
+        return response
+
+    except Exception as e:
+        logger.error(f">>>Error in get_current_count: {e} for {vectorstore}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.post("/list_chatbots" )
+async def list_chatbots(request: ListChatbotsRequest):
+    logger.info(f">>>list_chatbots API Triggered By {request.username}<<<")
+    try:
+        chatbots = user_management.list_tables(username=request.username)
+        response = create_success_response(200, {"chatbots": chatbots})
+        logger.info(f">>>Chatbots listed successfully for {request.username}.<<<")
+        return response
+
+    except Exception as e:
+        logger.error(f">>>Error in list_chatbots: {e} for {request.username}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.post("/get_chat_history" )
+async def chat_history(request: GetChatHistoryRequest):
+    logger.info(f">>>get_chat_history API Triggered By {request.vectorstore}<<<")
+    try:
+        _, username, chatbotName = request.vectorstore.split("$", 2)
+
+        history = supabase_client.table("ConversAI_ChatHistory").select(
+            "timestamp", "question", "response"
+        ).eq("username", username).eq("chatbotName", chatbotName).execute().data
+
+        response = create_success_response(200, {"history": history})
+        logger.info(f">>>Chat history fetched successfully for {request.vectorstore}.<<<")
+        return response
+
+
+    except IndexError:
+        logger.warning(f"Chat history not found for {request.vectorstore}")
+        return create_error_response(404, "Chat history not found for the given chatbot.")
+
+    except Exception as e:
+        logger.error(f">>>Error in get_chat_history: {e} for {request.vectorstore}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.post("/delete_chatbot" )
+async def delete_chatbot(request: DeleteChatbotRequest):
+    logger.info(f">>>delete_chatbot API Triggered By {request.vectorstore}<<<")
+    try:
+        username, chatbot_name = request.vectorstore.split("$")[1], request.vectorstore.split("$")[2]
+        supabase_client.table('ConversAI_ChatbotInfo').delete().eq('user_id', username).eq('chatbotname',
+                                                                                           chatbot_name).execute()
+        all_sources = supabase_client.table("ConversAI_ChatbotDataSources").select("*").eq("username", username).eq(
+            "chatbotName", chatbot_name).execute().data
+        all_sources = [x["sourceContentURL"].split("/")[-1] for x in all_sources]
+        supabase_client.table("ConversAI_ChatbotDataSources").delete().eq("username", username).eq("chatbotName",
+                                                                                                   chatbot_name).execute()
+        supabase_client.table("ConversAI_ActivityLog").update({"isActive": False}).eq("username", username).eq(
+            "chatbotName",
+            chatbot_name).execute()
+        supabase_client.table("ConversAI_ChatHistory").update({"isActive": False}).eq("username", username).eq(
+            "chatbotName",
+            chatbot_name).execute()
+        for source in all_sources:
+            supabase_client.table("ConversAI_Chatbot")
+            supabase_client.storage.from_("ConversAI").remove(source)
+        user_management.delete_table(table_name=chatbot_name)
+        user_management.delete_qdrant_cluster(vectorstorename=request.vectorstore)
+        response = create_success_response(200, {"message": "Chatbot deleted successfully"})
+        logger.info(f">>>Chatbot deleted successfully for {request.vectorstore}.<<<")
+        return response
+    except Exception as e:
+        logger.error(f">>>Error in delete_chatbot: {e} for {request.vectorstore}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.post("/add_qa_pair" )
+async def add_qa_pair(request: AddQAPairRequest):
+    logger.info(f">>>add_qa_pair API Triggered By {request.vectorstore}<<<")
+    try:
+        vectorstore, question, answer = request.vectorstore, request.question, request.answer
+        track_usage(vectorstore=vectorstore, endpoint="/add_qa_pair", supabase_client=supabase_client)
+        username, chat_bot_name = request.vectorstore.split("$")[1], request.vectorstore.split("$")[2]
+        normal_text = f"\nQUESTION: {question}\nANSWER: {answer}\n"
+        lim = token_limit_check(supabase_client=supabase_client, username=username, chatbot_name=chat_bot_name,
+                                text=normal_text)
+        if lim:
+            dct = {
+                "output": {"text": normal_text},
+                "source": "QA Pair",
+            }
+            num_token = len(normal_text.translate(str.maketrans('', '', string.punctuation)).split(" "))
+            dct = json.dumps(dct, indent=1).encode("utf-8", errors="replace")
+            file_name = user_management_pipeline.create_data_source_name(source_name="qa_pair", username=username)
+            supabase_client.storage.from_("ConversAI").upload(file=dct, path=f"{file_name}_data.json")
+            (
+                supabase_client.table("ConversAI_ChatbotDataSources")
+                .insert({"username": username,
+                         "chatbotName": chat_bot_name,
+                         "dataSourceName": file_name,
+                         "numTokens": num_token,
+                         "sourceEndpoint": "/add_qa_pair",
+                         "sourceContentURL": os.path.join(os.environ["SUPABASE_PUBLIC_BASE_URL"],
+                                                          f"{file_name}_data.json")})
+                .execute()
+            )
+
+            response = create_success_response(200, {"message": "Successfully added the qa pair."})
+            logger.info(f">>>QA Pair added successfully for {request.vectorstore}.<<<")
+
+            return response
+        else:
+            response = create_error_response(400,
+                                             "Exceeding limits, please try with a smaller chunks of information or subscribe to our premium plan.")
+            return response
+
+    except Exception as e:
+        logger.error(f">>>Error in add_qa_pair: {e} for {request.vectorstore}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.post("/load_edited_json" )
+async def load_edited_json(request: LoadEditedJson):
+    vectorstore, data_source_name, source_endpoint, json_data = request.vectorstore, request.data_source_name, request.source_endpoint, request.json_data
+    username, chatbot_name = request.vectorstore.split("$")[1], request.vectorstore.split("$")[2]
+
+    logger.info(f">>>loadEditedJson API Triggered By {request.vectorstore}<<<")
+    try:
+        track_usage(supabase_client=supabase_client, vectorstore=request.vectorstore,
+                    endpoint="/load_edited_json")
+        json_data = decode_base64(request.json_data)
+        json_data = json.dumps(json_data, indent=1).encode("utf-8", errors="replace")
+        file_name = user_management_pipeline.create_data_source_name(source_name=data_source_name,
+                                                                     username=username)
+        response = supabase_client.storage.from_("ConversAI").upload(file=json_data, path=f"{file_name}_data.json")
+        response = (
+            supabase_client.table("ConversAI_ChatbotDataSources")
+            .insert({"username": username,
+                     "chatbotName": chatbot_name,
+                     "dataSourceName": file_name,
+                     "sourceEndpoint": source_endpoint,
+                     "sourceContentURL": os.path.join(os.environ["SUPABASE_PUBLIC_BASE_URL"],
+                                                      f"{file_name}_data.json")})
+            .execute()
+        )
+        response = create_success_response(200, {"output": "Successfully loaded the edited json."})
+        logger.info(f">>>Edited json loaded successfully for {vectorstore}.<<<")
+
+        return response
+
+    except Exception as e:
+        logger.error(f">>>Error in loadEditedJson: {e} for {vectorstore}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.get("/list_chatbot_sources" )
+async def list_chatbot_sources(vectorstore: str):
+    try:
+        logger.info(f">>>list_chatbot_sources API Triggered By {vectorstore}<<<")
+
+        track_usage(supabase_client=supabase_client, vectorstore=vectorstore, endpoint="/list_chatbot_sources")
+        username, chatbot_name = vectorstore.split("$")[1], vectorstore.split("$")[2]
+        result = supabase_client.table("ConversAI_ChatbotDataSources").select("*").eq("username", username).eq(
+            "chatbotName",
+            chatbot_name).execute().data
+
+        response = create_success_response(200, {"output": result})
+        logger.info(f">>>Chatbot listed successfully for {vectorstore}.<<<")
+        return response
+
+    except Exception as e:
+        logger.error(f">>>Error in list_chatbot_sources: {e} for {vectorstore}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.get("/get_data_source" )
+async def get_data_source(vectorstore: str, source_url: str):
+    try:
+        logger.info(f">>>get_data_source API Triggered By {vectorstore}<<<")
+
+        track_usage(supabase_client=supabase_client, vectorstore=vectorstore, endpoint="/get_data_source")
+        r = requests.get(source_url)
+        res = encode_to_base64(eval(r.content.decode("utf-8", errors="replace")))
+
+        response = create_success_response(200, {"output": res})
+
+        return response
+
+    except Exception as e:
+        logger.error(f">>>Error in get_data_source: {e} for {vectorstore}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.post("/delete_chatbot_source" )
+async def delete_chatbot_source(request: DeleteChatbotSourceRequest):
+    vectorstore, data_source_name = request.vectorstore, request.data_source_name
+    try:
+
+        track_usage(supabase_client=supabase_client, vectorstore=vectorstore, endpoint="/delete_chatbot_source")
+        response = supabase_client.table("ConversAI_ChatbotDataSources").delete().eq("dataSourceName",
+                                                                                     data_source_name).execute()
+        response = supabase_client.storage.from_('ConversAI').remove(f"{data_source_name}_data.json")
+
+        response = create_success_response(200, {"output": f"Successfully deleted the {data_source_name} data source."})
+
+        logger.info(f">>>Data source deleted successfully for {vectorstore}.<<<")
+        return response
+
+
+    except Exception as e:
+        logger.error(f">>>Error in delete_chatbot_source: {e} for {vectorstore}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.post("/train_chatbot")
+async def train_chatbot(request: TrainChatbotRequest):
+    vectorstore, url_sources = request.vectorstore, request.urls
+    logger.info(f">>>train_chatbot API Triggered By {vectorstore}<<<")
+    try:
+        track_usage(supabase_client=supabase_client, vectorstore=vectorstore, endpoint="/train_chatbot")
+        texts = []
+        sources = []
+        fileTypes = [
+            supabase_client.table("ConversAI_ChatbotDataSources").select("sourceEndpoint").eq("sourceContentURL",
+                                                                                              x).execute().data[0][
+                "sourceEndpoint"] for x in url_sources]
+        for source, fileType in zip(url_sources, fileTypes):
+            if ((fileType == "/text_pdf_extraction") | (fileType == "/image_pdf_text_extraction")):
+                logger.info(f"Source is {source}")
+                r = requests.get(source)
+                file = eval(r.content.decode("utf-8", errors="replace"))
+                content = file["output"]
+                logger.info(f"content is {content}")
+                fileSource = file["source"]
+                texts.append(".".join(
+                    [content[key] for key in content.keys()]).replace(
+                    "\n", " "))
+
+                sources.append(fileSource)
+            elif fileType == "/add_text" or fileType == "/add_qa_pair":
+                r = requests.get(source)
+                file = eval(r.content.decode("utf-8", errors="replace"))
+                content = file["output"]["text"]
+                fileSource = file["source"]
+                texts.append(content.replace("\n", " "))
+                sources.append(fileSource)
+            elif ((fileType == "/fetch_text/urls") | (fileType == "/youtube_transcript")):
+                r = requests.get(source)
+                file = eval(r.content.decode("utf-8", errors="replace"))
+                content = file["output"]
+                fileSource = file["source"]
+                texts.append(".".join(
+                    [content[key] for key in content.keys()]).replace(
+                    "\n", " "))
+                sources.append(fileSource)
+            else:
+                pass
+        texts = [(text, source) for text, source in zip(texts, sources)]
+        conversai_pipeline.add_document_(texts, vectorstore)
+        response = create_success_response(200, {"message": "Chatbot trained successfully."})
+        logger.info(f">>>Chatbot trained successfully for {vectorstore}.<<<")
+
+        return response
+
+
+    except Exception as e:
+        logger.error(f">>>Error in train_chatbot: {e} for {vectorstore}.<<<")
+        raise e
+        # raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.get("/activity_log" )
+async def activity_log(username: str):
+    logger.info(f">>>activityLog API Triggered By {username}<<<")
+    try:
+        response = supabase_client.table("ConversAI_ActivityLog").select("*").eq("username", username).execute().data
+
+        logger.info(f">>>Activity log fetched successfully for {username}.<<<")
+
+        return response
+    except Exception as e:
+        logger.error(f">>>Error in activityLog: {e} for {username}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.post("/new_chatbot" )
+async def new_chatbot(request: NewChatbotRequest):
+    logger.info(f">>> new_chatbot API Triggered <<<")
+    try:
+        response = user_management.new_chatbot_(chatbot_name=request.chatbot_name, username=request.username)
+        logger.info(f">>> Chatbot created successfully for {request.username}.<<<")
+        return response
+
+    except Exception as e:
+        logger.error(f">>>Error in new_chatbot: {e} for {request.username}.<<<")
+        raise_http_exception(500, "Internal Server Error")

src/api/jewel_mirror.py

@@ -0,0 +1,27 @@
+from fastapi.routing import APIRouter
+from fastapi.requests import Request
+from src.jewel_mirror.jewel_langgraph import JewelGraphApp
+from src.utils.error_handling import create_success_response, raise_http_exception
+from src.models.apis_models import JewelQueryRequest
+from src import logging as logger
+jewel_api_router=APIRouter(tags=["jewelchatbot"])
+
+
+@jewel_api_router.post("/answer_jewel_query")
+async def answer_query(request:JewelQueryRequest):
+    logger.info(f">>>answer_jewel_query API Triggered <<<")
+    try:
+
+        jewel_app = JewelGraphApp(request.vectorstore)
+        output, followup_questions= jewel_app.get_response(query=request.query)
+
+
+        response = create_success_response(200, data={"output": output, "follow_up_questions": followup_questions,
+                                                      "source": None})
+        logger.info(f">>>Query answered successfully for .<<<")
+        return response
+
+    except Exception as e:
+        logger.error(f">>>Error in answer_query: {e} for {request.vectorstore}.<<<")
+        # raise HTTPException(status_code=500, detail="Internal Server Error")
+        raise e

src/api/jwt_bearer.py

@@ -0,0 +1,29 @@
+"""
+Created By: ishwor subedi
+Date: 2024-09-03
+"""
+import os
+from fastapi import Depends
+from supabase import create_client
+from src import logging as logger
+from src.utils.error_handling import create_error_response, raise_http_exception
+from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
+
+security = HTTPBearer()
+
+supabase_client = create_client(
+    os.getenv("SUPABASE_URL"),
+    os.getenv("SUPABASE_KEY")
+)
+
+
+async def access_check_bearer(credentials: HTTPAuthorizationCredentials = Depends(security)):
+    access_token = credentials.credentials
+    try:
+        supabase_client.auth.get_user(access_token)
+
+    except Exception as e:
+        logger.info(f">>> Invalid access token {e}<<<")
+        raise_http_exception(code=401,
+                                   message="Invalid Access Token", details=[
+                {"info": "Invalid access token or access token expired please login again"}])

src/api/speech_api.py

@@ -0,0 +1,85 @@
+"""
+Created By: ishwor subedi
+Date: 2024-07-31
+"""
+import os
+import tempfile
+from fastapi import Form
+from fastapi import UploadFile, HTTPException, status
+from src.models.apis_models import TextToSpeechRequest
+from fastapi.routing import APIRouter
+from src.pipeline.speech_transcription_pipeline import SpeechTranscriptionPipeline
+from src import logging as logger
+from src.utils.error_handling import create_success_response, raise_http_exception
+
+speech_translator_router = APIRouter(tags=["SpeechTranscription"])
+pipeline = SpeechTranscriptionPipeline()
+
+
+@speech_translator_router.post(
+    "/text_to_speech",
+)
+async def text_to_speech(request: TextToSpeechRequest):
+    logger.info(f">>>text_to_speech API Triggered <<<")
+    try:
+        audio_bytes = pipeline.text_to_speech(request.text, request.lang, request.tld)
+        if not audio_bytes:
+            raise ValueError("Audio generation failed.")
+        response = create_success_response(code=200, data={"audio": audio_bytes})
+        logger.info(f">>>text_to_speech API success <<<")
+        return response
+    except ValueError as ve:
+        logger.info(f">>>text_to_speech API failed {ve}<<<")
+        raise_http_exception(code=400, message="Text to speech failed")
+
+    except Exception as e:
+        logger.error(f">>> Error processing text-to-speech {e}<<<")
+        raise_http_exception(code=500, message="Internal server error")
+
+
+@speech_translator_router.post(
+    "/speech_to_text",
+
+)
+async def speech_to_text(audio: UploadFile, lang: str = Form(...)):
+    logger.info(f">>>speech_to_text API Triggered <<<")
+    try:
+        audio_bytes = await audio.read()
+        if not audio_bytes:
+            logger.error(f">>> Empty audio file <<<")
+            raise ValueError("Empty audio file")
+    except Exception as e:
+        logger.error(f">>> Invalid audio file {e}<<<")
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="Invalid audio file"
+        )
+
+    try:
+        with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as temp_audio_file:
+            temp_audio_file.write(audio_bytes)
+            temp_audio_file_path = temp_audio_file.name
+    except Exception as e:
+        logger.error(f">>> Error creating temporary file{e} <<<")
+        raise_http_exception(code=500, message="Internal server error")
+
+    try:
+        transcript = pipeline.speech_to_text(temp_audio_file_path, lang)
+        response = create_success_response(code=200, data={"transcript": transcript})
+        logger.info(f">>>speech_to_text API success <<<")
+
+        return response
+
+    except FileNotFoundError:
+        logger.error(f">>> Temporary file not found <<<")
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail="Temporary file not found"
+        )
+    except Exception as e:
+        logger.error(f">>> Error processing speech-to-text {e}<<<")
+        raise_http_exception(code=500, message="Error processing speech-to-text")
+
+    finally:
+        if os.path.exists(temp_audio_file_path):
+            os.remove(temp_audio_file_path)

src/api/user_management_api.py

@@ -0,0 +1,157 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""
+from fastapi import Depends
+from src import logging as logger
+from src.models.apis_models import *
+from fastapi.routing import APIRouter
+from src.api.jwt_bearer import access_check_bearer, supabase_client
+from src.pipeline.user_management_pipeline import SupabaseUserManagementPipeline
+from src.services.supabase.analytics.analytic_tables import track_usage
+from src.utils.error_handling import raise_http_exception, create_success_response, create_error_response
+
+user_management_api_router = APIRouter(tags=["User Management"])
+
+user_management = SupabaseUserManagementPipeline()
+
+
+@user_management_api_router.post("/user_signup")
+async def user_signup(request: UserSignupRequest):
+    logger.info(f">>>user_signup API Triggered <<<")
+    response = user_management.user_signup_(username=request.username, email=request.email, password=request.password)
+    logger.info(f">>>user_signup API Success<<<")
+
+    return response
+
+
+@user_management_api_router.post("/user_signin")
+async def user_signin(request: UserSigninRequest):
+    logger.info(f">>>user_signin API Triggered <<<")
+
+    response = user_management.user_signin_(email=request.email, password=request.password)
+    if response != None:
+        logger.info(f">>>user_signin API Success.<<<")
+        return response
+    else:
+        logger.info(f">>> Email or password is incorrect please try again.<<<")
+        response = create_error_response(400, "Email or password is incorrect please try again.")
+        return response
+
+
+@user_management_api_router.post("/get_user_data")
+async def get_user_data(request: GetUserDataRequest):
+    logger.info(f">>>get_user_data API Triggered <<<")
+    response = user_management.get_user_data_(access_token=request.access_token)
+    return response
+
+
+@user_management_api_router.post("/login_with_access_token")
+async def login_with_access_token(request: LoginWithAccessTokenRequest):
+    logger.info(f">>>login_with_access_token API Triggered <<<")
+
+    response = user_management.login_with_access_token_(access_token=request.access_token,
+                                                        refresh_token=request.refresh_token)
+    logger.info(f">>>login_with_access_token API Success<<<")
+    return response
+
+
+@user_management_api_router.post("/set_session_data")
+async def set_session_data(request: SetSessionDataRequest):
+    logger.info(f">>> set_session_data API Triggered <<<")
+
+    response = user_management.set_session_data_(access_token=request.access_token, refresh_token=request.refresh_token,
+                                                 user_id=request.user_id)
+    return response
+
+
+@user_management_api_router.post("/sign_out")
+async def sign_out():
+    logger.info(f">>> sign_out API Triggered <<<")
+
+    response = user_management.sign_out_()
+    logger.info(f">>>sign_out API Success<<<")
+    return response
+
+
+@user_management_api_router.post("/oauth_signin")
+async def oauth_signin():
+    logger.info(f">>> oauth_signin API Triggered <<<")
+    response = user_management.oauth_signin_()
+    logger.info(f">>>oauth_signin API Success<<<")
+    return response
+
+
+@user_management_api_router.post("/check_session")
+async def check_session():
+    logger.info(f">>>check_session API Triggered <<<")
+
+    response = user_management.check_session_()
+    return response
+
+
+@user_management_api_router.get("/get_public_chatbot")
+async def get_public_chatbots():
+    logger.info(f">>>get_public_chatbot API Triggered<<<")
+    try:
+        response = supabase_client.table("ConversAI_ChatbotInfo").select("*").eq("isPrivate", False).execute().data
+        logger.info(f">>>Public chatbots fetched successfully.<<<")
+        return response
+    except Exception as e:
+        logger.error(f">>>Error in get_public_chatbot: {e}<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@user_management_api_router.post("/public_private_check")
+async def public_or_private(request: PublicPrivateCheckRequest):
+    vectorstore, mode = request.vectorstore, request.mode
+    logger.info(f">>>public_private_check API Triggered for {vectorstore}.<<<")
+    try:
+        track_usage(supabase_client=supabase_client, vectorstore=vectorstore, endpoint="/public_private_check")
+        username, chatbot_name = vectorstore.split("$")[1], vectorstore.split("$")[2]
+        if len(mode) == 0:
+            value = (
+                supabase_client.table("ConversAI_ChatbotInfo")
+                .select("isPrivate")
+                .eq("user_id", username)
+                .eq("chatbotname", chatbot_name)
+                .execute()
+            )
+            value = value.data[0]["isPrivate"]
+            response = create_success_response(200, {"output": value})
+        else:
+            response = (
+                supabase_client.table("ConversAI_ChatbotInfo")
+                .update({"isPrivate": mode})
+                .eq("user_id", username)
+                .eq("chatbotname", chatbot_name)
+                .execute()
+            )
+            response = create_success_response(200, {"output": response})
+        logger.info(f">>>Public/Private check successful for {vectorstore}.<<<")
+        return response
+
+
+    except Exception as e:
+        logger.error(f">>>Error in public_private_check: {e} for {vectorstore}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@user_management_api_router.post("/refresh_session", dependencies=[Depends(access_check_bearer)])
+async def refresh_session(request: RefreshSessionRequest):
+    logger.info(f">>>refresh_session API Triggered <<<")
+    response = user_management.refresh_session__(refresh_token=request.refresh_token)
+    logger.info(f">>>refresh token fetched successfully.<<<")
+
+    return response
+
+
+@user_management_api_router.post("/username_creation_oauth", dependencies=[Depends(access_check_bearer)])
+async def username_creation_oauth(request: UsernameCreationOauthRequest):
+    logger.info(f">>> username_creation_oauth API Triggered <<<")
+
+    response = user_management.username_creation_oauth_(username=request.username, user_id=request.user_id,
+                                                        email=request.email)
+
+    logger.info(f">>>username creation successful.<<<")
+    return response

src/jewel_mirror/jewel_langgraph.py

@@ -0,0 +1,69 @@
+from langgraph.graph import StateGraph,END
+from src.jewel_mirror.jewel_mirror import *
+
+
+
+class JewelGraphApp:
+    def __init__(self,vectorstore):
+        self.jewelgraph = StateGraph(AgentState)
+        self.vectorstore=vectorstore
+        self.jewel_answer = jewel_answer(self.vectorstore)
+        self._setup_graph()
+
+
+    def get_response(self, query):
+        """Get response for a given query."""
+
+        response= self.jewelapp.invoke({"messages": [query]})["messages"][-1]
+        followups = self.jewel_answer.generate_vectorstore_followups(question=query,answer=response)
+        return response, followups
+
+    def _setup_graph(self):
+        # Add nodes
+
+        self.jewelgraph.add_node("firstagent", self.jewel_answer.function_1)
+        self.jewelgraph.add_node("nocontext", self.jewel_answer.context_end)
+        self.jewelgraph.add_node("secondagent", self.jewel_answer.function_2)
+        self.jewelgraph.add_node("JewelRag", self.jewel_answer.calling_jewel_rag)
+        self.jewelgraph.add_node("Goldprice", self.jewel_answer.calling_gold_price)
+        self.jewelgraph.add_node("JewelLLM", self.jewel_answer.calling_llm)
+
+        # Set entry point
+        self.jewelgraph.set_entry_point("firstagent")
+
+        # Add conditional edges
+        self.jewelgraph.add_conditional_edges(
+            "firstagent",
+            self.jewel_answer.router, {
+                "secondagent": "secondagent",
+                "nocontext": "nocontext",
+            }
+        )
+        self.jewelgraph.add_edge("nocontext", END)
+
+        self.jewelgraph.add_conditional_edges(
+            "secondagent",
+            self.jewel_answer.router2, {
+                "JewelRag": "JewelRag",
+                "Goldprice": "Goldprice",
+                "JewelLLM": "JewelLLM",
+            }
+        )
+
+        # Add end edges
+        self.jewelgraph.add_edge("JewelRag", END)
+        self.jewelgraph.add_edge("Goldprice", END)
+        self.jewelgraph.add_edge("JewelLLM", END)
+
+        # Compile graph
+        self.jewelapp = self.jewelgraph.compile()
+
+
+
+
+if __name__ == "__main__":
+    jewel_app = JewelGraphApp()
+    query = "What is the price of a Gold in india?"
+    response = jewel_app.get_response(query)
+    print(response)
+

src/jewel_mirror/jewel_mirror.py

@@ -0,0 +1,387 @@
+
+#Todo: To create a class based for this
+from typing import TypedDict, Annotated, Sequence
+import operator
+from langchain_core.messages import BaseMessage
+from langchain.prompts import PromptTemplate
+from langchain.output_parsers import PydanticOutputParser
+from src.models.utls import TopicSelectionParser
+from langchain_groq import ChatGroq
+from langchain_community.document_compressors import JinaRerank
+from langchain.retrievers import ContextualCompressionRetriever
+from src.utils.utils import json_parser
+import os
+from src.services.embeddings.jina_embeddings import jina_embedding
+from src.services.embeddings.Qdrant_BM25_embedding import qdrant_bm25_embedding
+from src.services.vector_db.qdrent.upload_document import  answer_query_from_existing_collection
+from langchain.prompts import  ChatPromptTemplate
+from langchain_core.output_parsers import StrOutputParser
+from langchain_core.runnables import RunnablePassthrough
+from langchain_community.document_loaders import WebBaseLoader
+from langchain.schema import Document
+os.environ["JINA_API_KEY"] = os.getenv("JINA_API")
+llm=ChatGroq(model_name="llama-3.1-70b-versatile")
+from src import logging as logger
+
+
+
+class AgentState(TypedDict):
+    # The 'messages' field should be a sequence of strings, and we annotate it with 'operator.add'
+    # This implies we might want to "add" new messages to the sequence later
+    messages: Annotated[Sequence[BaseMessage], operator.add]
+
+parser = PydanticOutputParser(pydantic_object=TopicSelectionParser)
+
+
+class jewel_answer:
+    def __init__(self, vector_store):
+        self.vector_store = vector_store
+        self.compressor = JinaRerank(model="jina-reranker-v2-base-multilingual")
+        self.vector_embed = jina_embedding()
+        self.sparse_embed = qdrant_bm25_embedding()
+        self.llm = ChatGroq(model_name="llama-3.1-70b-versatile")
+        self.parser = PydanticOutputParser(pydantic_object=TopicSelectionParser)
+        self.json_parser=json_parser()
+
+# Function to generate  a two output [About jewellery and None]
+    def function_1(self,state):
+        message = state["messages"]
+        question = message[-1]
+        template = """
+    Your task is to classify the given user query into one of the following categories [About Jewellery,None]:
+    - **About Jewellery**: Includes queries about:
+    - Jewelry-related topics, such as specific types, prices, or customization.
+    - JewelMirror's purpose, key features, and its functionalities (e.g., virtual try-on, batch selection, or sharing looks).
+    - FAQs related to JewelMirror (e.g., signing up, logging in, troubleshooting, or compatible devices).
+    - JewelMirror chatbot functionalities, such as assisting users with jewelry-related queries.
+    - **None**: Includes queries unrelated to jewelry, JewelMirror, or its functionalities (e.g., programming questions, general greetings without context, or unrelated topics like baking).
+
+    Only respond with the category name and nothing else.
+
+    Examples:
+
+    User query: "Can you explain how to use JewelMirror’s virtual try-on feature?"
+    Output: About Jewellery
+
+    User query: "Hi, how can I sign up for JewelMirror?"
+    Output: About Jewellery
+
+    User query: "What devices are compatible with the JewelMirror app?"
+    Output: About Jewellery
+
+    User query: "Hi, can you help me with Python programming?"
+    Output: None
+
+    User query: "How does the JewelMirror chatbot assist with jewelry questions?"
+    Output: About Jewellery
+
+
+    User query: "What is the best way to bake a cake?"
+    Output: None
+
+    User query: "Hello, what is the purpose of JewelMirror?"
+    Output: About Jewellery
+
+    User query: "Can JewelMirror help me share looks on social media?"
+    Output: About Jewellery
+
+
+    User query: "Can I customize my saree look with JewelMirror?"
+    Output: About Jewellery
+
+    User query: {question}
+    {format_instructions}
+    Output:
+    """
+
+        prompt = PromptTemplate(template=template,
+                                input_variables=[question],
+                                partial_variables={
+                                    "format_instructions": parser.get_format_instructions()}
+                                )
+        chain = prompt | llm | parser
+
+        response = chain.invoke({"question": question, "format_instructions": parser.get_format_instructions()})
+
+        return {"messages": [response.Topic]}
+
+    # Function to handle NONE condition
+    def context_end(self,state):
+        result="Please ask about the jewellery related queries.Thank you !"
+        return  {"messages": [result]}
+
+
+    #router to route the aboutjewellery and none
+    def router(self,state):
+        messages = state["messages"]
+        last_message = messages[-1]
+        print(last_message)
+        if 'About Jewellery' in last_message:
+            return 'secondagent'
+        else:
+            return 'nocontext'
+
+    #function to generate three output Rag,llm,price
+    def function_2(self,state):
+        messages = state['messages']
+        question = messages[0]
+        template = """Your task is to classify the given user query into one of the following categories [RAG,LLM,Price]:
+
+    `- **Gold Price**: If the query strictly asks about the price of ornaments -related jewelry. For example:
+    - "What is the price of a gold today?"
+    - "What is the price of a silver today?"
+    - "What is the price of gold in chennai for last 10days?"
+
+
+    - "How much does 24K gold cost?"
+
+    - **RAG**: If the query relates to JewelMirror’s features, operations, or FAQs, including:
+    - JewelMirror’s **chatbot** features, such as virtual try-on, batch selection, or sharing looks jewel mirro name can be anything jewelmirror jewelchatbot anything.
+    - JewelMirror’s **purpose**, **key features**, or how it enhances the shopping experience.
+    - User guides, such as signing up, logging in, managing the cart, or using the virtual try-on feature.
+    - Customizing jewelry or sarees, using batch selection, or downloading/sharing customized looks.
+    - FAQs about JewelMirror, including how it works, security of personal data, compatible devices, or troubleshooting.
+    - Using the JewelMirror chatbot for assistance with jewelry-related tasks.
+    - Anything operational, feature-specific, or directly related to JewelMirror’s functionality.
+
+    - **LLM**: If the query is a **generic, broader jewelry-related question** that does not specifically relate to JewelMirror features or price. For example:
+    - General differences between gold and diamonds.
+    - Why certain materials, like diamonds or gold, are used in jewelry.
+    - Types of jewelry, history of jewelry, or materials used.
+    - Examples:
+        - "What is the difference between gold and diamonds?"
+        - "Why are diamonds used in engagement rings?"
+        - "What are the most popular types of necklaces?"
+
+    **Key Rule**: Queries must strictly match the criteria for **Gold Price** or **RAG** to be classified under those categories. Only if the query does not match either, and is a generic jewelry-related question, classify it as **LLM**.
+
+    Only respond with the category name and nothing else.
+
+    Examples:
+
+    User query: "What is the price of a gold necklace?"
+    Output: Gold Price
+
+    User query: "How do I sign up for JewelMirror?"
+    Output: RAG
+
+    User query: "What are the key features of JewelMirror?"
+    Output: RAG
+
+    User query: "How can I use the virtual try-on feature?"
+    Output: RAG
+
+    User query: "What is the price of diamond rings?"
+    Output: Gold Price
+
+    User query: "How do I use batch selection to try different combinations?"
+    Output: RAG
+
+    User query: "What types of jewelry are available in JewelMirror?"
+    Output: RAG
+
+    User query: "How does the JewelMirror chatbot assist with jewelry shopping?"
+    Output: RAG
+
+    User query: "What is the difference between 18K and 24K gold?"
+    Output: LLM
+
+    User query: "Why are diamonds used in jewelry?"
+    Output: LLM
+
+    User query: "How does JewelMirror enhance the customer shopping experience?"
+    Output: RAG
+
+    User query: "Can JewelMirror handle batch selection for necklaces?"
+    Output: RAG
+
+    User query: "How do I troubleshoot the JewelMirror app if the virtual try-on doesn’t work?"
+    Output: RAG
+
+
+    User query: {question}
+    {format_instructions}
+    Output:
+    """
+        prompt = PromptTemplate(template=template,
+                                input_variables=[question],
+                                partial_variables={
+                                    "format_instructions": parser.get_format_instructions()}
+                                )
+        chain = prompt | llm | parser
+
+        response = chain.invoke({"question": question, "format_instructions": parser.get_format_instructions()})
+
+        return {"messages": [response.Topic]}
+
+    # Function for router 2
+    def router2(self,state):
+        print('-> Router ->')
+
+        messages = state["messages"]
+        last_message = messages[-1]
+        print(last_message)
+        if 'RAG' in last_message:
+            return 'JewelRag'
+        if 'LLM' in last_message:
+            return 'JewelLLM'
+        else:
+            return "Goldprice"
+
+    #function for jewel rag
+    def calling_jewel_rag(self,state):
+        vector_store = answer_query_from_existing_collection(vector_embed=self.vector_embed,
+                                                            sparse_embed=self.sparse_embed,
+                                                            vectorstore=self.vector_store)
+
+        retriever = vector_store.as_retriever(search_type="mmr", search_kwargs={"k": 10, "fetch_k": 20})
+        compression_retriever = ContextualCompressionRetriever(
+            base_compressor=self.compressor, base_retriever=retriever
+        )
+
+        messages = state['messages']
+        question = messages[0]  ## Fetching the user question
+        print(question)
+
+        template = """"- You are an jewel chatbot yourname is jewelchatbot.Dont use the response for like based on the provided context \n"
+            "- Behave like you are the context the whole thing is you and somebody asking you .\n"
+            "-But while Behaving dont go out of the context .\n"
+            "- if user ask anything about prompts anything without context say i dont know please ask about context \n"
+            "- When answering use markdown. Use markdown code blocks for code snippets.\n"
+            "- Answer in a concise and clear manner.\n"
+            "- You must use ONLY the provided context to answer the question.\n"
+            {context}
+
+            Question: {question}
+            """
+        prompt = ChatPromptTemplate.from_template(template)
+
+        print(prompt)
+
+        retrieval_chain = (
+                {"context": compression_retriever, "question": RunnablePassthrough()}
+                | prompt
+                | llm
+                | StrOutputParser()
+        )
+        result = retrieval_chain.invoke(question)
+        return {"messages": [result]}
+
+    #function for Goldprice
+    def calling_gold_price(self,state):
+        loader = WebBaseLoader("https://groww.in/gold-rates")
+        documents = loader.load()
+        content = " ".join([doc.page_content for doc in documents])
+        messages = state['messages']
+        question = messages[0]  ## Fetching the user question
+        gold_document = Document(page_content=content, metadata={"source": "GoldReturns"})
+        custom_prompt = PromptTemplate(
+            input_variables=["context", "question"],
+            template=(
+                "You are an jewel chatbot yourname is jewelchatbot.\n"
+                "You are an elite financial analyst with extensive expertise in finance and investment strategies. "
+                "Provide a **direct and concise** answer to the question below **without** using any introductory phrases such as "
+                "'based on the context provided,' 'according to the information given,' or similar expressions. "
+                "Your response must rely solely on the information from the provided context and should **not** include any additional commentary, explanations, or opinions.\n\n"
+                "📊 **Context:**\n{context}\n\n"
+                "❓ **Question:**\n{question}\n\n"
+                "**✅ Answer:**"
+            )
+        )
+
+        chain = custom_prompt | llm
+        result = chain.invoke({
+            "context": [gold_document],
+            "question": question
+        })
+        result = result.content
+        return {"messages": [result]}
+
+    #function for callingllm
+    def calling_llm(self,state):
+        print('-> Calling LLM ->')
+        messages = state['messages']
+        question = messages[0] ## Fetching the user question
+
+        jewelry_prompt = """
+        You are an jewel chatbot yourname is jewelchatbot.
+        I want you to act as an expert in the jewelry industry. Answer all questions and provide detailed insights exclusively related to the jewelry domain. This includes, but is not limited to:
+
+        - Manufacturing processes (e.g., materials like gold, silver, diamonds, gemstones, tools, and techniques such as casting, engraving, or 3D printing).
+        - Design trends (e.g., contemporary, vintage, or cultural styles).
+        - Marketing strategies (e.g., branding, customer engagement, sustainability).
+        - Supply chain dynamics (e.g., sourcing of raw materials, ethical considerations, and global trade).
+        - History and cultural significance (e.g., origins of certain jewelry styles, symbolic meanings in different cultures).
+        - Retail and customer behavior (e.g., pricing, customization trends, online vs. offline shopping).
+        - Sustainability in jewelry (e.g., ethical sourcing, lab-grown diamonds, recycling of materials).
+
+        You must limit your responses strictly to the jewelry industry and not discuss unrelated topics. Provide examples, actionable insights, and in-depth details whenever possible.
+        {question}
+
+            """
+        prompt = ChatPromptTemplate.from_template(jewelry_prompt)
+
+        retrieval_chain = (
+            prompt
+            | llm
+            | StrOutputParser()
+            )
+        result = retrieval_chain.invoke({"question": question})
+        return  {"messages": [result]}
+
+    def generate_vectorstore_followups(self, question: str, answer: str|None=None) -> str:
+        """
+        Generate follow-up questions using context from the vector store.
+        """
+
+        vector_store = answer_query_from_existing_collection(vector_embed=self.vector_embed,
+                                                             sparse_embed=self.sparse_embed,
+                                                             vectorstore=self.vector_store)
+
+        retriever = vector_store.as_retriever(search_type="mmr", search_kwargs={"k": 10, "fetch_k": 20})
+        compression_retriever = ContextualCompressionRetriever(
+            base_compressor=self.compressor, base_retriever=retriever
+        )
+
+        # Get relevant documents
+        relevant_docs = compression_retriever.get_relevant_documents(question)
+        context = "\n".join([doc.page_content for doc in relevant_docs])
+
+        followup_template = """You are jewelchatbot, an expert jewelry assistant. Generate follow-up questions based on the 
+        previous interaction and the provided context from our knowledge base.
+
+        Previous Question: {question}
+        Previous Answer: {answer}
+        Knowledge Base Context: {context}
+
+        Rules for generating questions:
+        1. Questions should be based on information present in the knowledge base context
+        2. Focus on aspects mentioned in the context that weren't covered in the previous answer
+        3. Questions should explore JewelMirror features or jewelry topics found in the context
+        4. Make sure questions are specific and can be answered using our knowledge base
+        5. Avoid questions about topics not present in the context
+        6. Don't repeat information already covered in the previous answer
+
+        Generate exactly 3 relevant follow-up questions.only in json format dont use here are and all . just generate questions only.
+        "{format_instructions}\n"
+        """
+
+        follow_up_prompt = PromptTemplate(
+            template=followup_template,
+            input_variables=["context", "answer","question"],
+            partial_variables={"format_instructions": self.json_parser.get_format_instructions()},
+        )
+        followup_chain = (
+                follow_up_prompt
+                | llm
+                | json_parser()
+        )
+
+        followups = followup_chain.invoke({
+            "question": question,
+            "answer": answer,
+            "context": context
+        })
+
+        return followups
+

src/models/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-07-31
+"""

src/models/apis_models.py

@@ -0,0 +1,208 @@
+"""
+Created By: ishwor subedi
+Date: 2024-07-31
+"""
+from pydantic import BaseModel, EmailStr
+from typing import List, Optional
+
+
+## ---------------------------------- SpeechTranscription API Models ----------------------------------
+class TextToSpeechRequest(BaseModel):
+    text: str
+    lang: str
+    tld: str
+
+
+class SpeechToTextRequest(BaseModel):
+    lang: str
+
+
+## ---------------------------------- Chatbot API Models ----------------------------------
+
+class AddTextRequest(BaseModel):
+    vectorstore: str
+    text: str
+
+
+class AddWebsiteRequest(BaseModel):
+    website_urls: List[str]
+    vectorstore: str
+    source: str
+
+
+class AnswerQueryRequest(BaseModel):
+    query: str
+    vectorstore: str
+    llm_model: str = "llama3-70b-8192"
+
+
+class DataAnalyzerRequest(BaseModel):
+    query: str
+
+
+class GetLinksRequest(BaseModel):
+    url: str
+
+
+class YoutubeTranscriptRequest(BaseModel):
+    vectorstore: str
+    urls: List[str]
+
+
+class WebsiteUrlTextExtractionRequest(BaseModel):
+    url: str
+
+
+class WebsiteUrlTextExtractionListRequest(BaseModel):
+    urls: List[str]
+
+
+class GetCurrentCountRequest(BaseModel):
+    vectorstore: str
+
+
+class ListChatbotsRequest(BaseModel):
+    username: str
+
+
+class GetChatHistoryRequest(BaseModel):
+    vectorstore: str
+
+
+class ChatHistoryItem(BaseModel):
+    timestamp: str
+    question: str
+    response: str
+
+
+class DeleteChatbotRequest(BaseModel):
+    vectorstore: str
+
+
+class AddQAPairRequest(BaseModel):
+    vectorstore: str
+    question: str
+    answer: str
+
+
+class TrainChatbotRequest(BaseModel):
+    vectorstore: str
+    urls: list[str]
+
+
+class LoadPDFRequest(BaseModel):
+    vectorstore: str
+
+
+class LoadEditedJson(BaseModel):
+    vectorstore: str
+    data_source_name: str
+    source_endpoint: str
+    json_data: dict
+
+
+class PublicPrivateCheckRequest(BaseModel):
+    vectorstore: str
+    mode: str | None = None
+
+
+class DeleteChatbotSourceRequest(BaseModel):
+    vectorstore: str
+    data_source_name: str
+
+
+## ---------------------------------- User Management API Models ----------------------------------
+
+class UserSignupRequest(BaseModel):
+    username: str
+    email: EmailStr
+    password: str
+
+
+class UserSigninRequest(BaseModel):
+    email: EmailStr
+    password: str
+
+
+class CheckSessionRequest(BaseModel):
+    user_id: str
+
+
+class GetUserDataRequest(BaseModel):
+    access_token: str
+
+
+class RefreshSessionRequest(BaseModel):
+    refresh_token: str
+
+
+class LoginWithAccessTokenRequest(BaseModel):
+    access_token: str
+    refresh_token: str
+
+
+class UsernameCreationOauthRequest(BaseModel):
+    username: str
+    user_id: str
+    email: str
+
+
+class SetSessionDataRequest(BaseModel):
+    access_token: str
+    refresh_token: str
+    user_id: str
+
+
+class SignOutRequest(BaseModel):
+    user_id: str
+
+
+class NewChatbotRequest(BaseModel):
+    chatbot_name: str
+    username: str
+
+
+## ---------------------------------- Analytics API Models ----------------------------------
+
+
+class FeedbackRequest(BaseModel):
+    feedback: str
+    user_id: str
+    vectorstore: Optional[str]
+
+
+class UserSatisfactionRateRequest(BaseModel):
+    start_date: Optional[str]
+    end_date: Optional[str]
+    vectorstore: Optional[str]
+
+
+class TokenUsageRequest(BaseModel):
+    start_date: Optional[str]
+    end_date: Optional[str]
+    vectorstore: Optional[str]
+
+
+class AverageSessionInteractionRequest(BaseModel):
+    start_date: Optional[str]
+    end_date: Optional[str]
+    vectorstore: Optional[str]
+
+
+class DailyActiveEndUserRequest(BaseModel):
+    start_date: Optional[str]
+    end_date: Optional[str]
+    vectorstore: Optional[str]
+
+
+class DailyChatCountRequest(BaseModel):
+    start_date: Optional[str]
+    end_date: Optional[str]
+    vectorstore: Optional[str]
+
+##---------------------GOldPr---------------------------
+
+class JewelQueryRequest(BaseModel):
+    query:str
+    vectorstore: str
+

src/models/response_handling_models.py

@@ -0,0 +1,27 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-28
+"""
+from typing import Any, Dict
+
+from pydantic import BaseModel
+
+
+## ---------------------------------- Response Handling API Models ----------------------------------
+class SuccessResponse(BaseModel):
+    status: str
+    code: int
+    data: Dict[str, Any]
+
+
+class ErrorResponse(BaseModel):
+    status: str
+    code: int
+    error: Dict[str, Any]
+
+
+class SuccessResponseUsermanagement(BaseModel):
+    status: str
+    code: int
+    message: str
+    data: Dict[str, Any]

src/models/utls.py

@@ -0,0 +1,17 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-29
+"""
+from pydantic import BaseModel, Field
+
+
+class FollowUps(BaseModel):
+    q1: str = Field(description="First Follow-up Question")
+    q2: str = Field(description="Second Follow-up Question")
+    q3: str = Field(description="Third Follow-up Question")
+
+
+## langgraph models
+class TopicSelectionParser(BaseModel):
+    Topic: str = Field(description='Selected Topic')
+    Reasoning: str = Field(description='Reasoning behind topic selection')

src/pipeline/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-07-31
+"""

src/pipeline/conversai_analytic_pipeline.py

@@ -0,0 +1,34 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-28
+"""
+import requests
+
+from src.api.conversai_api import supabase_client
+from src.services.supabase.analytics.analytic_tables import feedback_table, chat_history_table, add_feedback, \
+    track_usage
+
+
+class ConversAIAnalyticPipeline:
+    def __init__(self):
+        self.supabase_client = supabase_client
+
+    def feedback_table_(self, vectorstore):
+        return feedback_table(self.supabase_client, vectorstore)
+
+    def chat_history_table_(self, vectorstore):
+        return chat_history_table(self.supabase_client, vectorstore)
+
+    def add_feedback_(self, feedback, user_id, city, client_ip, vectorstore):
+        return add_feedback(self.supabase_client, feedback, user_id, city, client_ip, vectorstore)
+
+    def track_usage_(self, vectorstore: str, endpoint: str):
+        return track_usage(supabase_client=self.supabase_client, vectorstore=vectorstore, endpoint=endpoint)
+
+    def get_ip_info(self, ip: str):
+        try:
+            response = requests.get(f"https://ipinfo.io/{ip}/json")
+            data = response.json()
+            return data.get("city", "Unknown")
+        except Exception as e:
+            return "Unknown"

src/pipeline/conversai_pipeline.py

@@ -0,0 +1,76 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-02
+"""
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_core.prompts import PromptTemplate
+from src.services.embeddings.Qdrant_BM25_embedding import qdrant_bm25_embedding
+from src.services.document.add_document import AddDocument
+from src.services.answer_query.answerquery import AnswerQuery
+from src.services.file_analyzer.data_analyzer import DataAnalyzer
+from src.services.get_links.web_scraper import WebScraper
+# from src.services.ocr.easy_ocr.easy_ocr_ import EasyOCR_ as OCRService
+from src.services.pdf_extraction.image_pdf.image_pdf_text_extraction import get_text_from_image_pdf
+from src.services.pdf_extraction.text_pdf.text_pdf_extraction import extract_text_from_pdf
+from src.services.video_transcription.youtube_video_transcription.youtube_video_transcript import \
+    youtube_video_transcription
+# from src.services.website_url.text_extraction_urls import extract_text_from_url_list, extract_text_from_url
+from src.utils.utils import json_parser
+from src.prompts.custom_prompts import  _custom_prompts
+from src.services.ocr.replicate_ocr.replicate_ocr import ReplicateOCR as OCRService
+from src.services.embeddings.jina_embeddings import jina_embedding
+from src.services.website_url.text_extraction_urlsnew import  WebScrapertext
+
+
+class ConversAIPipeline:
+    def __init__(self):
+        prompt_template=_custom_prompts["RAG_ANSWER_PROMPT"]
+        follow_up_prompt_template = _custom_prompts["FOLLOW_UP_PROMPT"]
+        prompt = ChatPromptTemplate.from_template(prompt_template)
+        json_parser_ = json_parser()
+        follow_up_prompt = PromptTemplate(
+            template=follow_up_prompt_template,
+            input_variables=["context"],
+            partial_variables={"format_instructions": json_parser_.get_format_instructions()},
+        )
+        self.vector_embedding = jina_embedding()
+        self.sparse_embedding = qdrant_bm25_embedding()
+        self.add_document_service = AddDocument(self.vector_embedding, self.sparse_embedding)
+
+        self.answer_query_service = AnswerQuery(vector_embedding=self.vector_embedding,
+                                                                 sparse_embedding=self.sparse_embedding, prompt=prompt,
+                                                                          follow_up_prompt=follow_up_prompt, json_parser=json_parser_)
+        self.data_analyzer = DataAnalyzer()
+        self.get_website_links = WebScraper()
+        self.ocr_service = OCRService()
+        self.web_text_extractor=WebScrapertext()
+
+    def add_document_(self, texts: list[tuple[str]], vectorstore: str):
+        return self.add_document_service.add_documents(texts=texts, vectorstore=vectorstore)
+
+    def answer_query_(self, query: str, vectorstore: str, llm_model: str = "llama-3.1-70b-versatile"):
+        output, follow_up_questions, source = self.answer_query_service.answer_query(query=query,
+                                                                                     vectorstore=vectorstore,
+                                                                                     llmModel=llm_model)
+        return output, follow_up_questions, source
+
+    def data_analyzer_(self, query: str, dataframe):
+        return self.data_analyzer.analyze_data(query=query, dataframe=dataframe)
+
+    def get_links_(self, url: str, timeout: int):
+        return self.get_website_links.get_links(url=url, timeout=timeout)
+
+    def image_pdf_text_extraction_(self, image_pdf: bytes):
+        return get_text_from_image_pdf(pdf_bytes=image_pdf)
+
+    def text_pdf_extraction_(self, pdf: str):
+        return extract_text_from_pdf(pdf_path=pdf)
+
+    def youtube_transcript_(self, url: list):
+        return youtube_video_transcription(youtube_video_url=url)
+
+    def website_url_text_extraction_(self, url: str):
+        return self.web_text_extractor.extract_text_from_url(url=url)
+
+    def website_url_text_extraction_list_(self, urls: list):
+        return self.web_text_extractor.extract_text_from_urls(urls=urls)

src/pipeline/speech_transcription_pipeline.py

@@ -0,0 +1,20 @@
+"""
+Created By: ishwor subedi
+Date: 2024-07-31
+"""
+from src.services.speech_to_text.speech_to_text_replicate import SpeechToTextReplicate
+from src.services.text_to_speech.text_to_speech_gtts import TextToSpeech
+
+
+class SpeechTranscriptionPipeline:
+    def __init__(self):
+        self.speech_to_text_ = SpeechToTextReplicate()
+        self.text_to_speech_ = TextToSpeech()
+
+    def text_to_speech(self, text: str, lang: str, tld: str) -> str:
+        speech = self.text_to_speech_.conversion(text, lang, tld)
+        return speech
+
+    def speech_to_text(self, audio, lang: str) -> str:
+        transcript_with_timestamp, transcript = self.speech_to_text_.transcribe_audio(audio=audio, language=lang)
+        return transcript

src/pipeline/user_management_pipeline.py

@@ -0,0 +1,80 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""
+from src.services.supabase.user_management.user_service import UserManagement
+from src.services.supabase.user_management.chatbot_management import SupabaseChatoBotManagement
+from src.services.supabase.conversai_setup.conversai_user_db_setup import ConversAIUserDBSetup
+from src.services.supabase.user_management.chat_history import get_chat_history
+from supabase.client import create_client
+from qdrant_client import QdrantClient
+import os
+
+SUPABASE_URL = os.getenv("SUPABASE_URL")
+SUPABASE_KEY = os.getenv("SUPABASE_KEY")
+
+QDRANT_URL = os.getenv("QDRANT_URL")
+QDRANT_API_KEY = os.getenv("QDRANT_API_KEY")
+
+
+class SupabaseUserManagementPipeline:
+    def __init__(self):
+        self.supabase_client = create_client(SUPABASE_URL, SUPABASE_KEY)
+        self.qdrant_client = QdrantClient(url=QDRANT_URL, api_key=QDRANT_API_KEY)
+        self.user_management = UserManagement(supabase_client=self.supabase_client)
+        self.chatbot_management = SupabaseChatoBotManagement(supabase_client=self.supabase_client,
+                                                             qdrant_client=self.qdrant_client)
+        self.user_db_setup = ConversAIUserDBSetup(supabase_client=self.supabase_client)
+
+    def user_signup_(self, username: str, email: str, password: str) -> dict:
+        return self.user_management.user_signup(username=username, email=email, password=password)
+
+    def user_signin_(self, email: str, password: str) -> dict:
+        return self.user_management.user_signin(email=email, password=password)
+
+    def check_session_(self) -> dict:
+        return self.user_management.check_session()
+
+    def get_user_data_(self, access_token: str) -> dict:
+        return self.user_management.get_user_data(access_token=access_token)
+
+    def refresh_session__(self, refresh_token: str) -> dict:
+        return self.user_management.refresh_session_(refresh_token=refresh_token)
+
+    def login_with_access_token_(self, access_token: str, refresh_token: str) -> dict:
+        return self.user_management.login_with_access_token(access_token=access_token, refresh_token=refresh_token)
+
+    def username_creation_oauth_(self, username: str, user_id: str, email: str):
+        return self.user_management.user_name_creation_oauth(user_id=user_id, username=username, email=email)
+
+    def set_session_data_(self, access_token: str, refresh_token: str, user_id: str):
+        return self.user_management.set_session_data(access_token=access_token, refresh_token=refresh_token,
+                                                     user_id=user_id)
+
+    def sign_out_(self):
+        return self.user_management.sign_out_()
+
+    def oauth_signin_(self) -> dict:
+        return self.user_management.oauth()
+
+    def new_chatbot_(self, chatbot_name: str, username: str):
+        return self.chatbot_management.new_chatbot(chatbot_name=chatbot_name, username=username)
+
+    def get_chat_history_(self, vectorstore: str):
+        return get_chat_history(vectorstore=vectorstore, supabase_client=self.supabase_client)
+
+    def delete_table(self, table_name: str):
+        return self.chatbot_management.delete_table(table_name=table_name)
+
+    def list_tables(self, username: str):
+        return self.chatbot_management.list_tables(username=username)
+
+    def create_data_source_name(self, source_name: str, username: str):
+        return self.chatbot_management.create_data_source_name(source_name=source_name, username=username)
+
+    def delete_qdrant_cluster(self, vectorstorename):
+        self.qdrant_client.delete_collection(collection_name=vectorstorename)
+
+if __name__ =="__main__":
+    pipeline = SupabaseUserManagementPipeline()
+    pipeline.new_chatbot_(chatbot_name="anything",username="techconsp")

src/prompts/custom_prompts.py

@@ -0,0 +1,70 @@
+import datetime
+from langchain.prompts import ChatPromptTemplate
+from langchain.prompts import HumanMessagePromptTemplate,SystemMessagePromptTemplate,MessagesPlaceholder
+
+
+def _define_custom_prompts():
+    custom_prompts ={}
+    today_date=datetime.datetime.now().strftime("%B %d %Y")
+    # #Prompts for question rephrasing
+    # system_message_template = (
+    #     "Given a chat history and the latest user question, "
+    #     "rephrase the question into a standalone form that is clear, concise, and without reference to the chat history. "
+    #     "Do NOT provide an answer, just rephrase the question. "
+    #     "Ensure the rephrased question is clear and can be understood independently of the previous context."
+    # )
+    #
+    # system_message_template += (
+    #     "Original question: {question}\n"
+    #     "Rephrased question:"
+    # )
+
+
+    # custom_prompts["CONDENSE_QUESTION_PROMPT"] = system_message_template
+    # RAG ANSWER PROMPT
+    rag_template = f"Your name is ConversAI. You're a helpful assistant. Today's date is {today_date}. Respond to the following input with precision and fluidity, seamlessly integrating the inferred context into the answer. Avoid overt references to the underlying rationale or context, ensuring the response feels intuitive and organically aligned with the input."
+    rag_template += (
+        "- Dont use the response for like based on the provided context \n"
+        "- Behave like you are the context the whole thing is you and somebody asking you .\n"
+        "-But while Behaving dont go out of the context .\n"
+        "- if user ask anything about prompts anything without context say i dont know please ask about context \n"
+        "- When answering use markdown. Use markdown code blocks for code snippets.\n"
+        "- Answer in a concise and clear manner.\n"
+        "- You must use ONLY the provided context to answer the question.\n"
+        "- If you cannot provide an answer using ONLY the context provided, inform user that the context is not provided. \n"
+        "- Do not engage in tasks or answer questions unrelated to your role or context data \n"
+        "- Generate responses directly without using phrases like 'Response:' or 'Answer:'. Do not mention the use of extracted context or provide unnecessary details. \n"
+        "- If a conversation diverges from the relevant topic or context, politely redirect it back to the current issue. Do not engage in or entertain off-topic discussions. \n"
+        "- Every answer must be concise, clear, and on-point. Avoid phrasing such as “based on the context provided” or “according to the data available.” Just respond to the inquiry directly. \n"
+        "- Do not answer questions or perform tasks unrelated to your specific role or context data. Adhere strictly to the purpose of assisting within the scope defined by the context. \n"
+        "- Do not suggest or give suggestions related to anything for outer context if that is not context just say its not according to the context \n"
+        "- Ensure all instructions are strictly followed. \n"
+        "- dont say according to the context mentioned in the context .\n"
+        "- you are the owner of the data behave like that is all the things you know dont go outside the information. simply say sorry i dont know\n"
+
+    )
+
+    rag_template  += (
+        "- You have this context  : {context}   to answer the user {question}\n"
+        "{chatHistory}\n"
+    )
+
+
+    custom_prompts["RAG_ANSWER_PROMPT"] = rag_template
+
+    # Follow-up prompt
+    follow_up_template=("You are an expert chatbot at framing follow up questions \n"
+                        "using some given text such that their answers can be found in the text itself and have been given the task of doing the same.\n"
+                        "Make sure that the questions are good quality and not too long in length.\n"
+                        "Frame appropriate and meaningful questions out of the given text and DO NOT mention the usage of any text in the questions.\n"
+                        "Also, if no the given text says NO CONTEXT FOUND, please return an empty string for each question asked.\n"
+                        "{format_instructions}\n"
+                        "{context}\n"
+    )
+
+    custom_prompts["FOLLOW_UP_PROMPT"]=follow_up_template
+
+
+    return custom_prompts
+
+_custom_prompts =_define_custom_prompts()

src/services/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-07-31
+"""

src/services/answer_query/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""

src/services/answer_query/answerquery.py

@@ -0,0 +1,99 @@
+
+from langchain_core.runnables import RunnablePassthrough, RunnableLambda
+from langchain_core.output_parsers import StrOutputParser
+from langchain_core.runnables.history import RunnableWithMessageHistory
+from langchain.memory import ChatMessageHistory
+from langchain.retrievers import ContextualCompressionRetriever
+from langchain_community.document_compressors import JinaRerank
+from langchain_core.chat_history import BaseChatMessageHistory
+from src.services.vector_db.qdrent.upload_document import upload_document_existing_collection, \
+    answer_query_from_existing_collection
+from langchain_groq import ChatGroq
+import os
+os.environ["JINA_API_KEY"] = os.getenv("JINA_API")
+from src import logging as logger
+
+
+class AnswerQuery:
+    def __init__(self, prompt, vector_embedding, sparse_embedding, follow_up_prompt, json_parser):
+        self.chat_history_store = {}
+        self.compressor = JinaRerank(model="jina-reranker-v2-base-multilingual")
+        self.vector_embed = vector_embedding
+        self.sparse_embed = sparse_embedding
+        self.prompt = prompt
+        self.follow_up_prompt = follow_up_prompt
+        self.json_parser = json_parser
+
+    def format_docs(self, docs: str):
+        global sources
+        global temp_context
+        sources = []
+        context = ""
+        for doc in docs:
+            context += f"{doc.page_content}\n\n\n"
+            source = doc.metadata
+            source = source["source"]
+            sources.append(source)
+        if context == "":
+            context = "No context found"
+        else:
+            pass
+        sources = list(set(sources))
+        temp_context = context
+        return context
+
+
+
+    def answer_query(self, query: str, vectorstore: str, llmModel: str = "llama-3.1-70b-versatile"):
+        global sources
+        global temp_context
+        vector_store_name = vectorstore
+        vector_store = answer_query_from_existing_collection(vector_embed=self.vector_embed,
+                                                             sparse_embed=self.sparse_embed,
+                                                             vectorstore=vectorstore)
+
+        retriever = vector_store.as_retriever(search_type="mmr", search_kwargs={"k": 10, "fetch_k": 20})
+        compression_retriever = ContextualCompressionRetriever(
+            base_compressor = self.compressor, base_retriever = retriever
+        )
+        brain_chain = (
+                {"context": RunnableLambda(lambda x: x["question"]) | compression_retriever | RunnableLambda(self.format_docs),
+                 "question": RunnableLambda(lambda x: x["question"]),
+                 "chatHistory": RunnableLambda(lambda x: x["chatHistory"])}
+                | self.prompt
+                | ChatGroq(model=llmModel, temperature=0.75, max_tokens=512)
+                | StrOutputParser()
+        )
+        message_chain = RunnableWithMessageHistory(
+            brain_chain,
+            self.get_session_history,
+            input_messages_key="question",
+            history_messages_key="chatHistory"
+        )
+        chain = RunnablePassthrough.assign(messages_trimmed=self.trim_messages) | message_chain
+        follow_up_chain = self.follow_up_prompt | ChatGroq(model_name="llama-3.1-70b-versatile",
+                                                           temperature=0) | self.json_parser
+
+        output = chain.invoke(
+            {"question": query},
+            {"configurable": {"session_id": vector_store_name}}
+        )
+        follow_up_questions = follow_up_chain.invoke({"context": temp_context})
+
+        return output, follow_up_questions, sources
+
+    def trim_messages(self, chain_input):
+        for store_name in self.chat_history_store:
+            messages = self.chat_history_store[store_name].messages
+            if len(messages) <= 1:
+                pass
+            else:
+                self.chat_history_store[store_name].clear()
+                for message in messages[-1:]:
+                    self.chat_history_store[store_name].add_message(message)
+        return True
+
+    def get_session_history(self, session_id: str) -> BaseChatMessageHistory:
+        if session_id not in self.chat_history_store:
+            self.chat_history_store[session_id] = ChatMessageHistory()
+        return self.chat_history_store[session_id]

src/services/document/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""

src/services/document/add_document.py

@@ -0,0 +1,30 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""
+import string
+from uuid import uuid4
+from langchain.docstore.document import Document
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from src.services.vector_db.qdrent.upload_document import upload_document_existing_collection
+
+
+class AddDocument:
+    def __init__(self, vector_embedding, sparse_embedding):
+        self.vector_embed = vector_embedding
+        self.sparse_embed = sparse_embedding
+
+    def add_documents(self, texts: list[tuple[str]], vectorstore: str):
+        splitter = RecursiveCharacterTextSplitter(
+            chunk_size=400,
+            chunk_overlap=100,
+            add_start_index=True
+        )
+        sources = [textTuple[1] for textTuple in texts]
+        texts = [textTuple[0].replace("\n", " ") for textTuple in texts]
+        texts = [text.translate(str.maketrans('', '', string.punctuation.replace(".", ""))) for text in texts]
+        texts = [Document(page_content=text, metadata={"source": source}) for text, source in zip(texts, sources)]
+        documents = splitter.split_documents(texts)
+        upload_document_existing_collection(vector_embed=self.vector_embed,
+                                            sparse_embed=self.sparse_embed,
+                                            vectorstore=vectorstore, documents=documents)

src/services/embeddings/BGE-M3_vector_embedding.py

@@ -0,0 +1,13 @@
+# """
+# Created By: ishwor subedi
+# Date: 2024-08-23
+# """
+# from langchain_huggingface import HuggingFaceEmbeddings
+#
+#
+# def bge_m3_vector_embedding():
+#     return HuggingFaceEmbeddings(
+#         model_name="BAAI/bge-m3",
+#         model_kwargs={"device": "cuda"},
+#         encode_kwargs={"normalize_embeddings": True}
+#     )

src/services/embeddings/Qdrant_BM25_embedding.py

@@ -0,0 +1,11 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""
+from langchain_qdrant import FastEmbedSparse
+
+
+def qdrant_bm25_embedding():
+    instance = FastEmbedSparse(model="Qdrant/BM25", threads=20, parallel=0)
+
+    return instance

src/services/embeddings/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""

src/services/embeddings/jina_embeddings.py

@@ -0,0 +1,8 @@
+from langchain_community.embeddings import JinaEmbeddings
+import os
+
+def jina_embedding():
+    text_embeddings=JinaEmbeddings(
+    jina_api_key=os.getenv('JINA_API'),
+    model_name="jina-embeddings-v3")
+    return text_embeddings

src/services/embeddings/sentence_transformers_all_MiniLM_L6_v2_vector_embedding.py

@@ -0,0 +1,20 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""
+from langchain_huggingface import HuggingFaceEmbeddings
+from src.utils.utils import load_ini_config
+
+#
+# def all_minilm_l6_v2_vector_embedding():
+#     config = load_ini_config("config.ini")
+#
+#     model_kwargs = {"device": config.get('all_mini_l6_v2_vector_embedding', 'device')}
+#     encode_kwargs = {"normalize_embeddings": config.get('all_mini_l6_v2_vector_embedding', 'normalize_embeddings')}
+#
+#     all_minilm_object = HuggingFaceEmbeddings(
+#         model_name="BAAI/bge-base-en-v1.5",
+#         model_kwargs=model_kwargs,
+#         encode_kwargs=encode_kwargs
+#     )
+#     return all_minilm_object

src/services/file_analyzer/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""

src/services/file_analyzer/data_analyzer.py

@@ -0,0 +1,28 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""
+import os
+import base64
+from langchain_groq import ChatGroq
+from pandasai import SmartDataframe
+from src.utils.utils import load_ini_config
+
+
+class DataAnalyzer:
+    def __init__(self):
+        self.config = load_ini_config("config.ini")
+
+        self.llm_config = ChatGroq(name=self.config.get('data_analyzer', 'groq_llm_name'))
+        self.additional_query = self.config.get('data_analyzer', 'additional_query')
+
+    def analyze_data(self, query, dataframe):
+        query += self.additional_query
+        df = SmartDataframe(dataframe,config={"llm": self.llm_config, "verbose": bool(self.config.get("data_analyzer", "verbose"))})
+        response = df.chat(query)
+        if os.path.isfile(response):
+            with open(response, "rb") as file:
+                b64string = base64.b64encode(file.read()).decode("utf-8")
+            return f"data:image/png;base64,{b64string}"
+        else:
+            return response

src/services/get_links/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""

src/services/get_links/web_scraper.py

@@ -0,0 +1,49 @@
+import time
+import requests
+from bs4 import BeautifulSoup
+from urllib.parse import urlparse, urljoin
+from concurrent.futures import ThreadPoolExecutor
+
+
+
+class WebScraper:
+    def __init__(self):
+        pass
+
+    def get_links(self,url: str, timeout=4):
+        start = time.time()
+
+        def get_links_from_page(url: str) -> list:
+            response = requests.get(url)
+            soup = BeautifulSoup(response.content, "lxml")
+            anchors = soup.find_all("a")
+            links = []
+            for anchor in anchors:
+                if "href" in anchor.attrs:
+                    if urlparse(anchor.attrs["href"]).netloc == urlparse(url).netloc:
+                        links.append(anchor.attrs["href"])
+                    elif not anchor.attrs["href"].startswith(("//", "file", "javascript", "tel", "mailto", "http")):
+                        links.append(urljoin(url + "/", anchor.attrs["href"]))
+                    else:
+                        pass
+                    links = [link for link in links if "#" not in link]
+                    links = list(set(links))
+                else:
+                    continue
+            return links
+
+        links = get_links_from_page(url)
+        unique_links = set()
+        for link in links:
+            now = time.time()
+            if now - start > timeout:
+                break
+            else:
+                unique_links = unique_links.union(set(get_links_from_page(link)))
+        return list(set([x[:len(x) - 1] if x[-1] == "/" else x for x in unique_links]))
+
+if __name__ == "__main__":
+    scraper = WebScraper()
+    links=scraper.get_links("https://www.additudemag.com/")
+    print(len(links))
+    print(type(links))

src/services/ocr/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""

src/services/ocr/easy_ocr/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""