Modulation and logging to console

.gitignore

@@ -7,5 +7,6 @@ chroma
 uploads/
 /flask_session
 log.txt
+*.log
 *.db
 __pycache__

app.py

@@ -1,41 +1,18 @@
-from flask import Flask, request, jsonify
+from flask import Flask, Response, after_this_request, request, jsonify
 from flask import render_template
 from werkzeug.utils import secure_filename
-from apscheduler.schedulers.background import BackgroundScheduler
+from flask_cors import CORS
 
 from langchain_community.embeddings import HuggingFaceInferenceAPIEmbeddings
 import google.generativeai as genai
-from langchain.text_splitter import RecursiveCharacterTextSplitter
-from utils import add_file_to_chroma, remove_file_from_chroma, generate_query_response
+from langchain_chroma import Chroma
 
+from utils import add_file_to_chroma, remove_file_from_chroma, generate_query_response
 from db import init_db,add_file_to_db,remove_file_from_db,get_all_files
+from env_vars import HF_TOKEN,GOOGLE_API_KEY,CHROMA_PATH,UPLOAD_FOLDER,DB_PATH,PROMPT_TEMPLATE
+from logger import logger
 
-from langchain_chroma import Chroma
 import os
-from dotenv import load_dotenv
-import logging
-from flask_cors import CORS
-
-logging.basicConfig(filename='log.txt', filemode='w', level=logging.DEBUG,
-                   format='%(asctime)s - %(levelname)s - %(message)s')
-logger = logging.getLogger()
-
-load_dotenv()
-
-HF_TOKEN = os.getenv('HF_TOKEN')
-GOOGLE_API_KEY = os.getenv('GOOGLE_API_KEY')
-CHROMA_PATH = "chroma"
-UPLOAD_FOLDER = "uploads"
-DB_PATH = "files.db"
-PROMPT_TEMPLATE = """
-Answer the given query based only on the context given below.
-context:
-{context}
-
----
-based on the context above answer the following query:
-{query}
-"""
 
 # make required folder for files to upload
 os.makedirs(UPLOAD_FOLDER, exist_ok=True)    
@@ -58,7 +35,18 @@ db = Chroma(persist_directory=CHROMA_PATH, embedding_function=hugging_face_ef)
 
 
 # Initialize database
-init_db()
+init_db(DB_PATH)
+
+# log requests
+@app.before_request
+def log_details():
+    method = request.method
+    url = request.url
+
+    @after_this_request
+    def log_details_callback(response: Response):
+        logger.info(f'method: {method}\n url: {url}\n status: {response.status}')
+        return response
 
 @app.route('/')
 def index():
@@ -68,7 +56,7 @@ def index():
 def get_files():
     """Get list of all files."""
     try:
-        files = get_all_files()
+        files = get_all_files(DB_PATH)
         return jsonify({
             'status': 'success',
             'files': files
@@ -102,7 +90,7 @@ def upload_file():
         add_file_to_chroma(file_path, file_id, hugging_face_ef, db, logger)
         
         # Add file to SQLite database
-        add_file_to_db(file_id, filename, file_type)
+        add_file_to_db(file_id, filename, file_type,DB_PATH)
         
         return jsonify({
             'message': 'File uploaded successfully',
@@ -132,7 +120,7 @@ def remove_file():
         # Remove from ChromaDB
         if remove_file_from_chroma(file_id, db):
             # Remove from SQLite database
-            remove_file_from_db(file_id)
+            remove_file_from_db(file_id,DB_PATH)
             return jsonify({
                 'message': 'File deleted successfully',
                 'status': 'success'

db.py

@@ -1,7 +1,8 @@
 import sqlite3
+from env_vars import DB_PATH
 
 # Initialize SQLite database for file tracking
-def init_db(DB_PATH = "files.db"):
+def init_db(DB_PATH = DB_PATH):
     conn = sqlite3.connect(DB_PATH)
     c = conn.cursor()
     c.execute('''CREATE TABLE IF NOT EXISTS files
@@ -12,7 +13,7 @@ def init_db(DB_PATH = "files.db"):
     conn.commit()
     conn.close()
 
-def add_file_to_db(file_id, file_name, file_type,DB_PATH = "files.db"):
+def add_file_to_db(file_id, file_name, file_type,DB_PATH = DB_PATH):
     conn = sqlite3.connect(DB_PATH)
     c = conn.cursor()
     c.execute('INSERT INTO files (file_id, file_name, file_type) VALUES (?, ?, ?)',
@@ -20,14 +21,14 @@ def add_file_to_db(file_id, file_name, file_type,DB_PATH = "files.db"):
     conn.commit()
     conn.close()
 
-def remove_file_from_db(file_id,DB_PATH = "files.db"):
+def remove_file_from_db(file_id,DB_PATH = DB_PATH):
     conn = sqlite3.connect(DB_PATH)
     c = conn.cursor()
     c.execute('DELETE FROM files WHERE file_id = ?', (file_id,))
     conn.commit()
     conn.close()
 
-def get_all_files(DB_PATH = "files.db"):
+def get_all_files(DB_PATH = DB_PATH):
     conn = sqlite3.connect(DB_PATH)
     c = conn.cursor()
     c.execute('SELECT file_id, file_name, file_type, created_at FROM files ORDER BY created_at DESC')

env_vars.py

@@ -0,0 +1,19 @@
+import os
+from dotenv import load_dotenv
+
+load_dotenv()
+
+HF_TOKEN = os.getenv('HF_TOKEN')
+GOOGLE_API_KEY = os.getenv('GOOGLE_API_KEY')
+CHROMA_PATH = "chroma"
+UPLOAD_FOLDER = "uploads"
+DB_PATH = "files.db"
+PROMPT_TEMPLATE = """
+Answer the given query based only on the context given below.
+context:
+{context}
+
+---
+based on the context above answer the following query:
+{query}
+"""

logger.py

@@ -0,0 +1,6 @@
+import logging
+
+logging.basicConfig(level=logging.DEBUG,
+                   format='%(asctime)s - %(levelname)s - %(message)s',
+                   handlers=[logging.StreamHandler(),logging.FileHandler('logs.log')])
+logger = logging.getLogger()

utils.py

@@ -1,11 +1,7 @@
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain_community.document_loaders import PyPDFLoader,UnstructuredWordDocumentLoader,TextLoader,UnstructuredHTMLLoader,UnstructuredMarkdownLoader
 import os 
-import logging
-
-logging.basicConfig(filename='log.txt', filemode='w', level=logging.DEBUG,
-                   format='%(asctime)s - %(levelname)s - %(message)s')
-logger = logging.getLogger()
+from logger import logger
 
 def add_file_to_chroma(file_path, file_id, hugging_face_ef, db, logger):
     """Add file chunks to ChromaDB with advanced document handling."""
@@ -90,6 +86,8 @@ def generate_query_response(query, db, llm_model, PROMPT_TEMPLATE):
         prompt = PROMPT_TEMPLATE.format(context=context, query=query)
         answer = llm_model.generate_content(prompt).text
         
+        logger.info(f"Query : {query}\nResponse:{answer}")
+        
         # Prepare response with sources
         return {
             "is_relevant": is_relevant,