initial commit for huggingface

.gitignore

@@ -0,0 +1,168 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+debug.log
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+Pipfile.lock
+
+# poetry
+poetry.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyderproject.*
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# VS Code
+.vscode/
+
+# Windows
+Thumbs.db
+Desktop.ini
+
+# macOS
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Linux
+*~
+
+# Test artifacts
+test/
+artifacts/
+
+# Database
+*.db
+*.sqlite
+
+# Logs
+*.log
+
+# Excel
+*.xlsx
+*.xls
+
+# Misc
+*.bak
+*.tmp
+*.swp
+*.swo
+
+# Python virtual environment
+backend_env/

backend/.dockerfile

@@ -0,0 +1,17 @@
+# Use the official Python 3.11.5 image
+FROM python:3.11.5
+
+# Set the working directory
+WORKDIR /app
+
+# Copy the backend folder into the container
+COPY backend /app
+
+# Install dependencies
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Expose the port FastAPI runs on
+EXPOSE 7860
+
+# Run FastAPI app
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]

backend/app/api/endpoints.py

@@ -0,0 +1,82 @@
+from langchain_openai import ChatOpenAI
+from config.settings import settings
+
+# --- LLM-based classifier for user input ---
+
+import pandas as pd
+# Use LLMClassifier from services.utility
+from app.services.utility import UtilityClass
+from fastapi import APIRouter
+from pydantic import BaseModel
+from app.services.sql_generator import generate_sql_query
+from app.services.query_executor import execute_sql_query
+from app.services.result_formatter import df_to_chart
+from app.services.query_executor import execute_sql_query
+from app.db.schema_reader import get_schema
+from app.services.query_executor import run_and_handle_sql_query
+
+router = APIRouter()
+
+class QueryRequest(BaseModel):
+    question: str
+
+from fastapi import HTTPException
+
+@router.post("/ask")
+def ask_query(req: QueryRequest):
+    try:
+        sql = generate_sql_query(req.question)
+
+        # If generate_sql_query returns a chat_message, treat as normal chat or empty SQL result
+        if isinstance(sql, dict) and "chat_message" in sql:
+            return {"message": sql["chat_message"]}
+
+
+        # Use utility method to check if the string is a valid SQL query
+        formattedSqlQuery =  UtilityClass.is_valid_sql_query(sql)
+        if not formattedSqlQuery:
+            return {"message": str(sql) if sql else "No SQL query could be generated for your question."}
+
+        
+        df = run_and_handle_sql_query(formattedSqlQuery, req.question)
+
+        # If run_and_handle_sql_query returns a chat_message (for empty SQL results), return it
+        if isinstance(df, dict) and "chat_message" in df:
+            return {"message": df["chat_message"]}
+
+        chart = None
+        # if len(df) > 0 and isinstance(df, list) and len(df[0]) > 0 and len(df[0].keys()) >= 2:
+        #     chart = df_to_chart(pd.DataFrame(df))
+
+        # Prepare heading and records JSON using LLM
+        result_json = UtilityClass.prepare_llm_heading_and_records(req.question, df)
+
+        return {
+            "sql": sql,
+            "rows": result_json["records"],
+            "heading": result_json["heading"],
+            "chart": chart
+        }
+    except Exception as e:
+        raise HTTPException(status_code=400, detail=f"An error occurred: {str(e)}")
+
+@router.get("/schema")
+def read_schema():
+    return get_schema()
+
+@router.get("/query")
+def run_query(question: str):
+    try:
+        sql_query = generate_sql_query(question)
+        results = execute_sql_query(sql_query)
+        return {
+            "question": question,
+            "sql_query": sql_query,
+            "results": results
+        }
+    except Exception as e:
+        raise HTTPException(status_code=400, detail=f"An error occurred: {str(e)}")
+    
+@router.get("/")
+def root():
+    return {"message": "Welcome to AI SQL Query Generator!"}

backend/app/db/db_connector.py

@@ -0,0 +1,51 @@
+from sqlalchemy import create_engine
+from sqlalchemy.engine import Engine
+from sqlalchemy.orm import sessionmaker
+import sqlite3
+from pathlib import Path
+import os
+from dotenv import load_dotenv
+load_dotenv()
+
+# Path to SQLite DB file, configurable via environment variable or .env
+print("DB_PATH from env:", os.getenv("DB_PATH"))
+DB_PATH = os.getenv("DB_PATH")
+if not DB_PATH:
+    BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+    DB_PATH = os.path.join(BASE_DIR, "BankingDb.db")
+
+DATABASE_URL = f"sqlite:///{DB_PATH}"
+
+def get_connection():
+    db_path = DB_PATH
+    # Only allow connection if DB file exists
+    if not os.path.isfile(db_path):
+        raise FileNotFoundError(f"Database file not found at {db_path}. Set DB_PATH env variable or .env to the correct location.")
+    try:
+        conn = sqlite3.connect(db_path, timeout=30, check_same_thread=False)
+        conn.row_factory = sqlite3.Row  # access columns by name
+        return conn
+    except sqlite3.OperationalError as e:
+        raise Exception(f"Error connecting to DB at {db_path}: {e}")
+
+# Create engine
+engine = create_engine(
+    DATABASE_URL,
+    connect_args={"check_same_thread": False}  # Needed for SQLite threading
+)
+
+# Session factory
+SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
+
+def get_db_engine():
+    """Return SQLAlchemy engine (used by LangChain SQLDatabase)."""
+    return engine
+
+
+def get_db():
+    """Provide DB session for queries."""
+    db = SessionLocal()
+    try:
+        yield db
+    finally:
+        db.close()

backend/app/db/schema_reader.py

@@ -0,0 +1,50 @@
+from sqlalchemy import inspect
+from .db_connector import get_db_engine
+from .db_connector import get_connection
+
+def get_schema():
+    """Fetch database schema information (tables & columns)."""
+    schema = {}
+    conn = get_connection()
+    try:
+        cursor = conn.cursor()
+        
+        # Get all tables
+        cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%';")
+        tables = cursor.fetchall()
+
+        for t in tables:
+            table_name = t[0]
+            cursor.execute(f"PRAGMA table_info({table_name});")
+            columns = cursor.fetchall()
+            schema[table_name] = [col[1] for col in columns]
+
+        return schema
+    finally:
+        cursor.close()
+        conn.close()
+
+def get_schema2():
+    """Fetch database schema information (tables & columns)."""
+    engine = get_db_engine()
+    inspector = inspect(engine)
+
+    schema = {}
+    for table_name in inspector.get_table_names():
+        columns = [
+            {"name": col["name"], "type": str(col["type"])}
+            for col in inspector.get_columns(table_name)
+        ]
+        schema[table_name] = columns
+
+    return schema
+
+# def get_schema():
+#     engine = get_db_engine()
+#     inspector = inspect(engine)
+
+#     schema_info = {}
+#     for table in inspector.get_table_names():
+#         columns = [col["name"] for col in inspector.get_columns(table)]
+#         schema_info[table] = columns
+#     return schema_info

backend/app/services/query_executor.py

@@ -0,0 +1,43 @@
+import pandas as pd
+from sqlalchemy import text
+from app.db.db_connector import get_db_engine
+
+
+
+from langchain_openai import ChatOpenAI
+from config.settings import settings
+
+def execute_sql_query(sql_query: str):
+    """Run a SQL query and return results as list of dicts."""
+    engine = get_db_engine()
+    with engine.connect() as conn:
+        result = conn.execute(text(sql_query))
+        rows = [dict(row) for row in result.mappings()]
+    return rows
+
+def run_and_handle_sql_query(sql_query: str, user_question: str):
+    """
+    Executes the SQL query, and if no results, uses LLM to generate a friendly message.
+    Returns either a list of dicts (rows) or {"chat_message": ...}.
+    """
+    try:
+        rows = execute_sql_query(sql_query)
+        if not rows:
+            llm = ChatOpenAI(model="gpt-4o-mini", temperature=0, api_key=settings.OPENAI_API_KEY)
+            no_result_prompt = (
+                f"The following SQL query was generated for the user's question, but it returned no results. "
+                f"User question: {user_question}\nSQL query: {sql_query}\n"
+                "Please explain to the user in a friendly way that no matching records were found for their request."
+            )
+            no_result_response = llm.invoke([{"role": "user", "content": no_result_prompt}]).content.strip()
+            return {"chat_message": no_result_response}
+        return rows
+    except Exception as e:
+        return {"error": str(e), "sql_query": sql_query}
+
+# def run_query(sql: str):
+#     engine = get_db_engine()
+#     with engine.connect() as conn:
+#         result = conn.execute(text(sql))
+#         df = pd.DataFrame(result.fetchall(), columns=result.keys())
+#     return df

backend/app/services/result_formatter.py

@@ -0,0 +1,21 @@
+import matplotlib.pyplot as plt
+import io
+import base64
+
+def df_to_chart(df, chart_type="bar"):
+    import pandas as pd
+    plt.clf()
+    y_col = df.columns[1] if len(df.columns) > 1 else None
+    if y_col is not None and pd.api.types.is_numeric_dtype(df[y_col]):
+        # Numeric y-column: plot as usual
+        ax = df.plot(kind=chart_type, x=df.columns[0], y=y_col, legend=False)
+    else:
+        # Non-numeric: plot value counts of the first column
+        value_counts = df[df.columns[0]].value_counts()
+        ax = value_counts.plot(kind="bar")
+        ax.set_xlabel(df.columns[0])
+        ax.set_ylabel("Count")
+    buf = io.BytesIO()
+    plt.savefig(buf, format="png")
+    buf.seek(0)
+    return base64.b64encode(buf.read()).decode("utf-8")

backend/app/services/sql_generator.py

@@ -0,0 +1,60 @@
+from langchain_community.utilities import SQLDatabase
+from langchain.chains import create_sql_query_chain
+from langchain_openai import ChatOpenAI  # or your LLaMA adapter
+from config.settings import settings
+from langchain.prompts import PromptTemplate, ChatPromptTemplate
+from app.db.schema_reader import get_schema
+from app.db.db_connector import get_db_engine
+from app.db.db_connector import get_connection
+
+# llm = ChatOpenAI(model="gpt-4", temperature=0, api_key=settings.OPENAI_API_KEY)
+llm = ChatOpenAI(model="gpt-4o-mini", temperature=0, api_key=settings.OPENAI_API_KEY)
+
+
+def generate_sql_query(user_question: str):
+    schema = get_schema()
+    schema_str = "\n".join(
+        [f"{table}: {', '.join(cols)}" for table, cols in schema.items()]
+    )
+
+    prompt = ChatPromptTemplate.from_template("""
+    You are an expert SQLite query generator.
+    Use ONLY the following tables and columns:
+
+    {schema}
+
+    Generate a correct SQLite query (no explanations, only SQL).
+    Do not add single quotes or any quotes around table names or column names.
+    If the user asks for data, generate a SELECT query and that outcome should have only 
+    sqlite compatible, don't include any other text.
+    If you cannot answer, say: "I cannot generate this query."
+
+    User question: {question}
+    """)
+
+    messages = prompt.format_messages(schema=schema_str, question=user_question)
+    sql_query = llm.invoke(messages).content.strip()
+
+    # If the LLM says it cannot generate a query, treat as normal chat
+    if sql_query.lower().startswith("i cannot generate this query"):
+        # Use the LLM as a chatbot for normal conversation
+        chat_prompt = f"You are a helpful assistant. Respond conversationally to: {user_question}"
+        chat_response = llm.invoke([{"role": "user", "content": chat_prompt}]).content.strip()
+        return {"chat_message": chat_response}
+
+    return sql_query
+
+def generate_sql_query2(natural_language_query: str) -> str:
+    """Generate SQL query from natural language using LangChain."""
+    engine = get_db_engine()
+    db = SQLDatabase(engine)
+
+    # You can swap ChatOpenAI with LLaMA wrapper
+    llm = ChatOpenAI(model="gpt-4o-mini", temperature=0, api_key=settings.OPENAI_API_KEY)
+
+    sql_chain = create_sql_query_chain(llm, db)
+
+    sql_query = sql_chain.invoke({"question": natural_language_query})
+    return sql_query
+
+

backend/app/services/utility.py

@@ -0,0 +1,105 @@
+import openai
+from config.settings import settings
+from langchain_openai import ChatOpenAI
+import sqlparse
+import re
+class UtilityClass:
+    """
+    Classifies user input as 'sql' (SQL/data context) or 'chat' (normal conversation) using an LLM.
+    """
+    @staticmethod
+    def classify_user_input(text: str) -> str:
+        prompt = (
+            "Classify the following message as 'sql' if it is a database/data/SQL question, "
+            "or 'chat' if it is normal conversation.\n"
+            f"Message: {text}\n"
+            "Respond with only 'sql' or 'chat'."
+        )
+        response = openai.ChatCompletion.create(
+            model="gpt-3.5-turbo",
+            messages=[{"role": "user", "content": prompt}],
+            api_key=settings.OPENAI_API_KEY
+        )
+        label = response.choices[0].message.content.strip().lower()
+        return label if label in ("sql", "chat") else "chat"
+
+    @staticmethod
+    def chat_response(text: str) -> str:
+        chat_prompt = f"You are a helpful assistant. Respond conversationally to: {text}"
+        response = openai.ChatCompletion.create(
+            model="gpt-3.5-turbo",
+            messages=[{"role": "user", "content": chat_prompt}],
+            api_key=settings.OPENAI_API_KEY
+        )
+        return response.choices[0].message.content.strip()
+    
+    # Helper to prepare JSON with heading from LLM and records
+    @staticmethod
+    def prepare_llm_heading_and_records(user_question, rows):
+        llm = ChatOpenAI(model="gpt-4o-mini", temperature=0, api_key=settings.OPENAI_API_KEY)
+        record_count = len(rows)
+        column_names = list(rows[0].keys()) if rows and isinstance(rows[0], dict) else []
+        heading_prompt = (
+            "Given the user's question and the following column names and record count, generate:\n"
+            "1. A short, clear, and specific heading for the results (do NOT use generic phrases like 'Here are the top X results I found').\n"
+            "2. A 1-2 sentence summary or description of what the results represent, using the question and columns for context.\n"
+            f"User question: {user_question}\nColumns: {column_names}\nRecord count: {record_count}\n"
+            "Return your answer as JSON with keys 'heading' and 'summary'. Do not mention the actual data."
+        )
+        heading = llm.invoke([{"role": "user", "content": heading_prompt}]).content.strip()
+        return {"heading": heading, "records": rows}
+    
+    @staticmethod
+    def is_valid_sql_query33(sql: str) -> bool:
+        """
+        Returns True if the string is a valid SQL statement (not just a keyword in text).
+        Uses sqlparse to check for a valid statement structure.
+        Accepts queries starting with 'sql ' followed by a valid SQL statement.
+        """
+        if not sql or not isinstance(sql, str):
+            return False
+        sql_strip = sql.strip()
+        # Remove leading 'sql' if present
+        if sql_strip.lower().startswith('sql'):
+            sql_strip = sql_strip[4:].lstrip()
+        parsed = sqlparse.parse(sql_strip)
+        if not parsed or not parsed[0].tokens:
+            return False
+        stmt = parsed[0]
+        first_token = stmt.token_first(skip_cm=True, skip_ws=True)
+        if first_token is None:
+            return False
+        # Accept only if the first token is a SQL keyword and there is more than one token
+        return first_token.ttype in sqlparse.tokens.Keyword.DML and len(stmt.tokens) > 1
+    
+    @staticmethod
+    def is_valid_sql_query(text: str):
+        if not text or not text.strip():
+            return False
+
+        # Remove markdown formatting like ```sql ... ```
+        text = re.sub(r"```sql|```", "", text, flags=re.IGNORECASE).strip()
+
+        # Collapse multiple spaces & line breaks into a single space
+        normalized = re.sub(r"\s+", " ", text).strip().upper()
+
+        # Common SQL query structure patterns (heuristics)
+        sql_patterns = [
+            r"^SELECT\s+.+\s+FROM\s+.+",          # SELECT ... FROM ...
+            r"^INSERT\s+INTO\s+.+\s+VALUES\s*\(", # INSERT INTO ... VALUES (...)
+            r"^UPDATE\s+.+\s+SET\s+.+",           # UPDATE ... SET ...
+            r"^DELETE\s+FROM\s+.+",               # DELETE FROM ...
+            r"^CREATE\s+(TABLE|DATABASE)\s+.+",   # CREATE TABLE/DATABASE ...
+            r"^DROP\s+(TABLE|DATABASE)\s+.+",     # DROP TABLE/DATABASE ...
+            r"^ALTER\s+TABLE\s+.+",               # ALTER TABLE ...
+            r"^WITH\s+.+\s+AS\s*\(.+\)"           # WITH ... AS (...)
+        ]
+
+        for pattern in sql_patterns:
+                # Format the SQL for readability
+                formatted_sql = sqlparse.format(text, reindent=True, keyword_case='upper')
+                return formatted_sql
+
+        return False
+
+

backend/app/services/utility_sql.py

@@ -0,0 +1,21 @@
+import sqlparse
+
+class UtilityClass:
+    @staticmethod
+    def is_valid_sql_query(sql: str) -> bool:
+        """
+        Returns True if the string is a valid SQL statement (not just a keyword in text).
+        Uses sqlparse to check for a valid statement structure.
+        """
+        if not sql or not isinstance(sql, str):
+            return False
+        parsed = sqlparse.parse(sql)
+        if not parsed or not parsed[0].tokens:
+            return False
+        # Check if the first token is a DML/DDL keyword and the statement is not just a keyword
+        stmt = parsed[0]
+        first_token = stmt.token_first(skip_cm=True, skip_ws=True)
+        if first_token is None:
+            return False
+        # Accept only if the first token is a SQL keyword and there is more than one token
+        return first_token.ttype in sqlparse.tokens.Keyword.DML and len(stmt.tokens) > 1

backend/config/settings.py

@@ -0,0 +1,13 @@
+import os
+from dotenv import load_dotenv
+
+load_dotenv()
+
+class Settings:
+    ENV = os.getenv("ENV")  # Default to "local" if ENV is not set
+    # LLM API Token
+    OPENAI_API_KEY = os.getenv("OPENAI_API_KEY") or os.environ.get("OPENAI_API_KEY")
+
+    # GEMENI_API_KEY_TOKEN = os.getenv("OPENAI_API_KEY") or os.environ.get("OPENAI_API_KEY")
+
+settings = Settings()

backend/main.py

@@ -0,0 +1,26 @@
+from fastapi import FastAPI
+from app.api.endpoints import router
+from fastapi.middleware.cors import CORSMiddleware
+
+app = FastAPI(title="AI Query Generator")
+# app.include_router(router)
+
+# Allowed origins (Update this for production)
+
+#router = APIRouter()
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],  # Replace "*" with your frontend domain in production
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# Include API routes from endpoints.py
+app.include_router(router, prefix="/api")  # You can remove prefix if not needed
+
+@app.get("/")
+async def root():
+    return {"message": "Hello from FastAPI"}
+
+#app.include_router(router)

backend/requirements.txt

@@ -0,0 +1,15 @@
+sqlparse
+fastapi
+uvicorn
+streamlit
+requests
+langchain
+langchain-openai
+langchain-community
+sqlalchemy
+pydantic
+faiss-cpu          # optional for embeddings
+pandas
+matplotlib
+python-dotenv
+