all in one go

.gitignore

@@ -1,207 +1,28 @@
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[codz]
-*$py.class
-
-# C extensions
-*.so
-
-# Distribution / packaging
-.Python
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-share/python-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-MANIFEST
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.nox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-*.py.cover
-.hypothesis/
-.pytest_cache/
-cover/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-db.sqlite3
-db.sqlite3-journal
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-.pybuilder/
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# IPython
-profile_default/
-ipython_config.py
-
-# pyenv
-#   For a library or package, you might want to ignore these files since the code is
-#   intended to run in multiple environments; otherwise, check them in:
-# .python-version
-
-# pipenv
-#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
-#   However, in case of collaboration, if having platform-specific dependencies or dependencies
-#   having no cross-platform support, pipenv may install dependencies that don't work, or not
-#   install all needed dependencies.
-#Pipfile.lock
-
-# UV
-#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
-#   This is especially recommended for binary packages to ensure reproducibility, and is more
-#   commonly ignored for libraries.
-#uv.lock
-
-# poetry
-#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
-#   This is especially recommended for binary packages to ensure reproducibility, and is more
-#   commonly ignored for libraries.
-#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
-#poetry.lock
-#poetry.toml
+# gitignore template for LangChain products, e.g., LangGraph, LangSmith
+# website: https://www.langchain.com/
+# website: https://www.langchain.com/langgraph
 
-# pdm
-#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
-#   pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
-#   https://pdm-project.org/en/latest/usage/project/#working-with-version-control
-#pdm.lock
-#pdm.toml
-.pdm-python
-.pdm-build/
+# LangGraph
+.langgraph_api/
+ 
+vector_store/
 
-# pixi
-#   Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
-#pixi.lock
-#   Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
-#   in the .venv directory. It is recommended not to include this directory in version control.
-.pixi
 
-# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
-__pypackages__/
-
-# Celery stuff
-celerybeat-schedule
-celerybeat.pid
-
-# SageMath parsed files
-*.sage.py
-
-# Environments
+# ignore .env
 .env
-.envrc
-.venv
-env/
-venv/
-ENV/
-env.bak/
-venv.bak/
-
-# Spyder project settings
-.spyderproject
-.spyproject
-
-# Rope project settings
-.ropeproject
-
-# mkdocs documentation
-/site
-
-# mypy
-.mypy_cache/
-.dmypy.json
-dmypy.json
-
-# Pyre type checker
-.pyre/
-
-# pytype static type analyzer
-.pytype/
-
-# Cython debug symbols
-cython_debug/
-
-# PyCharm
-#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
-#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
-#  and can be added to the global gitignore or merged into this file.  For a more nuclear
-#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
-
-# Abstra
-# Abstra is an AI-powered process automation framework.
-# Ignore directories containing user credentials, local state, and settings.
-# Learn more at https://abstra.io/docs
-.abstra/
-
-# Visual Studio Code
-#  Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore 
-#  that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
-#  and can be added to the global gitignore or merged into this file. However, if you prefer, 
-#  you could uncomment the following to ignore the entire vscode folder
-# .vscode/
-
-# Ruff stuff:
-.ruff_cache/
+__pycache__/
+src/__pycache__/
+# Environment
+.env
+.env.*
 
-# PyPI configuration file
-.pypirc
+# Python cache
+__pycache__/
+*.pyc
 
-# Cursor
-#  Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to
-#  exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
-#  refer to https://docs.cursor.com/context/ignore-files
-.cursorignore
-.cursorindexingignore
+# Vector DB
+faiss_store/
+vector_store/
 
-# Marimo
-marimo/_static/
-marimo/_lsp/
-__marimo__/
+# LangGraph
+.langgraph_api/

Data/machine_learning.txt

@@ -0,0 +1,15 @@
+Machine Learning Basics
+
+Machine learning is a subset of artificial intelligence that enables systems to learn and improve
+from experience without being explicitly programmed. It focuses on developing computer programs
+that can access data and use it to learn for themselves.
+
+Types of Machine Learning:
+1. Supervised Learning: Learning with labeled data
+2. Unsupervised Learning: Finding patterns in unlabeled data
+3. Reinforcement Learning: Learning through rewards and penalties
+
+Applications include image recognition, speech processing, and recommendation systems
+
+
+    

Data/python_intro.txt

@@ -0,0 +1,13 @@
+Python Programming Introduction
+
+Python is a high-level, interpreted programming language known for its simplicity and readability.
+Created by Guido van Rossum and first released in 1991, Python has become one of the most popular
+programming languages in the world.
+
+Key Features:
+- Easy to learn and use
+- Extensive standard library
+- Cross-platform compatibility
+- Strong community support
+
+Python is widely used in web development, data science, artificial intelligence, and automation.

Dockerfile

@@ -0,0 +1,64 @@
+# Modern Multi-stage Dockerfile for RAG System
+FROM python:3.11-slim as builder
+
+# Set environment variables
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    POETRY_NO_INTERACTION=1 \
+    POETRY_VENV_IN_PROJECT=1 \
+    POETRY_CACHE_DIR=/tmp/poetry_cache
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential \
+    curl \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install Python dependencies
+WORKDIR /app
+COPY requirements.txt .
+RUN pip install --no-cache-dir --upgrade pip && \
+    pip install --no-cache-dir -r requirements.txt
+
+# Production stage  
+FROM python:3.11-slim as production
+
+# Set environment variables
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PYTHONPATH="/app/src:$PYTHONPATH"
+
+# Install runtime dependencies only
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    curl \
+    && rm -rf /var/lib/apt/lists/*
+
+# Create non-root user
+RUN groupadd -r appuser && useradd -r -g appuser appuser
+
+# Copy Python packages from builder stage
+COPY --from=builder /usr/local/lib/python3.11/site-packages/ /usr/local/lib/python3.11/site-packages/
+COPY --from=builder /usr/local/bin/ /usr/local/bin/
+
+# Set working directory
+WORKDIR /app
+
+# Copy application code
+COPY --chown=appuser:appuser . .
+
+# Create necessary directories
+RUN mkdir -p /app/faiss_store /app/data /app/logs && \
+    chown -R appuser:appuser /app
+
+# Switch to non-root user 
+USER appuser
+
+# Health check
+HEALTHCHECK --interval=30s --timeout=30s --start-period=5s --retries=3 \
+    CMD curl -f http://localhost:8000/health || exit 1
+
+# Expose port
+EXPOSE 8000
+
+# Default command
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8000", "--workers", "1"]

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2026 Jeevant
+Copyright (c) 2025 Jeevant
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -0,0 +1 @@
+# Langchain-Model

app.py

@@ -0,0 +1,130 @@
+import os
+from typing import List, Optional
+from contextlib import asynccontextmanager
+
+from fastapi import FastAPI, HTTPException
+from fastapi.middleware.cors import CORSMiddleware
+from pydantic import BaseModel
+from dotenv import load_dotenv
+
+from src.search import RAGSearch, RetrievalResult
+
+import uvicorn
+
+load_dotenv()
+
+# Global variable for RAG system
+rag_search: Optional[RAGSearch] = None
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    # Startup
+    global rag_search
+    try:
+        persist_dir = os.getenv("PERSIST_DIR", "faiss_store")
+        embedding_model = os.getenv("EMBEDDING_MODEL", "all-MiniLM-L6-v2")
+        llm_model = os.getenv("LLM_MODEL", "llama-3.1-8b-instant")
+
+        rag_search = RAGSearch(
+            persist_dir=persist_dir,
+            embedding_model=embedding_model,
+            llm_model=llm_model,
+        )
+        print("[INFO] RAG system loaded successfully")
+    except Exception as e:
+        print(f"[ERROR] Failed to load RAG system: {e}")
+        raise
+    
+    yield  # Application runs here
+    
+    # Shutdown (cleanup if needed)
+    print("[INFO] Shutting down RAG system")
+
+# -------------------------
+# FastAPI App
+# -------------------------
+app = FastAPI(
+    title="RAG Question Answering API",
+    description="FAISS + SentenceTransformers + Groq LLM",
+    version="2.0.0",
+    lifespan=lifespan
+)
+
+# CORS for React/Node clients
+cors_origins = os.getenv("CORS_ORIGINS", "*").split(",")
+app.add_middleware(     
+    CORSMiddleware,
+    allow_origins=[o.strip() for o in cors_origins] if cors_origins else ["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# -------------------------
+# Request / Response Models
+# -------------------------
+class SourceItem(BaseModel):
+    index: int
+    distance: float
+    text: Optional[str] = None
+
+class QueryRequest(BaseModel):
+    query: str
+    top_k: int = 3
+
+class QueryResponse(BaseModel):
+    query: str
+    answer: str
+    sources: List[SourceItem]
+
+
+# -------------------------
+# Routes
+# -------------------------
+@app.get("/")
+def root():
+    return {"message": "RAG API is running. Go to /docs"}
+
+@app.get("/health")
+def health():
+    if not rag_search:
+        return {"ready": False}
+    meta_count = len(rag_search.vectorstore.metadata) if rag_search.vectorstore else 0
+    return {
+        "ready": True,
+        "persist_dir": rag_search.vectorstore.persist_dir,
+        "documents_indexed": meta_count,
+        "embedding_model": rag_search.embedding_model,
+        "llm_model": rag_search.llm_model,
+    }
+
+@app.post("/query", response_model=QueryResponse)
+def query_rag(payload: QueryRequest):
+    if not rag_search:
+        raise HTTPException(status_code=503, detail="RAG system not ready")
+
+    try:
+        # Retrieve and summarize
+        sources: List[RetrievalResult] = rag_search.retrieve(payload.query, top_k=payload.top_k)
+        answer: str = rag_search.summarize(payload.query, sources)
+
+        # Map sources for response
+        resp_sources = [
+            SourceItem(index=s.index, distance=float(s.distance), text=s.text)
+            for s in sources
+        ]
+        return QueryResponse(query=payload.query, answer=answer, sources=resp_sources)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+# -------------------------
+# Run locally
+# -------------------------
+if __name__ == "__main__":
+    uvicorn.run(
+        "app:app",
+        host="0.0.0.0",
+        port=int(os.getenv("PORT", "8000")),
+        reload=True
+    )

config/config.yaml

@@ -0,0 +1,76 @@
+# Environment-specific Configuration
+environment: "development"
+
+# Database and Storage Settings
+storage:
+  faiss_store_path: "faiss_store"
+  backup_enabled: true
+  backup_frequency: "daily"
+
+# Model Configuration
+models:
+  embedding:
+    provider: "sentence-transformers"
+    model_name: "all-MiniLM-L6-v2"
+    device: "cpu" # or "cuda" if GPU available
+
+  llm:
+    provider: "groq"
+    model_name: "llama-3.1-8b-instant"
+    api_key_env: "GROQ_API_KEY"
+    parameters:
+      temperature: 0.1
+      max_tokens: 1000
+      top_p: 0.9
+
+# Processing Settings
+processing:
+  text_splitter:
+    type: "recursive_character"
+    chunk_size: 1000
+    chunk_overlap: 200
+    separators: ["\n\n", "\n", " ", ""]
+
+  batch_processing:
+    enabled: true
+    batch_size: 100
+
+# API Settings
+api:
+  server:
+    host: "0.0.0.0"
+    port: 8000
+    workers: 1
+    reload: true
+
+  security:
+    cors_enabled: true
+    cors_origins: ["*"]
+    rate_limiting: false
+
+  features:
+    health_check: true
+    metrics: false
+    documentation: true
+
+# Logging and Monitoring
+logging:
+  level: "INFO"
+  file_enabled: false
+  file_path: "logs/app.log"
+  console_enabled: true
+
+monitoring:
+  metrics_enabled: false
+  health_checks: true
+
+# Development Settings
+development:
+  debug: true
+  hot_reload: true
+
+# Production Settings (override in production)
+production:
+  debug: false
+  workers: 4
+  logging_level: "WARNING"

main.py

@@ -0,0 +1,33 @@
+"""
+Simple CLI test runner for RAG retrieval + summarization.
+Run: python main.py
+"""
+
+from src.search import RAGSearch
+import os
+
+if __name__ == "__main__":
+    # Check for GROQ API key
+    if not os.getenv("GROQ_API_KEY"):
+        print("WARNING: GROQ_API_KEY not found in environment variables.")
+        print("Please set it in a .env file or environment variable to use the LLM features.")
+        print("You can still test document loading and embeddings without it.")
+    
+    try:
+        rag_search = RAGSearch(
+            persist_dir="faiss_store",
+            embedding_model="all-MiniLM-L6-v2",
+            llm_model="llama-3.1-8b-instant"
+        )
+        
+        query = "What is Database Management System?"
+        answer = rag_search.search_and_summarize(query=query, top_k=3)
+        print("Query:", query)
+        print("Answer:\n", answer)
+        
+    except Exception as e:
+        print(f"Error: {e}")
+        print("\nTroubleshooting:")
+        print("1. Make sure you have documents in 'Research/data/', 'data/', or 'Data/' directory")
+        print("2. Set GROQ_API_KEY in your .env file")
+        print("3. Run: pip install -r requirements.txt")

params.yaml

@@ -0,0 +1,43 @@
+# RAG System Configuration
+rag:
+  embedding:
+    model: "all-MiniLM-L6-v2"
+    chunk_size: 1000
+    chunk_overlap: 200
+
+  vectorstore:
+    persist_dir: "faiss_store"
+    index_type: "faiss"
+
+  llm:
+    provider: "groq"
+    model: "llama-3.1-8b-instant"
+    temperature: 0.1
+    max_tokens: 1000
+
+  retrieval:
+    top_k: 5
+    similarity_threshold: 0.7
+
+# API Configuration
+api:
+  host: "0.0.0.0"
+  port: 8000
+  cors_origins: "*"
+  reload: true
+
+# Data Configuration
+data:
+  input_dir: "data"
+  supported_formats:
+    - "pdf"
+    - "txt"
+    - "csv"
+    - "xlsx"
+    - "docx"
+    - "json"
+
+# Logging Configuration
+logging:
+  level: "INFO"
+  format: "[%(levelname)s] %(asctime)s - %(name)s - %(message)s"

requirements.txt

@@ -0,0 +1,29 @@
+# Core LangChain packages
+langchain>=0.3.0
+langchain-core>=0.3.0
+langchain-community>=0.3.0
+langchain-text-splitters>=0.3.0
+langchain-groq>=0.2.0
+
+# Document processing
+pypdf>=4.0.1
+pymupdf>=1.24.0
+
+# Embeddings and vector databases
+sentence-transformers>=3.0.0
+faiss-cpu>=1.8.0
+chromadb>=0.5.0
+
+# Search and indexing
+typesense>=0.19.0
+
+# Web framework and API
+fastapi>=0.115.0
+uvicorn[standard]>=0.32.0
+pydantic>=2.9.0
+
+# Environment and utilities
+python-dotenv>=1.0.0
+
+# Build tools (only if needed for development)
+setuptools>=75.0.0

setup.py

@@ -0,0 +1,54 @@
+import setuptools
+
+with open("README.md", "r", encoding="utf-8") as f:
+    long_description = f.read()
+    
+__version__ = "2.0.0"
+
+REPO_NAME = "Langchain_RAG_System"
+AUTHOR_USER_NAME = "jeevant"
+SRC_REPO = "langchain_rag"
+AUTHOR_EMAIL = "jeevantmudgil10@gmail.com"
+
+setuptools.setup(
+    name=SRC_REPO,
+    version=__version__,
+    author=AUTHOR_USER_NAME,
+    author_email=AUTHOR_EMAIL,
+    description="A modern RAG system using LangChain, FAISS, and Groq LLM",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    url=f"https://github.com/{AUTHOR_USER_NAME}/{REPO_NAME}",
+    project_urls={
+        "Bug Tracker": f"https://github.com/{AUTHOR_USER_NAME}/{REPO_NAME}/issues",
+        "Documentation": f"https://github.com/{AUTHOR_USER_NAME}/{REPO_NAME}#readme",
+    },
+    package_dir={"": "src"},
+    packages=setuptools.find_packages(where="src"),
+    python_requires=">=3.8",
+    install_requires=[
+        "langchain>=0.3.0",
+        "langchain-core>=0.3.0",
+        "langchain-community>=0.3.0",
+        "langchain-text-splitters>=0.3.0",
+        "langchain-groq>=0.2.0",
+        "fastapi>=0.115.0",
+        "uvicorn[standard]>=0.32.0",
+        "sentence-transformers>=3.0.0",
+        "faiss-cpu>=1.8.0",
+        "python-dotenv>=1.0.0",
+    ],
+    classifiers=[
+        "Development Status :: 4 - Beta",
+        "Intended Audience :: Developers",
+        "License :: OSI Approved :: MIT License",
+        "Operating System :: OS Independent",
+        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
+        "Topic :: Software Development :: Libraries :: Python Modules",
+        "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    ],
+)

src/data_loader.py

@@ -0,0 +1,157 @@
+from pathlib import Path
+from typing import List, Any
+from langchain_community.document_loaders import PyPDFLoader, TextLoader, CSVLoader
+from langchain_community.document_loaders import Docx2txtLoader
+from langchain_community.document_loaders.excel import UnstructuredExcelLoader
+from langchain_community.document_loaders import JSONLoader
+
+
+def load_all_documents(data_dir: str) -> List[Any] :
+    
+    data_path = Path(data_dir).resolve()
+    print(f"[DEBUG] Data Path : {data_path}")
+    document = []
+    
+    
+    ##  Pdf files
+    
+    pdf_files = list(data_path.glob('**/*.pdf'))
+    print(f"[DEBUG] Found {len(pdf_files)} PDF files : { [str(f) for f in pdf_files] }")
+    for pdf_file in pdf_files:
+        print(f"[DEBUG] Loading PDF: {pdf_file}")
+        try:
+            loader = PyPDFLoader(str(pdf_file))
+            loaded = loader.load()
+            print(f"[DEBUG] Loaded {len(loaded)} PDF docs from {pdf_file}")
+            document.extend(loaded)
+        except Exception as e:
+            print(f"[ERROR] Failed to load PDF {pdf_file} : {e}")
+        
+            
+            
+    ## TXT files
+    
+    txt_files = list(data_path.glob('**/*.txt'))
+    print(f'[DEBUG] Founded {len(txt_files)} TXT files: {[str(f) for f in txt_files]} ')
+    for txt_file in txt_files:
+        print(f"[DEBUG] Loading TXT : {txt_file}")
+        try:
+            loader = TextLoader(str(txt_file), encoding="utf-8")
+            loaded = loader.load()
+            print(f"[DEBUG] Loaded {len(loaded)} TXT docs from {txt_file}")
+            document.extend(loaded)
+        except Exception as e:
+            print(f"[ERROR] failed to load TXT {txt_file} : {e}")
+            
+    
+    
+    ## CSV files
+    
+    
+    csv_files = list(data_path.glob('**/*.csv'))
+    print(f"[DEBUG] Found {len((csv_files))} CSV files : {[str(f) for f in csv_files]}")
+    for csv_file in csv_files:
+        print(f"[DEBUG] Loading CSV : {csv_file}")
+        try :
+            loader = CSVLoader(str(csv_file))
+            loaded = loader.load()
+            print(f"[DEBUG] Loaded {len(loaded)} CSV docs from {csv_file}")
+            document.extend(loaded)
+        except Exception as e:
+            print(f"[ERROR] failed to load CSV {csv_file} : {e}")
+            
+            
+    
+    ## Excel files
+    
+    
+    xlsx_files = list(data_path.glob('**/*.xlsx'))
+    print(f"[DEBUG] Found {len(xlsx_files)} xlsx files : {[str(f) for f in xlsx_files]}")
+    for xlsx_file in xlsx_files:
+        print(f"[DEBUG] Loading XLSX : {xlsx_file}")
+        try :
+            loader = UnstructuredExcelLoader(str(xlsx_file))
+            loaded = loader.load()
+            print(f"[DEBUG] Loaded {(len(loaded))} XLSX docs from {xlsx_file}")
+            document.extend(loaded)
+        except Exception as e :
+            print(f"[ERROR] failed to load Excel {xlsx_file} : {e} ")
+            
+            
+    
+    ## Word file
+    
+    
+    word_files = list(data_path.glob('**/*.docx'))
+    print(f"[DEBUG] Found {len(word_files)} word files : {[str(f) for f in word_files]}")
+    for word_file in word_files:
+        print(f"[DEBUG] Loaded WORD : {word_file}")
+        try :
+            loader = Docx2txtLoader(str(word_file))
+            loaded = loader.load()
+            print(f"[DEBUG] Loaded {(len(loaded))} Word docs from {word_file}")
+            document.extend(loaded)
+        except Exception as e:
+            print(f"[ERROR] failed to load Word Document {word_file} : {e}")
+            
+    
+    
+    
+    ## JSON files
+    
+    
+    json_files = list(data_path.glob('**/*.json'))
+    print(f"[DEBUG] Found {(len(json_files))} json files : {[str(f) for f in json_files]}")
+    for json_file in json_files:
+        print(f"[DEBUG] Loaded JSON : {(str(json_file))}")
+        try:
+            loader = JSONLoader(
+                str(json_file),
+                jq_schema=".",
+                text_content=False
+                )
+            loaded = loader.load()
+            print(f"[DEBUG] Loaded {(len(loaded))} JSON from {json_file}")
+            document.extend(loaded)
+        except Exception as e:
+            print(f"[ERROR] Failed to load JSON file{json_file} : {e}")
+        
+        
+    print(f"[DEBUG] Total Loaded Documents : {len(document)}")
+    return document
+
+
+if __name__ == '__main__':
+    docs = load_all_documents('Research/data/pdf')
+    print(f"Loaded {len(docs)} document.")
+    print("Example document:", docs[0] if docs else None)
+            
+    
+# def process_all_pdf(pdf_directory):
+    
+#     all_documents = []
+#     pdf_dir = Path(pdf_directory)
+    
+#     pdf_files = list(pdf_dir.glob('**/*.pdf'))
+    
+#     print(f"found {len(pdf_files)} PDF files to Process")
+    
+#     for pdf_files in pdf_files:
+#         print(f"\npreprocessing : {pdf_files.name}")
+#         try:
+#             loader = PyPDFLoader(str(pdf_files))
+#             documents = loader.load()
+            
+#             for doc in documents:
+#                 doc.metadata['source_file'] = pdf_files.name
+#                 doc.metadata['file_type'] = 'pdf'
+                
+#             all_documents.extend(documents)
+#             print(f" Loaded {len(documents)} pages")
+#         except Exception as e:
+#             print(f" Error : {e}")
+            
+#     print(f"\n Total documents loaded : {len(all_documents)}")
+#     return all_documents    
+
+# all_pdf_files = process_all_pdf('./data')

src/embedding.py

@@ -0,0 +1,40 @@
+from typing import List, Any, Optional
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from sentence_transformers import SentenceTransformer
+import numpy as np
+from src.data_loader import load_all_documents
+
+class EmbeddingPipeline:
+    def __init__(self, model_name: str = 'all-MiniLM-L6-v2', chunk_size: int = 1000, chunk_overlap: int = 200):
+        self.model_name = model_name
+        self.chunk_size = chunk_size
+        self.chunk_overlap = chunk_overlap
+        self.model = SentenceTransformer(model_name)
+        print(f"[INFO] Loaded embedding model: {model_name}")
+        
+    def chunk_documents(self, documents: List[Any]) -> List[Any]:
+        splitter = RecursiveCharacterTextSplitter(
+            chunk_size=self.chunk_size,
+            chunk_overlap=self.chunk_overlap,
+            length_function=len,
+            separators=["\n\n", "\n", " ", ""]
+        )
+        
+        chunks = splitter.split_documents(documents)
+        print(f"[INFO] Split {len(documents)} documents into {len(chunks)} chunks.")
+        return chunks
+    
+    def embed_chunks(self, chunks: List[Any]) -> np.ndarray:
+        texts = [chunk.page_content for chunk in chunks]
+        print(f"[INFO] Generating embeddings for {len(texts)} chunks...")
+        embeddings = self.model.encode(texts, show_progress_bar = True)
+        print(f"[INFO] Embeddings shape : {embeddings.shape}")
+        return embeddings
+    
+if __name__ == "__main__":
+    docs = load_all_documents('Research/data/pdf')
+    emb_pipe = EmbeddingPipeline()
+    chunks = emb_pipe.chunk_documents(docs)
+    embeddings = emb_pipe.embed_chunks(chunks)
+    print(f"[INFO] Example embeddings:", embeddings[0] if len(embeddings) > 0 else None)
+        

src/search.py

@@ -0,0 +1,114 @@
+import os
+from dataclasses import dataclass
+from typing import List, Optional
+
+from dotenv import load_dotenv
+from langchain_groq import ChatGroq
+from langchain_core.messages import HumanMessage, SystemMessage
+
+from src.vectorstore import FaissVectorStore
+
+load_dotenv()
+
+@dataclass
+class RetrievalResult:
+    index: int
+    distance: float
+    text: Optional[str]
+
+class RAGSearch:
+    def __init__(
+        self,
+        persist_dir: str = "faiss_store",
+        embedding_model: str = "all-MiniLM-L6-v2",
+        llm_model: str = "llama-3.1-8b-instant",
+    ):
+        # Vector store setup
+        self.vectorstore = FaissVectorStore(persist_dir=persist_dir, embedding_model=embedding_model)
+
+        faiss_path = os.path.join(persist_dir, "faiss.index")
+        meta_path = os.path.join(persist_dir, "metadata.pkl")
+
+        if not (os.path.exists(faiss_path) and os.path.exists(meta_path)):
+            # Build from local 'Research/data' directory if index doesn't exist  
+            from src.data_loader import load_all_documents
+            # Try multiple possible data directories
+            data_dirs = ["Research/data", "data", "Data"]
+            docs = []
+            
+            for data_dir in data_dirs:
+                if os.path.exists(data_dir):
+                    print(f"[INFO] Checking for documents in: {data_dir}")
+                    docs = load_all_documents(data_dir)
+                    if docs:
+                        print(f"[INFO] Found {len(docs)} documents in {data_dir}")
+                        break
+                        
+            if not docs:
+                print("[WARNING] No documents found in any data directory. Vector store will be empty.")
+                print("[INFO] Please add documents to 'Research/data/', 'data/', or 'Data/' directory.")
+                # Create empty index for now
+                self.vectorstore.index = None
+                self.vectorstore.metadata = []
+            else:
+                self.vectorstore.build_from_documents(docs)
+        else:
+            self.vectorstore.load()
+
+        # LLM setup
+        groq_api_key = os.getenv("GROQ_API_KEY")
+        if not groq_api_key:
+            raise ValueError("GROQ_API_KEY missing in environment")
+        self.llm_model = llm_model
+        self.embedding_model = embedding_model
+        self.llm = ChatGroq(api_key=groq_api_key, model=llm_model, temperature=0.1)
+        print(f"[INFO] Groq LLM initialized: {llm_model}")
+
+    def retrieve(self, query: str, top_k: int = 5) -> List[RetrievalResult]:
+        # Check if vector store is empty
+        if self.vectorstore.index is None or len(self.vectorstore.metadata) == 0:
+            print("[WARNING] Vector store is empty. No documents to search.")
+            return []
+            
+        results = self.vectorstore.query(query_text=query, top_k=top_k)
+        out: List[RetrievalResult] = []
+        for r in results:
+            text = r["metadata"]["texts"] if r.get("metadata") and r["metadata"].get("texts") else None
+            out.append(RetrievalResult(index=int(r["index"]), distance=float(r["distance"]), text=text))
+        return out
+
+    def summarize(self, query: str, retrieved: List[RetrievalResult]) -> str:
+        if not retrieved:
+            return "No documents are available in the vector store. Please add some documents to the data directory and restart the application."
+            
+        texts = [r.text for r in retrieved if r.text]
+        context = "\n\n".join(texts)
+        if not context:
+            return "No relevant documents found for your query."
+        
+        # Using proper message formatting for better LLM interaction
+        system_message = SystemMessage(content="You are a helpful assistant that summarizes documents based on queries. Provide clear, concise summaries with relevant quotes when appropriate.")
+        human_message = HumanMessage(content=f"""
+Based on the following context, answer the query: '{query}'
+
+Context:
+{context}
+
+Please provide a comprehensive answer based solely on the provided context. If you quote specific information, indicate it clearly.
+""")
+        
+        try:
+            response = self.llm.invoke([system_message, human_message])
+            return response.content
+        except Exception as e:
+            return f"Error generating response: {str(e)}. Please check your GROQ_API_KEY is set correctly."
+
+    def search_and_summarize(self, query: str, top_k: int = 5) -> str:
+        retrieved = self.retrieve(query, top_k=top_k)
+        return self.summarize(query, retrieved)
+
+if __name__ == "__main__":
+    rag_search = RAGSearch()
+    query = "What is Database Management System?"
+    summary = rag_search.search_and_summarize(query, top_k=3)
+    print("Summary:", summary)

src/vectorstore.py

@@ -0,0 +1,103 @@
+import os
+import faiss
+import numpy as np
+import pickle
+from typing import List, Any
+from sentence_transformers import SentenceTransformer
+from src.embedding import EmbeddingPipeline
+
+class FaissVectorStore:
+    def __init__(
+        self,
+        persist_dir: str = "faiss_store",
+        embedding_model: str = "all-MiniLM-L6-v2",
+        chunk_size: int = 1000,
+        chunk_overlap: int = 200,
+    ):
+        self.persist_dir = persist_dir
+        os.makedirs(self.persist_dir, exist_ok=True)
+        self.index = None
+        self.metadata = []
+        self.embedding_model = embedding_model
+        self.model = SentenceTransformer(embedding_model)
+        self.chunk_size = chunk_size
+        self.chunk_overlap = chunk_overlap
+        print(f"[INFO] Loaded embedding model: {embedding_model}")
+
+    def build_from_documents(self, documents: List[Any]):
+        print(f"[INFO] Building vector store from {len(documents)} raw document(s)...")
+        emb_pipe = EmbeddingPipeline(
+            model_name=self.embedding_model,
+            chunk_size=self.chunk_size,
+            chunk_overlap=self.chunk_overlap,
+        )
+        chunks = emb_pipe.chunk_documents(documents)
+        embeddings = emb_pipe.embed_chunks(chunks)
+        metadatas = [{"texts": chunk.page_content} for chunk in chunks]
+        self.add_embeddings(np.array(embeddings).astype("float32"), metadatas)
+        self.save()
+        print(f"[INFO] Vector Store built and saved to {self.persist_dir}")
+
+    def add_embeddings(self, embeddings: np.ndarray, metadatas: List[Any] = None):
+        # Handle empty embeddings case
+        if embeddings.size == 0:
+            print("[WARNING] No embeddings to add. Vector store remains empty.")
+            return
+            
+        dim = embeddings.shape[1]
+        if self.index is None:
+            self.index = faiss.IndexFlatL2(dim)
+        self.index.add(embeddings)
+        if metadatas:
+            self.metadata.extend(metadatas)
+        print(f"[INFO] Added {embeddings.shape[0]} vectors to Faiss Index.")
+
+    def save(self):
+        if self.index is None:
+            print("[WARNING] Cannot save: index is empty. Skipping save operation.")
+            return
+        faiss_path = os.path.join(self.persist_dir, "faiss.index")
+        meta_path = os.path.join(self.persist_dir, "metadata.pkl")
+        faiss.write_index(self.index, faiss_path)
+        with open(meta_path, "wb") as f:
+            pickle.dump(self.metadata, f)
+        print(f"[INFO] Saved Faiss index and metadata to {self.persist_dir}")
+
+    def load(self):
+        faiss_path = os.path.join(self.persist_dir, "faiss.index")
+        meta_path = os.path.join(self.persist_dir, "metadata.pkl")
+        if not (os.path.exists(faiss_path) and os.path.exists(meta_path)):
+            raise FileNotFoundError(f"Missing index/metadata in {self.persist_dir}. Build the store first.")
+        self.index = faiss.read_index(faiss_path)
+        with open(meta_path, "rb") as f:
+            self.metadata = pickle.load(f)
+        print(f"[INFO] Loaded Faiss Index and metadata from {self.persist_dir}")
+
+    def search(self, query_embeddings: np.ndarray, top_k: int = 5):
+        if self.index is None:
+            print("[WARNING] Vector store is empty. No results to return.")
+            return []
+            
+        D, I = self.index.search(query_embeddings, top_k)
+        results = []
+        for idx, dist in zip(I[0], D[0]):
+            meta = self.metadata[idx] if idx < len(self.metadata) else None
+            results.append({"index": int(idx), "distance": float(dist), "metadata": meta})
+        return results
+
+    def query(self, query_text: str, top_k: int = 5):
+        if self.index is None:
+            print("[WARNING] Vector store is empty. No results to return.")
+            return []
+            
+        print(f"[INFO] Querying vector store for: '{query_text}'")
+        query_emb = self.model.encode([query_text]).astype("float32")
+        return self.search(query_emb, top_k=top_k)
+
+if __name__ == "__main__":
+    from src.data_loader import load_all_documents
+    docs = load_all_documents("data")
+    store = FaissVectorStore("faiss_store")
+    store.build_from_documents(docs)
+    store.load()
+    print(store.query("What is Database Management System?", top_k=3))

temp.py

@@ -0,0 +1,40 @@
+import os
+from pathlib import Path
+import logging
+
+logging.basicConfig(level=logging.INFO, format='[%(asctime)s]:(message)s:')
+
+project_name = "testSummarizer"
+
+list_of_files = [
+    ".github/workflows/.gitkeep",
+    f"src/data_loader.py",
+    f"src/embedding.py",
+    f"src/__init__.py",
+    f"src/search.py",
+    f"src/vectorstore.py",
+    "config/config.yaml",
+    "params.yaml",
+    "app.py",
+    "main.py",
+    "Dockerfile",
+    "setup.py",
+    "research/trails.ipynb"
+    
+]
+
+for filepath in list_of_files:
+    filepath = Path(filepath)
+    filedir, filename = os.path.split(filepath)
+    
+    if filedir != "":
+        os.makedirs(filedir, exist_ok=True)
+        logging.info(f"Creating directry:{filepath} for the file {filepath}")
+        
+    if (not os.path.exists(filepath)) or (os.path.getsize(filepath) == 0):
+        with open(filepath, 'w') as f:
+            pass
+            logging.info(f"Creating empty file: {filepath}")
+            
+    else :
+        logging.info(f"{filename} is already exists.")

test_updates.py

@@ -0,0 +1,123 @@
+#!/usr/bin/env python3
+"""
+Test script to verify the updated Langchain RAG system works correctly.
+"""
+import sys
+import os
+import traceback
+
+def test_imports():
+    """Test that all required imports work."""
+    print("Testing imports...")
+    try:
+        # Core imports
+        from src.search import RAGSearch, RetrievalResult
+        from src.embedding import EmbeddingPipeline
+        from src.vectorstore import FaissVectorStore
+        from src.data_loader import load_all_documents
+        
+        # External dependencies
+        import langchain
+        import langchain_core
+        import langchain_community
+        import langchain_text_splitters
+        import sentence_transformers
+        import faiss
+        import pydantic
+        import fastapi
+        import uvicorn
+        
+        print("✅ All imports successful!")
+        
+        # Print versions
+        print(f"\nPackage versions:")
+        print(f"- LangChain: {langchain.__version__}")
+        print(f"- LangChain Core: {langchain_core.__version__}")
+        print(f"- Sentence Transformers: {sentence_transformers.__version__}")
+        print(f"- Pydantic: {pydantic.__version__}")
+        print(f"- FastAPI: {fastapi.__version__}")
+        
+        return True
+    except Exception as e:
+        print(f"❌ Import failed: {e}")
+        traceback.print_exc()
+        return False
+
+def test_basic_functionality():
+    """Test basic functionality without requiring API keys."""
+    print("\nTesting basic functionality...")
+    try:
+        from src.embedding import EmbeddingPipeline
+        from src.vectorstore import FaissVectorStore
+        
+        # Test embedding pipeline initialization
+        embedding_pipeline = EmbeddingPipeline()
+        print("✅ Embedding pipeline initialized")
+        
+        # Test vector store initialization
+        vector_store = FaissVectorStore(persist_dir="test_store", embedding_model="all-MiniLM-L6-v2")
+        print("✅ Vector store initialized")
+        
+        return True
+    except Exception as e:
+        print(f"❌ Basic functionality test failed: {e}")
+        traceback.print_exc()
+        return False
+
+def test_config_loading():
+    """Test configuration file loading."""
+    print("\nTesting configuration loading...")
+    try:
+        import yaml
+        
+        # Test params.yaml
+        if os.path.exists("params.yaml"):
+            with open("params.yaml", "r") as f:
+                params = yaml.safe_load(f)
+            print("✅ params.yaml loaded successfully")
+            
+        # Test config.yaml
+        if os.path.exists("config/config.yaml"):
+            with open("config/config.yaml", "r") as f:
+                config = yaml.safe_load(f)
+            print("✅ config/config.yaml loaded successfully")
+            
+        return True
+    except Exception as e:
+        print(f"❌ Config loading failed: {e}")
+        return False
+
+def main():
+    """Run all tests."""
+    print("=== Updated Langchain RAG System Test ===\n")
+    
+    all_passed = True
+    
+    # Test imports
+    if not test_imports():
+        all_passed = False
+    
+    # Test basic functionality  
+    if not test_basic_functionality():
+        all_passed = False
+    
+    # Test config loading
+    if not test_config_loading():
+        all_passed = False
+    
+    print(f"\n{'='*50}")
+    if all_passed:
+        print("🎉 All tests passed! Your project is successfully updated!")
+        print("\nNext steps:")
+        print("1. Set your GROQ_API_KEY in a .env file")
+        print("2. Place your documents in the 'data' directory")
+        print("3. Run: python main.py")
+        print("4. Or start the API: python app.py")
+    else:
+        print("❌ Some tests failed. Please check the errors above.")
+        return 1
+    
+    return 0
+
+if __name__ == "__main__":
+    sys.exit(main())