Add RAG system code, API, requirements, folder structure, and placeholder documents

.gitattributes

@@ -34,3 +34,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 documents/meeting_notes.docx filter=lfs diff=lfs merge=lfs -text
+documents/reports/meeting_notes.docx filter=lfs diff=lfs merge=lfs -text

app.py

@@ -1 +1,210 @@
-# Your RAG and API code from the previous steps should be copied here.
+
+import torch
+from transformers import T5ForConditionalGeneration, T5Tokenizer
+from sentence_transformers import SentenceTransformer
+from sklearn.metrics.pairwise import cosine_similarity
+from fastapi import FastAPI, File, UploadFile, HTTPException
+from pydantic import BaseModel
+import os
+import shutil
+import PyPDF2
+import docx
+import math
+
+# --- RAG System Code ---
+
+# Define functions for document loading
+def load_document(file_path):
+    """Loads text content from various document types."""
+    if file_path.endswith(".pdf"):
+        return load_pdf(file_path)
+    elif file_path.endswith(".docx"):
+        return load_docx(file_path)
+    elif file_path.endswith(".txt"):
+        return load_txt(file_path)
+    else:
+        print(f"Unsupported file type: {file_path}")
+        return None
+
+def load_pdf(file_path):
+    """Loads text from a PDF file."""
+    text = ""
+    try:
+        with open(file_path, 'rb') as file:
+            reader = PyPDF2.PdfReader(file)
+            for page_num in range(len(reader.pages)):
+                text += reader.pages[page_num].extract_text()
+        return text
+    except Exception as e:
+        print(f"Error loading PDF {file_path}: {e}")
+        return None
+
+def load_docx(file_path):
+    """Loads text from a DOCX file."""
+    text = ""
+    try:
+        doc = docx.Document(file_path)
+        for paragraph in doc.paragraphs:
+            text += paragraph.text + "\n"
+        return text
+    except Exception as e:
+        print(f"Error loading DOCX {file_path}: {e}")
+        return None
+
+def load_txt(file_path):
+    """Loads text from a TXT file."""
+    try:
+        with open(file_path, 'r', encoding='utf-8') as file:
+            text = file.read()
+        return text
+    except Exception as e:
+        print(f"Error loading TXT {file_path}: {e}")
+        return None
+
+# Implement a text chunking function
+def chunk_text(text, chunk_size=500, overlap=50):
+    """Splits text into smaller chunks."""
+    chunks = []
+    start = 0
+    while start < len(text):
+        end = start + chunk_size
+        chunk = text[start:end]
+        chunks.append(chunk)
+        start += chunk_size - overlap
+        if start >= len(text): # Handle the last chunk
+            break
+    return chunks
+
+# Load the chosen embedding model and language model
+embedding_model_name = "sentence-transformers/all-MiniLM-L6-v2"
+language_model_name = "google/flan-t5-small"
+
+# Load models outside of the request handlers to avoid reloading on each request
+embedding_model = SentenceTransformer(embedding_model_name)
+tokenizer = T5Tokenizer.from_pretrained(language_model_name)
+language_model = T5ForConditionalGeneration.from_pretrained(language_model_name)
+
+# Create a function to generate embeddings
+def generate_embeddings(texts):
+    """Generates embeddings for a list of text chunks."""
+    return embedding_model.encode(texts, convert_to_tensor=True)
+
+# Implement a similarity search function
+def find_similar_chunks(query_embedding, chunk_embeddings, top_k=3):
+    """Finds the top_k most similar chunks to the query."""
+    # Ensure top_k is not greater than the number of available chunks
+    actual_top_k = min(top_k, chunk_embeddings.size(0))
+    if actual_top_k == 0:
+        return [] # Return empty list if no chunks available
+
+    similarities = cosine_similarity(query_embedding.unsqueeze(0), chunk_embeddings)[0]
+    # Use torch.topk for efficiency
+    top_k_indices = torch.topk(torch.tensor(similarities), int(actual_top_k)).indices.tolist()
+    return top_k_indices
+
+# Create a function to generate a response
+def generate_response(query, context_chunks):
+    """Generates a response based on the query and context."""
+    context = " ".join(context_chunks)
+    prompt = f"Context: {context}\n\nQuestion: {query}\n\nAnswer:"
+    inputs = tokenizer(prompt, return_tensors="pt", max_length=512, truncation=True)
+    outputs = language_model.generate(**inputs, max_length=150, num_return_sequences=1)
+    response = tokenizer.decode(outputs[0], skip_special_tokens=True)
+    return response
+
+# --- FastAPI App Code ---
+
+# Initialize FastAPI app
+app = FastAPI()
+
+# Global variable to store chunks and embeddings
+# This will act as our in-memory vector store for the API's current session
+document_chunks = []
+chunk_embeddings = None
+
+# Define request model for question answering
+class QueryRequest(BaseModel):
+    query: str
+
+# API endpoint for document ingestion
+@app.post("/ingest-document/")
+async def ingest_document(file: UploadFile = File(...)):
+    global document_chunks, chunk_embeddings
+
+    # Create a temporary directory to save the uploaded file
+    upload_folder = "temp_uploads"
+    os.makedirs(upload_folder, exist_ok=True)
+    file_path = os.path.join(upload_folder, file.filename)
+
+    try:
+        # Save the uploaded file
+        with open(file_path, "wb") as f:
+            shutil.copyfileobj(file.file, f)
+
+        # Load and process the document
+        document_text = load_document(file_path)
+
+        if document_text is None:
+            raise HTTPException(status_code=400, detail="Unsupported file type or error loading document.")
+
+        # Chunk the text
+        chunks = chunk_text(document_text)
+
+        if not chunks:
+             raise HTTPException(status_code=400, detail="No text extracted or document is empty.")
+
+        # Generate embeddings for the chunks
+        new_chunk_embeddings = generate_embeddings(chunks)
+
+        # Append new chunks and embeddings to the global storage
+        document_chunks.extend(chunks)
+        if chunk_embeddings is None:
+            chunk_embeddings = new_chunk_embeddings
+        else:
+            # Ensure embeddings are on the same device if applicable (e.g., CPU)
+            chunk_embeddings = torch.cat((chunk_embeddings.to(new_chunk_embeddings.device), new_chunk_embeddings), dim=0)
+
+
+        return {"message": f"Successfully ingested {len(chunks)} chunks from {file.filename}"}
+
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"An error occurred during document ingestion: {e}")
+    finally:
+        # Clean up the temporary file and directory
+        if os.path.exists(upload_folder):
+             shutil.rmtree(upload_folder)
+
+
+# API endpoint for answering questions
+@app.post("/answer-query/")
+async def answer_query(query_request: QueryRequest):
+    global document_chunks, chunk_embeddings
+
+    if not document_chunks or chunk_embeddings is None:
+        raise HTTPException(status_code=400, detail="No documents have been ingested yet. Please ingest a document first.")
+
+    try:
+        # Generate embedding for the query
+        query_embedding = generate_embeddings([query_request.query])[0]
+
+        # Find relevant chunks
+        relevant_chunk_indices = find_similar_chunks(query_embedding, chunk_embeddings)
+
+        if not relevant_chunk_indices:
+            return {"answer": "Could not find relevant information in the ingested documents."}
+
+        relevant_chunks = [document_chunks[i] for i in relevant_chunk_indices]
+
+        # Generate response
+        response = generate_response(query_request.query, relevant_chunks)
+
+        return {"answer": response}
+
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"An error occurred during query processing: {e}")
+
+# Add a root endpoint for testing
+@app.get("/")
+async def read_root():
+    return {"message": "NORA AI Agent API is running. Use /ingest-document/ to upload documents and /answer-query/ to ask questions."}
+

documents/general/research_benefits.txt

@@ -0,0 +1,4 @@
+This document discusses the benefits of using AI in research.
+AI can help analyze large datasets and identify patterns.
+It can also automate tedious tasks, speeding up the research process.
+Benefits include increased efficiency and accuracy.

documents/reports/meeting_notes.docx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58b116ed5c74f3f0f7f1afb1f8d3599416af4e73972e6504e92f4e04c77c1ba6
+size 36710

documents/technical_docs/project_x_summary.txt

@@ -0,0 +1,5 @@
+Technical Documentation Summary:
+This document outlines the technical specifications of Project X.
+It includes details about the software architecture and deployment process.
+Key components are the data ingestion module and the API endpoint.
+Deployment is planned for a cloud-based platform.

requirements.txt

@@ -10,7 +10,6 @@ fastapi>=0.95.0
 uvicorn>=0.21.0
 pypdf2>=3.0.0
 python-docx>=0.8.11
-textract>=1.6.0
 faiss-cpu>=1.7.0
 chromadb>=0.3.0
 tiktoken>=0.4.0