Add document summary endpoint and enhance vector database functionality

- Introduced `get_all_texts` method in `VectorDatabase` to retrieve all stored text documents.
- Added new `DocumentSummaryRequest` and `DocumentSummaryResponse` models for handling document summary requests in the FastAPI application.
- Implemented `/document-summary` endpoint to generate structured summaries from uploaded documents, including key topics, entities, word cloud data, and document structure.
- Enhanced frontend to integrate document summary display and settings management for improved user experience.
- Updated global CSS with styles for word cloud visualization.

aimakerspace/vectordatabase.py

@@ -53,6 +53,15 @@ class VectorDatabase:
             self.insert(text, np.array(embedding))
         return self
 
+    def get_all_texts(self) -> List[str]:
+        """
+        Returns all the text documents stored in the vector database.
+        
+        Returns:
+            List[str]: A list of all text documents
+        """
+        return list(self.vectors.keys())
+
 
 if __name__ == "__main__":
     list_of_text = [

api/main.py

@@ -1,6 +1,7 @@
 import os
 import tempfile
 import shutil
+import json
 from typing import List
 from fastapi import FastAPI, UploadFile, File, Form, HTTPException
 from fastapi.middleware.cors import CORSMiddleware
@@ -58,6 +59,15 @@ class QueryResponse(BaseModel):
     response: str
     session_id: str
 
+class DocumentSummaryRequest(BaseModel):
+    session_id: str
+
+class DocumentSummaryResponse(BaseModel):
+    keyTopics: List[str]
+    entities: List[str]
+    wordCloudData: List[dict]
+    documentStructure: List[dict]
+
 @app.post("/upload")
 async def upload_file(file: UploadFile = File(...), session_id: str = Form(...)):
     if file.content_type not in ["text/plain", "application/pdf"]:
@@ -172,6 +182,108 @@ async def query(request: QueryRequest):
     
     return {"response": response_text, "session_id": session_id}
 
+@app.post("/document-summary", response_model=DocumentSummaryResponse)
+async def get_document_summary(request: DocumentSummaryRequest):
+    session_id = request.session_id
+    
+    # Check if session exists
+    if session_id not in user_sessions:
+        raise HTTPException(status_code=404, detail="Session not found. Please upload a document first.")
+    
+    # Get the retrieval pipeline from the session
+    retrieval_pipeline = user_sessions[session_id]
+    
+    # Get access to the document content
+    vector_db = retrieval_pipeline.vector_db_retriever
+    
+    # We'll use all the text chunks to create a comprehensive summary
+    # Get all text chunks from the vector store
+    all_texts = vector_db.get_all_texts()
+    
+    # Combine a sample of the texts (to avoid hitting token limits)
+    sample_texts = all_texts[:10] if len(all_texts) > 10 else all_texts
+    doc_content = "\n".join(sample_texts)
+    
+    # Create the LLM summary prompt
+    summary_prompt = f"""
+    Analyze the following document content and generate a structured summary in JSON format:
+    
+    ```
+    {doc_content}
+    ```
+    
+    Return ONLY a JSON object with the following structure:
+    
+    {{
+      "keyTopics": [list of 5-7 key topics in the document],
+      "entities": [list of 5-8 important named entities such as organizations, technologies, or people],
+      "wordCloudData": [
+        {{ "text": "word1", "value": frequency_score }},
+        {{ "text": "word2", "value": frequency_score }},
+        ...
+      ],
+      "documentStructure": [
+        {{ 
+          "title": "Section title",
+          "subsections": ["Subsection1", "Subsection2", ...] 
+        }},
+        ...
+      ]
+    }}
+    
+    The wordCloudData should contain 15-20 important terms with their relative frequency scores (higher numbers = more important/frequent).
+    The documentStructure should reflect the hierarchical organization of the document with main sections and their subsections.
+    """
+    
+    # Get LLM response
+    try:
+        llm = retrieval_pipeline.llm
+        response = await llm.acreate_single_response(summary_prompt)
+        
+        # Parse the JSON
+        # Find JSON content (sometimes the LLM adds extra text)
+        import re
+        json_match = re.search(r'({[\s\S]*})', response)
+        
+        if json_match:
+            json_str = json_match.group(1)
+            summary_data = json.loads(json_str)
+        else:
+            # If no JSON found, create a basic structure with an error message
+            summary_data = {
+                "keyTopics": ["Error parsing document structure"],
+                "entities": ["Please try again"],
+                "wordCloudData": [{"text": "Error", "value": 50}],
+                "documentStructure": [{"title": "Document structure unavailable", "subsections": []}]
+            }
+            
+        # Ensure the response has all required fields
+        if "keyTopics" not in summary_data:
+            summary_data["keyTopics"] = ["Topic extraction failed"]
+        if "entities" not in summary_data:
+            summary_data["entities"] = ["Entity extraction failed"]
+        if "wordCloudData" not in summary_data:
+            summary_data["wordCloudData"] = [{"text": "Data", "value": 50}]
+        if "documentStructure" not in summary_data:
+            summary_data["documentStructure"] = [{"title": "Structure unavailable", "subsections": []}]
+            
+        return summary_data
+        
+    except Exception as e:
+        # Return a fallback summary on error
+        return {
+            "keyTopics": ["Error analyzing document"],
+            "entities": ["Try refreshing the page"],
+            "wordCloudData": [
+                {"text": "Error", "value": 60},
+                {"text": "Document", "value": 40},
+                {"text": "Analysis", "value": 30}
+            ],
+            "documentStructure": [
+                {"title": "Error in document analysis", "subsections": ["Please try again"]}
+            ]
+        }
+
 # Serve the frontend
 @app.get("/")
 async def read_root():

app/frontend/package.json

@@ -8,6 +8,7 @@
     "@radix-ui/react-label": "^2.1.3",
     "@radix-ui/react-separator": "^1.1.3",
     "@radix-ui/react-slot": "^1.2.0",
+    "@radix-ui/react-switch": "^1.0.3",
     "@radix-ui/react-toast": "^1.2.7",
     "@testing-library/jest-dom": "^5.16.5",
     "@testing-library/react": "^13.4.0",

app/frontend/src/App.js

@@ -3,8 +3,10 @@ import { v4 as uuidv4 } from 'uuid';
 import FileUpload from './components/FileUpload';
 import FileManager from './components/FileManager';
 import Chat from './components/Chat';
+import DocumentSummary from './components/DocumentSummary';
 import { ThemeProvider } from './components/ui/theme-provider';
 import { ThemeToggle } from './components/ui/theme-toggle';
+import { SettingsDialog } from './components/ui/settings-dialog';
 
 function App() {
   const [sessionId, setSessionId] = useState('');
@@ -12,6 +14,13 @@ function App() {
   const [uploadedFiles, setUploadedFiles] = useState([]);
   const [activeFileIndex, setActiveFileIndex] = useState(0);
   const [selectedQuestion, setSelectedQuestion] = useState('');
+  
+  // App settings with localStorage persistence
+  const [settings, setSettings] = useState(() => {
+    // Initialize from localStorage, default to true if not set
+    const saved = localStorage.getItem('appSettings');
+    return saved ? JSON.parse(saved) : { showDashboard: true };
+  });
 
   // Get active file data
   const activeFile = uploadedFiles[activeFileIndex] || null;
@@ -23,6 +32,15 @@ function App() {
     }
   }, [sessionId]);
 
+  // Save settings to localStorage when they change
+  useEffect(() => {
+    localStorage.setItem('appSettings', JSON.stringify(settings));
+  }, [settings]);
+
+  const handleSettingsChange = (newSettings) => {
+    setSettings(newSettings);
+  };
+
   const handleFileUploadSuccess = (fileName, description, questions) => {
     // Add the new file to the uploaded files array
     setUploadedFiles(prev => [
@@ -65,7 +83,15 @@ function App() {
               <span role="img" aria-label="brain" className="text-2xl">🧠</span>
               <h1 className="text-2xl font-bold">RAG Chat</h1>
             </div>
-            <ThemeToggle />
+            <div className="flex items-center gap-4">
+              {!showUploadForm && uploadedFiles.length > 0 && (
+                <SettingsDialog 
+                  settings={settings}
+                  onSettingsChange={handleSettingsChange}
+                />
+              )}
+              <ThemeToggle />
+            </div>
           </div>
         </header>
         
@@ -97,6 +123,13 @@ function App() {
                   <p className="text-sm text-muted-foreground line-clamp-3">{activeFile.description}</p>
                 </div>
                 
+                {settings.showDashboard && 
+                  <DocumentSummary 
+                    fileName={activeFile.name} 
+                    sessionId={activeFile.sessionId} 
+                  />
+                }
+                
                 <div className="min-h-[100px] mb-4">
                   {activeFile.suggestedQuestions && activeFile.suggestedQuestions.length > 0 && (
                     <div className="space-y-2 bg-card p-4 rounded-md border border-border transition-colors duration-300 h-full">

app/frontend/src/components/DocumentSummary.js

@@ -0,0 +1,225 @@
+import React, { useState, useEffect } from 'react';
+import { Button } from './ui/button';
+import axios from 'axios';
+
+// Fallback data in case of API failure
+const fallbackData = {
+  keyTopics: ['Document analysis unavailable'],
+  entities: ['Try refreshing the page'],
+  wordCloudData: [
+    { text: 'Document', value: 50 },
+    { text: 'Summary', value: 40 },
+    { text: 'Unavailable', value: 30 },
+  ],
+  documentStructure: [
+    { title: 'Document structure unavailable', subsections: ['Please refresh the page or try a different document'] }
+  ]
+};
+
+const DocumentSummary = ({ fileName, sessionId }) => {
+  const [activeTab, setActiveTab] = useState('overview');
+  const [summaryData, setSummaryData] = useState(null);
+  const [loading, setLoading] = useState(true);
+  const [error, setError] = useState(null);
+  
+  useEffect(() => {
+    const fetchDocumentSummary = async () => {
+      if (!sessionId) return;
+      
+      setLoading(true);
+      setError(null);
+      
+      try {
+        const response = await axios.post('/document-summary', {
+          session_id: sessionId
+        });
+        
+        setSummaryData(response.data);
+      } catch (err) {
+        console.error('Error fetching document summary:', err);
+        setError('Failed to load document summary. Please try again.');
+        setSummaryData(fallbackData);
+      } finally {
+        setLoading(false);
+      }
+    };
+    
+    fetchDocumentSummary();
+  }, [sessionId, fileName]);
+
+  if (loading) {
+    return (
+      <div className="bg-card border border-border rounded-lg p-4 mb-6 transition-colors duration-300">
+        <h2 className="text-lg font-semibold mb-4 flex items-center gap-2">
+          <span role="img" aria-label="chart" className="text-xl">📊</span>
+          Document Summary Dashboard
+        </h2>
+        <div className="py-8 text-center">
+          <div className="animate-pulse flex flex-col items-center">
+            <div className="h-4 bg-muted-foreground/20 rounded w-1/4 mb-4"></div>
+            <div className="h-2 bg-muted-foreground/20 rounded w-1/2 mb-2"></div>
+            <div className="h-2 bg-muted-foreground/20 rounded w-1/3 mb-2"></div>
+            <div className="h-2 bg-muted-foreground/20 rounded w-2/5 mb-2"></div>
+            <div className="mt-4 text-sm text-muted-foreground">Generating document insights...</div>
+          </div>
+        </div>
+      </div>
+    );
+  }
+
+  if (error && !summaryData) {
+    return (
+      <div className="bg-card border border-border rounded-lg p-4 mb-6 transition-colors duration-300">
+        <h2 className="text-lg font-semibold mb-4 flex items-center gap-2">
+          <span role="img" aria-label="chart" className="text-xl">📊</span>
+          Document Summary Dashboard
+        </h2>
+        <div className="py-4 text-center text-destructive">
+          <span role="img" aria-label="error" className="text-2xl block mb-2">⚠️</span>
+          {error}
+        </div>
+      </div>
+    );
+  }
+
+  if (!summaryData) {
+    return (
+      <div className="bg-card border border-border rounded-lg p-4 mb-6 transition-colors duration-300">
+        <h2 className="text-lg font-semibold mb-4 flex items-center gap-2">
+          <span role="img" aria-label="chart" className="text-xl">📊</span>
+          Document Summary Dashboard
+        </h2>
+        <div className="py-4 text-center">No document summary available</div>
+      </div>
+    );
+  }
+
+  return (
+    <div className="bg-card border border-border rounded-lg p-4 mb-6 transition-colors duration-300">
+      <h2 className="text-lg font-semibold mb-4 flex items-center gap-2">
+        <span role="img" aria-label="chart" className="text-xl">📊</span>
+        Document Summary Dashboard
+      </h2>
+      
+      <div className="mb-4 border-b border-border">
+        <div className="flex space-x-2">
+          <Button 
+            variant={activeTab === 'overview' ? "default" : "ghost"}
+            size="sm"
+            onClick={() => setActiveTab('overview')}
+            className="rounded-b-none"
+          >
+            Overview
+          </Button>
+          <Button 
+            variant={activeTab === 'wordcloud' ? "default" : "ghost"}
+            size="sm"
+            onClick={() => setActiveTab('wordcloud')}
+            className="rounded-b-none"
+          >
+            Word Cloud
+          </Button>
+          <Button 
+            variant={activeTab === 'structure' ? "default" : "ghost"}
+            size="sm"
+            onClick={() => setActiveTab('structure')}
+            className="rounded-b-none"
+          >
+            Document Structure
+          </Button>
+        </div>
+      </div>
+      
+      {activeTab === 'overview' && (
+        <div className="space-y-4">
+          <div>
+            <h3 className="text-sm font-medium mb-2">Key Topics</h3>
+            <div className="flex flex-wrap gap-2">
+              {summaryData.keyTopics.map((topic, idx) => (
+                <span 
+                  key={idx} 
+                  className="bg-primary/10 text-primary px-3 py-1 rounded-full text-sm"
+                >
+                  {topic}
+                </span>
+              ))}
+            </div>
+          </div>
+          
+          <div>
+            <h3 className="text-sm font-medium mb-2">Key Entities</h3>
+            <div className="flex flex-wrap gap-2">
+              {summaryData.entities.map((entity, idx) => (
+                <span 
+                  key={idx} 
+                  className="bg-secondary/10 text-secondary-foreground px-3 py-1 rounded-full text-sm"
+                >
+                  {entity}
+                </span>
+              ))}
+            </div>
+          </div>
+        </div>
+      )}
+      
+      {activeTab === 'wordcloud' && (
+        <div className="py-4">
+          <div className="bg-muted rounded-lg p-4 min-h-[200px] flex items-center justify-center">
+            <div className="word-cloud-container">
+              {summaryData.wordCloudData.map((word, idx) => {
+                // Calculate font size based on value (scale from 14 to 36)
+                const fontSize = 14 + (word.value - 16) * (22 / 48);
+                const opacity = 0.5 + (word.value / 64) * 0.5;
+                
+                return (
+                  <span 
+                    key={idx} 
+                    className="inline-block px-2 py-1 m-1 cursor-pointer hover:text-primary transition-colors duration-200"
+                    style={{ 
+                      fontSize: `${fontSize}px`, 
+                      opacity,
+                      transform: `rotate(${Math.random() * 20 - 10}deg)`
+                    }}
+                  >
+                    {word.text}
+                  </span>
+                );
+              })}
+            </div>
+          </div>
+          <p className="text-xs text-muted-foreground mt-2 text-center">
+            Word size represents frequency in the document. Click on any word to search for it.
+          </p>
+        </div>
+      )}
+      
+      {activeTab === 'structure' && (
+        <div className="py-4">
+          <div className="bg-muted rounded-lg p-4 min-h-[200px]">
+            <ul className="space-y-2">
+              {summaryData.documentStructure.map((section, idx) => (
+                <li key={idx} className="border-l-2 border-primary pl-3 py-1">
+                  <div className="font-medium">{section.title}</div>
+                  {section.subsections.length > 0 && (
+                    <ul className="ml-4 mt-1 space-y-1">
+                      {section.subsections.map((subsection, subIdx) => (
+                        <li key={subIdx} className="border-l-2 border-secondary pl-3 py-1 text-sm">
+                          {subsection}
+                        </li>
+                      ))}
+                    </ul>
+                  )}
+                </li>
+              ))}
+            </ul>
+          </div>
+          <p className="text-xs text-muted-foreground mt-2 text-center">
+            Document structure extracted from headings and content organization.
+          </p>
+        </div>
+      )}
+    </div>
+  );
+};
+
+export default DocumentSummary; 

app/frontend/src/components/ui/dialog.js

@@ -0,0 +1,94 @@
+import * as React from "react"
+import * as DialogPrimitive from "@radix-ui/react-dialog"
+import { X } from "lucide-react"
+
+import { cn } from "./utils"
+
+const Dialog = DialogPrimitive.Root
+
+const DialogTrigger = DialogPrimitive.Trigger
+
+const DialogPortal = DialogPrimitive.Portal
+
+const DialogClose = DialogPrimitive.Close
+
+const DialogOverlay = React.forwardRef(({ className, ...props }, ref) => (
+  <DialogPrimitive.Overlay
+    ref={ref}
+    className={cn(
+      "fixed inset-0 z-50 bg-black/80 data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0",
+      className
+    )}
+    {...props} />
+))
+DialogOverlay.displayName = DialogPrimitive.Overlay.displayName
+
+const DialogContent = React.forwardRef(({ className, children, ...props }, ref) => (
+  <DialogPortal>
+    <DialogOverlay />
+    <DialogPrimitive.Content
+      ref={ref}
+      className={cn(
+        "fixed left-[50%] top-[50%] z-50 grid w-full max-w-lg translate-x-[-50%] translate-y-[-50%] gap-4 border bg-background p-6 shadow-lg duration-200 data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95 data-[state=closed]:slide-out-to-left-1/2 data-[state=closed]:slide-out-to-top-[48%] data-[state=open]:slide-in-from-left-1/2 data-[state=open]:slide-in-from-top-[48%] sm:rounded-lg",
+        className
+      )}
+      {...props}>
+      {children}
+      <DialogPrimitive.Close
+        className="absolute right-4 top-4 rounded-sm opacity-70 ring-offset-background transition-opacity hover:opacity-100 focus:outline-none focus:ring-2 focus:ring-ring focus:ring-offset-2 disabled:pointer-events-none data-[state=open]:bg-accent data-[state=open]:text-muted-foreground">
+        <X className="h-4 w-4" />
+        <span className="sr-only">Close</span>
+      </DialogPrimitive.Close>
+    </DialogPrimitive.Content>
+  </DialogPortal>
+))
+DialogContent.displayName = DialogPrimitive.Content.displayName
+
+const DialogHeader = ({
+  className,
+  ...props
+}) => (
+  <div
+    className={cn("flex flex-col space-y-1.5 text-center sm:text-left", className)}
+    {...props} />
+)
+DialogHeader.displayName = "DialogHeader"
+
+const DialogFooter = ({
+  className,
+  ...props
+}) => (
+  <div
+    className={cn("flex flex-col-reverse sm:flex-row sm:justify-end sm:space-x-2", className)}
+    {...props} />
+)
+DialogFooter.displayName = "DialogFooter"
+
+const DialogTitle = React.forwardRef(({ className, ...props }, ref) => (
+  <DialogPrimitive.Title
+    ref={ref}
+    className={cn("text-lg font-semibold leading-none tracking-tight", className)}
+    {...props} />
+))
+DialogTitle.displayName = DialogPrimitive.Title.displayName
+
+const DialogDescription = React.forwardRef(({ className, ...props }, ref) => (
+  <DialogPrimitive.Description
+    ref={ref}
+    className={cn("text-sm text-muted-foreground", className)}
+    {...props} />
+))
+DialogDescription.displayName = DialogPrimitive.Description.displayName
+
+export {
+  Dialog,
+  DialogPortal,
+  DialogOverlay,
+  DialogClose,
+  DialogTrigger,
+  DialogContent,
+  DialogHeader,
+  DialogFooter,
+  DialogTitle,
+  DialogDescription,
+} 

app/frontend/src/components/ui/label.js

@@ -0,0 +1,19 @@
+import * as React from "react"
+import * as LabelPrimitive from "@radix-ui/react-label"
+import { cva } from "class-variance-authority"
+
+import { cn } from "./utils"
+
+const labelVariants = cva(
+  "text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70"
+)
+
+const Label = React.forwardRef(({ className, ...props }, ref) => (
+  <LabelPrimitive.Root
+    ref={ref}
+    className={cn(labelVariants(), className)}
+    {...props} />
+))
+Label.displayName = LabelPrimitive.Root.displayName
+
+export { Label } 

app/frontend/src/components/ui/settings-dialog.js

@@ -0,0 +1,65 @@
+import React from 'react';
+import { Button } from './button';
+import { 
+  Dialog,
+  DialogContent,
+  DialogDescription,
+  DialogFooter,
+  DialogHeader,
+  DialogTitle,
+  DialogTrigger
+} from './dialog';
+import { Switch } from './switch';
+import { Label } from './label';
+
+export function SettingsDialog({ settings, onSettingsChange }) {
+  const handleDashboardToggle = (checked) => {
+    onSettingsChange({
+      ...settings,
+      showDashboard: checked
+    });
+  };
+
+  return (
+    <Dialog>
+      <DialogTrigger asChild>
+        <Button 
+          variant="ghost" 
+          size="sm"
+          className="flex items-center gap-1"
+        >
+          <span role="img" aria-label="settings" className="text-lg">⚙️</span>
+          <span className="hidden sm:inline">Settings</span>
+        </Button>
+      </DialogTrigger>
+      <DialogContent className="sm:max-w-[425px]">
+        <DialogHeader>
+          <DialogTitle>Application Settings</DialogTitle>
+          <DialogDescription>
+            Customize your document analysis experience
+          </DialogDescription>
+        </DialogHeader>
+        <div className="py-4 space-y-4">
+          <div className="flex items-center justify-between">
+            <div className="space-y-0.5">
+              <Label htmlFor="dashboard-toggle">Document Dashboard</Label>
+              <p className="text-sm text-muted-foreground">
+                Show document summary with topics and structure visualization
+              </p>
+            </div>
+            <Switch 
+              id="dashboard-toggle"
+              checked={settings.showDashboard}
+              onCheckedChange={handleDashboardToggle}
+            />
+          </div>
+        </div>
+        <DialogFooter>
+          <p className="text-sm text-muted-foreground mr-auto">
+            Settings are saved automatically
+          </p>
+        </DialogFooter>
+      </DialogContent>
+    </Dialog>
+  );
+} 

app/frontend/src/components/ui/switch.js

@@ -0,0 +1,22 @@
+import * as React from "react"
+import * as SwitchPrimitives from "@radix-ui/react-switch"
+
+import { cn } from "./utils"
+
+const Switch = React.forwardRef(({ className, ...props }, ref) => (
+  <SwitchPrimitives.Root
+    className={cn(
+      "peer inline-flex h-6 w-11 shrink-0 cursor-pointer items-center rounded-full border-2 border-transparent transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 focus-visible:ring-offset-background disabled:cursor-not-allowed disabled:opacity-50 data-[state=checked]:bg-primary data-[state=unchecked]:bg-input",
+      className
+    )}
+    {...props}
+    ref={ref}>
+    <SwitchPrimitives.Thumb
+      className={cn(
+        "pointer-events-none block h-5 w-5 rounded-full bg-background shadow-lg ring-0 transition-transform data-[state=checked]:translate-x-5 data-[state=unchecked]:translate-x-0"
+      )} />
+  </SwitchPrimitives.Root>
+))
+Switch.displayName = SwitchPrimitives.Root.displayName
+
+export { Switch } 

app/frontend/src/globals.css

@@ -219,4 +219,29 @@
 button, a {
   backface-visibility: hidden;
   transform: translateZ(0);
+}
+
+/* Word Cloud Styles */
+.word-cloud-container {
+  display: flex;
+  flex-wrap: wrap;
+  justify-content: center;
+  align-items: center;
+  width: 100%;
+  height: 100%;
+  text-align: center;
+}
+
+.word-cloud-container span {
+  display: inline-block;
+  padding: 4px 8px;
+  margin: 4px;
+  border-radius: 4px;
+  transition: all 0.2s ease;
+}
+
+.word-cloud-container span:hover {
+  color: hsl(var(--primary));
+  transform: scale(1.1);
+  z-index: 1;
 }