Implement complete document upload and processing pipeline with Modal integration

✅ Real Document Upload:
- Created SQLite storage replacing in-memory storage with full document metadata
- Added file upload API with multer supporting PDFs, images, text files (50MB limit)
- Implemented document processing pipeline with Modal integration for heavy workloads

✅ PDF/Image Processing Pipeline:
- Modal integration for OCR text extraction using PyPDF2 and Tesseract
- Batch processing capabilities for multiple documents concurrently
- Document status tracking (pending -> processing -> completed/failed)

✅ Vector Embeddings Storage & Search:
- Real embeddings generation using Nebius AI and storage in SQLite
- Vector index building using Modal's FAISS implementation
- High-performance vector search with relevance scoring

✅ Database Storage:
- SQLite storage with proper schema for file metadata, processing status
- Document lifecycle management with file path tracking
- Search history and results persistence

✅ Batch Processing:
- Modal distributed computing for processing large document collections
- Index building from processed documents with configurable parameters
- Concurrent document processing with proper error handling

✅ Frontend Components:
- Document upload UI with drag-and-drop, progress tracking, file validation
- Vector search interface with comparison mode vs traditional search
- Document management with processing status and batch operations
- Integration into main knowledge base with new tabs

🔧 Heavy Workloads Now Supported:
- OCR processing for PDFs and images via Modal
- FAISS vector index building for large document collections
- Distributed embedding generation and similarity search
- Batch document processing with 2-4GB memory allocation per task

The Modal integration is now actively used for real computational workloads instead of being theoretical.

client/src/components/knowledge-base/document-upload.tsx

@@ -0,0 +1,489 @@
+import React, { useState, useCallback } from 'react';
+import { Upload, FileText, Image, FileCode, Loader2, CheckCircle, XCircle, AlertCircle } from 'lucide-react';
+import { Button } from '@/components/ui/button';
+import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card';
+import { Progress } from '@/components/ui/progress';
+import { Badge } from '@/components/ui/badge';
+import { Alert, AlertDescription } from '@/components/ui/alert';
+import { Input } from '@/components/ui/input';
+import { Label } from '@/components/ui/label';
+
+interface UploadedFile {
+  id: string;
+  file: File;
+  status: 'pending' | 'uploading' | 'processing' | 'completed' | 'failed';
+  progress: number;
+  documentId?: number;
+  error?: string;
+}
+
+interface Document {
+  id: number;
+  title: string;
+  fileName: string;
+  fileSize: number;
+  mimeType: string;
+  processingStatus: string;
+  createdAt: string;
+}
+
+export default function DocumentUpload() {
+  const [files, setFiles] = useState<UploadedFile[]>([]);
+  const [isDragging, setIsDragging] = useState(false);
+  const [uploadTitle, setUploadTitle] = useState('');
+  const [uploadSource, setUploadSource] = useState('');
+  const [documents, setDocuments] = useState<Document[]>([]);
+  const [isProcessing, setIsProcessing] = useState(false);
+
+  const acceptedTypes = [
+    'application/pdf',
+    'image/jpeg',
+    'image/png', 
+    'image/gif',
+    'image/webp',
+    'text/plain',
+    'text/markdown',
+    'application/json',
+    'application/msword',
+    'application/vnd.openxmlformats-officedocument.wordprocessingml.document'
+  ];
+
+  const getFileIcon = (mimeType: string) => {
+    if (mimeType.startsWith('image/')) return <Image className="w-5 h-5" />;
+    if (mimeType === 'application/pdf') return <FileText className="w-5 h-5" />;
+    if (mimeType.includes('text') || mimeType.includes('json')) return <FileCode className="w-5 h-5" />;
+    return <FileText className="w-5 h-5" />;
+  };
+
+  const getStatusIcon = (status: string) => {
+    switch (status) {
+      case 'completed':
+        return <CheckCircle className="w-4 h-4 text-green-500" />;
+      case 'failed':
+        return <XCircle className="w-4 h-4 text-red-500" />;
+      case 'processing':
+      case 'uploading':
+        return <Loader2 className="w-4 h-4 text-blue-500 animate-spin" />;
+      default:
+        return <AlertCircle className="w-4 h-4 text-yellow-500" />;
+    }
+  };
+
+  const formatFileSize = (bytes: number): string => {
+    if (bytes === 0) return '0 Bytes';
+    const k = 1024;
+    const sizes = ['Bytes', 'KB', 'MB', 'GB'];
+    const i = Math.floor(Math.log(bytes) / Math.log(k));
+    return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
+  };
+
+  const handleDragOver = useCallback((e: React.DragEvent) => {
+    e.preventDefault();
+    setIsDragging(true);
+  }, []);
+
+  const handleDragLeave = useCallback((e: React.DragEvent) => {
+    e.preventDefault();
+    setIsDragging(false);
+  }, []);
+
+  const handleDrop = useCallback((e: React.DragEvent) => {
+    e.preventDefault();
+    setIsDragging(false);
+    
+    const droppedFiles = Array.from(e.dataTransfer.files);
+    handleFiles(droppedFiles);
+  }, []);
+
+  const handleFileSelect = (e: React.ChangeEvent<HTMLInputElement>) => {
+    if (e.target.files) {
+      const selectedFiles = Array.from(e.target.files);
+      handleFiles(selectedFiles);
+    }
+  };
+
+  const handleFiles = (newFiles: File[]) => {
+    const validFiles = newFiles.filter(file => {
+      if (!acceptedTypes.includes(file.type)) {
+        alert(`File type ${file.type} is not supported`);
+        return false;
+      }
+      if (file.size > 50 * 1024 * 1024) { // 50MB limit
+        alert(`File ${file.name} is too large. Maximum size is 50MB`);
+        return false;
+      }
+      return true;
+    });
+
+    const uploadFiles: UploadedFile[] = validFiles.map(file => ({
+      id: `${Date.now()}-${Math.random()}`,
+      file,
+      status: 'pending',
+      progress: 0
+    }));
+
+    setFiles(prev => [...prev, ...uploadFiles]);
+  };
+
+  const uploadFiles = async () => {
+    const pendingFiles = files.filter(f => f.status === 'pending');
+    if (pendingFiles.length === 0) return;
+
+    for (const uploadFile of pendingFiles) {
+      try {
+        // Update status to uploading
+        setFiles(prev => prev.map(f => 
+          f.id === uploadFile.id 
+            ? { ...f, status: 'uploading', progress: 0 }
+            : f
+        ));
+
+        const formData = new FormData();
+        formData.append('files', uploadFile.file);
+        if (uploadTitle) formData.append('title', uploadTitle);
+        if (uploadSource) formData.append('source', uploadSource);
+
+        const response = await fetch('/api/documents/upload', {
+          method: 'POST',
+          body: formData,
+        });
+
+        if (!response.ok) {
+          throw new Error(`Upload failed: ${response.statusText}`);
+        }
+
+        const result = await response.json();
+        
+        if (result.success && result.documents?.length > 0) {
+          const document = result.documents[0];
+          
+          // Update file status to processing
+          setFiles(prev => prev.map(f => 
+            f.id === uploadFile.id 
+              ? { 
+                  ...f, 
+                  status: 'processing', 
+                  progress: 100,
+                  documentId: document.id 
+                }
+              : f
+          ));
+
+          // Add to documents list
+          setDocuments(prev => [document, ...prev]);
+
+          // If file requires processing, start processing
+          if (document.processingStatus === 'pending') {
+            await processDocument(document.id, uploadFile.id);
+          } else {
+            // Mark as completed if no processing needed
+            setFiles(prev => prev.map(f => 
+              f.id === uploadFile.id 
+                ? { ...f, status: 'completed' }
+                : f
+            ));
+          }
+        } else {
+          throw new Error(result.message || 'Upload failed');
+        }
+
+      } catch (error) {
+        console.error('Upload error:', error);
+        setFiles(prev => prev.map(f => 
+          f.id === uploadFile.id 
+            ? { 
+                ...f, 
+                status: 'failed', 
+                error: error instanceof Error ? error.message : 'Upload failed'
+              }
+            : f
+        ));
+      }
+    }
+  };
+
+  const processDocument = async (documentId: number, fileId: string) => {
+    try {
+      const response = await fetch(`/api/documents/process/${documentId}`, {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json'
+        },
+        body: JSON.stringify({
+          operations: ['extract_text', 'generate_embedding']
+        })
+      });
+
+      const result = await response.json();
+
+      if (result.success) {
+        // Update file status to completed
+        setFiles(prev => prev.map(f => 
+          f.id === fileId 
+            ? { ...f, status: 'completed' }
+            : f
+        ));
+
+        // Update document in list
+        setDocuments(prev => prev.map(doc => 
+          doc.id === documentId 
+            ? { ...doc, processingStatus: 'completed' }
+            : doc
+        ));
+      } else {
+        throw new Error(result.message || 'Processing failed');
+      }
+
+    } catch (error) {
+      console.error('Processing error:', error);
+      setFiles(prev => prev.map(f => 
+        f.id === fileId 
+          ? { 
+              ...f, 
+              status: 'failed', 
+              error: error instanceof Error ? error.message : 'Processing failed'
+            }
+          : f
+      ));
+    }
+  };
+
+  const batchProcessDocuments = async () => {
+    const completedDocuments = documents.filter(doc => doc.processingStatus === 'completed');
+    if (completedDocuments.length === 0) {
+      alert('No processed documents available for batch operations');
+      return;
+    }
+
+    setIsProcessing(true);
+    
+    try {
+      // Build vector index
+      const response = await fetch('/api/documents/index/build', {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json'
+        },
+        body: JSON.stringify({
+          documentIds: completedDocuments.map(doc => doc.id),
+          indexName: 'main_index'
+        })
+      });
+
+      const result = await response.json();
+
+      if (result.success) {
+        alert(`Vector index built successfully with ${result.documentCount} documents`);
+      } else {
+        throw new Error(result.message || 'Index building failed');
+      }
+
+    } catch (error) {
+      console.error('Batch processing error:', error);
+      alert(`Batch processing failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
+    } finally {
+      setIsProcessing(false);
+    }
+  };
+
+  const removeFile = (fileId: string) => {
+    setFiles(prev => prev.filter(f => f.id !== fileId));
+  };
+
+  const clearCompleted = () => {
+    setFiles(prev => prev.filter(f => f.status !== 'completed' && f.status !== 'failed'));
+  };
+
+  return (
+    <div className="space-y-6">
+      {/* Upload Area */}
+      <Card>
+        <CardHeader>
+          <CardTitle className="flex items-center gap-2">
+            <Upload className="w-5 h-5" />
+            Document Upload
+          </CardTitle>
+        </CardHeader>
+        <CardContent className="space-y-4">
+          {/* Optional metadata */}
+          <div className="grid grid-cols-2 gap-4">
+            <div>
+              <Label htmlFor="title">Title (optional)</Label>
+              <Input
+                id="title"
+                placeholder="Document title"
+                value={uploadTitle}
+                onChange={(e) => setUploadTitle(e.target.value)}
+              />
+            </div>
+            <div>
+              <Label htmlFor="source">Source (optional)</Label>
+              <Input
+                id="source"
+                placeholder="Document source"
+                value={uploadSource}
+                onChange={(e) => setUploadSource(e.target.value)}
+              />
+            </div>
+          </div>
+
+          {/* Drag and drop area */}
+          <div
+            className={`
+              border-2 border-dashed rounded-lg p-8 text-center transition-colors
+              ${isDragging 
+                ? 'border-blue-500 bg-blue-50 dark:bg-blue-950' 
+                : 'border-gray-300 hover:border-gray-400 dark:border-gray-600'
+              }
+            `}
+            onDragOver={handleDragOver}
+            onDragLeave={handleDragLeave}
+            onDrop={handleDrop}
+          >
+            <Upload className="w-12 h-12 mx-auto text-gray-400 mb-4" />
+            <p className="text-lg font-medium mb-2">
+              Drop files here or click to browse
+            </p>
+            <p className="text-sm text-gray-500 mb-4">
+              Supports PDF, images, text files, Word documents (max 50MB each)
+            </p>
+            <input
+              type="file"
+              multiple
+              accept={acceptedTypes.join(',')}
+              onChange={handleFileSelect}
+              className="hidden"
+              id="file-upload"
+            />
+            <Button asChild variant="outline">
+              <label htmlFor="file-upload" className="cursor-pointer">
+                Browse Files
+              </label>
+            </Button>
+          </div>
+
+          {/* File list */}
+          {files.length > 0 && (
+            <div className="space-y-2">
+              <div className="flex justify-between items-center">
+                <h3 className="font-medium">Files ({files.length})</h3>
+                <div className="space-x-2">
+                  <Button onClick={uploadFiles} size="sm" disabled={!files.some(f => f.status === 'pending')}>
+                    Upload All
+                  </Button>
+                  <Button onClick={clearCompleted} variant="outline" size="sm">
+                    Clear Completed
+                  </Button>
+                </div>
+              </div>
+
+              {files.map((file) => (
+                <div key={file.id} className="flex items-center justify-between p-3 bg-gray-50 dark:bg-gray-800 rounded-lg">
+                  <div className="flex items-center gap-3 flex-1">
+                    {getFileIcon(file.file.type)}
+                    <div className="flex-1 min-w-0">
+                      <p className="font-medium truncate">{file.file.name}</p>
+                      <p className="text-sm text-gray-500">
+                        {formatFileSize(file.file.size)}
+                      </p>
+                    </div>
+                  </div>
+
+                  <div className="flex items-center gap-3">
+                    <Badge variant={
+                      file.status === 'completed' ? 'default' :
+                      file.status === 'failed' ? 'destructive' : 
+                      'secondary'
+                    }>
+                      {file.status}
+                    </Badge>
+                    
+                    {getStatusIcon(file.status)}
+                    
+                    {(file.status === 'uploading' || file.status === 'processing') && (
+                      <div className="w-24">
+                        <Progress value={file.progress} className="h-2" />
+                      </div>
+                    )}
+
+                    {(file.status === 'pending' || file.status === 'failed') && (
+                      <Button 
+                        variant="ghost" 
+                        size="sm"
+                        onClick={() => removeFile(file.id)}
+                      >
+                        <XCircle className="w-4 h-4" />
+                      </Button>
+                    )}
+                  </div>
+                </div>
+              ))}
+            </div>
+          )}
+
+          {/* Error alerts */}
+          {files.some(f => f.error) && (
+            <Alert variant="destructive">
+              <AlertCircle className="w-4 h-4" />
+              <AlertDescription>
+                Some files failed to upload. Check individual file status above.
+              </AlertDescription>
+            </Alert>
+          )}
+        </CardContent>
+      </Card>
+
+      {/* Document Management */}
+      {documents.length > 0 && (
+        <Card>
+          <CardHeader>
+            <CardTitle className="flex items-center justify-between">
+              <span>Uploaded Documents ({documents.length})</span>
+              <Button 
+                onClick={batchProcessDocuments}
+                disabled={isProcessing || documents.filter(d => d.processingStatus === 'completed').length === 0}
+              >
+                {isProcessing ? (
+                  <>
+                    <Loader2 className="w-4 h-4 mr-2 animate-spin" />
+                    Building Index...
+                  </>
+                ) : (
+                  'Build Vector Index'
+                )}
+              </Button>
+            </CardTitle>
+          </CardHeader>
+          <CardContent>
+            <div className="space-y-2">
+              {documents.map((doc) => (
+                <div key={doc.id} className="flex items-center justify-between p-3 border rounded-lg">
+                  <div className="flex items-center gap-3 flex-1">
+                    {getFileIcon(doc.mimeType)}
+                    <div className="flex-1 min-w-0">
+                      <p className="font-medium truncate">{doc.title}</p>
+                      <p className="text-sm text-gray-500">
+                        {doc.fileName} • {formatFileSize(doc.fileSize)}
+                      </p>
+                    </div>
+                  </div>
+
+                  <div className="flex items-center gap-3">
+                    <Badge variant={
+                      doc.processingStatus === 'completed' ? 'default' :
+                      doc.processingStatus === 'failed' ? 'destructive' : 
+                      'secondary'
+                    }>
+                      {doc.processingStatus}
+                    </Badge>
+                    {getStatusIcon(doc.processingStatus)}
+                  </div>
+                </div>
+              ))}
+            </div>
+          </CardContent>
+        </Card>
+      )}
+    </div>
+  );
+}

client/src/components/knowledge-base/vector-search.tsx

@@ -0,0 +1,344 @@
+import React, { useState } from 'react';
+import { Search, Zap, Database, Loader2, ArrowRight } from 'lucide-react';
+import { Button } from '@/components/ui/button';
+import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card';
+import { Input } from '@/components/ui/input';
+import { Label } from '@/components/ui/label';
+import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select';
+import { Badge } from '@/components/ui/badge';
+import { Alert, AlertDescription } from '@/components/ui/alert';
+import { Separator } from '@/components/ui/separator';
+
+interface VectorSearchResult {
+  id: string;
+  title: string;
+  content: string;
+  source: string;
+  relevanceScore: number;
+  rank: number;
+  snippet: string;
+}
+
+interface SearchResults {
+  success: boolean;
+  query: string;
+  indexName: string;
+  results: VectorSearchResult[];
+  totalFound: number;
+  searchTime?: number;
+  error?: string;
+}
+
+export default function VectorSearch() {
+  const [query, setQuery] = useState('');
+  const [indexName, setIndexName] = useState('main_index');
+  const [maxResults, setMaxResults] = useState(10);
+  const [isSearching, setIsSearching] = useState(false);
+  const [searchResults, setSearchResults] = useState<SearchResults | null>(null);
+  const [comparisonMode, setComparisonMode] = useState(false);
+  const [traditionalResults, setTraditionalResults] = useState<any>(null);
+
+  const handleVectorSearch = async () => {
+    if (!query.trim()) return;
+
+    setIsSearching(true);
+    try {
+      const response = await fetch('/api/documents/search/vector', {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json'
+        },
+        body: JSON.stringify({
+          query: query.trim(),
+          indexName,
+          maxResults
+        })
+      });
+
+      const result = await response.json();
+      setSearchResults(result);
+
+      // If comparison mode is enabled, also run traditional search
+      if (comparisonMode) {
+        await runTraditionalSearch();
+      }
+
+    } catch (error) {
+      console.error('Vector search error:', error);
+      setSearchResults({
+        success: false,
+        query: query.trim(),
+        indexName,
+        results: [],
+        totalFound: 0,
+        error: error instanceof Error ? error.message : 'Search failed'
+      });
+    } finally {
+      setIsSearching(false);
+    }
+  };
+
+  const runTraditionalSearch = async () => {
+    try {
+      const response = await fetch('/api/search', {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json'
+        },
+        body: JSON.stringify({
+          query: query.trim(),
+          searchType: 'keyword',
+          limit: maxResults,
+          offset: 0
+        })
+      });
+
+      const result = await response.json();
+      setTraditionalResults(result);
+
+    } catch (error) {
+      console.error('Traditional search error:', error);
+      setTraditionalResults(null);
+    }
+  };
+
+  const handleKeyPress = (e: React.KeyboardEvent) => {
+    if (e.key === 'Enter' && !isSearching) {
+      handleVectorSearch();
+    }
+  };
+
+  const formatRelevanceScore = (score: number): string => {
+    return (score * 100).toFixed(1) + '%';
+  };
+
+  const getScoreColor = (score: number): string => {
+    if (score >= 0.8) return 'bg-green-100 text-green-800 dark:bg-green-900 dark:text-green-200';
+    if (score >= 0.6) return 'bg-blue-100 text-blue-800 dark:bg-blue-900 dark:text-blue-200';
+    if (score >= 0.4) return 'bg-yellow-100 text-yellow-800 dark:bg-yellow-900 dark:text-yellow-200';
+    return 'bg-gray-100 text-gray-800 dark:bg-gray-900 dark:text-gray-200';
+  };
+
+  return (
+    <div className="space-y-6">
+      {/* Search Interface */}
+      <Card>
+        <CardHeader>
+          <CardTitle className="flex items-center gap-2">
+            <Zap className="w-5 h-5 text-blue-500" />
+            Vector Search (Modal + FAISS)
+          </CardTitle>
+        </CardHeader>
+        <CardContent className="space-y-4">
+          {/* Search Input */}
+          <div className="space-y-2">
+            <Label htmlFor="vector-query">Search Query</Label>
+            <div className="flex gap-2">
+              <Input
+                id="vector-query"
+                placeholder="Enter your search query for semantic similarity matching..."
+                value={query}
+                onChange={(e) => setQuery(e.target.value)}
+                onKeyPress={handleKeyPress}
+                className="flex-1"
+              />
+              <Button 
+                onClick={handleVectorSearch}
+                disabled={isSearching || !query.trim()}
+              >
+                {isSearching ? (
+                  <Loader2 className="w-4 h-4 animate-spin" />
+                ) : (
+                  <Search className="w-4 h-4" />
+                )}
+              </Button>
+            </div>
+          </div>
+
+          {/* Search Options */}
+          <div className="grid grid-cols-3 gap-4">
+            <div>
+              <Label htmlFor="index-name">Vector Index</Label>
+              <Select value={indexName} onValueChange={setIndexName}>
+                <SelectTrigger>
+                  <SelectValue />
+                </SelectTrigger>
+                <SelectContent>
+                  <SelectItem value="main_index">Main Index</SelectItem>
+                  <SelectItem value="test_index">Test Index</SelectItem>
+                  <SelectItem value="academic_index">Academic Papers</SelectItem>
+                </SelectContent>
+              </Select>
+            </div>
+
+            <div>
+              <Label htmlFor="max-results">Max Results</Label>
+              <Select value={maxResults.toString()} onValueChange={(value) => setMaxResults(parseInt(value))}>
+                <SelectTrigger>
+                  <SelectValue />
+                </SelectTrigger>
+                <SelectContent>
+                  <SelectItem value="5">5 results</SelectItem>
+                  <SelectItem value="10">10 results</SelectItem>
+                  <SelectItem value="20">20 results</SelectItem>
+                  <SelectItem value="50">50 results</SelectItem>
+                </SelectContent>
+              </Select>
+            </div>
+
+            <div className="flex items-end">
+              <Button
+                variant="outline"
+                onClick={() => setComparisonMode(!comparisonMode)}
+                className="w-full"
+              >
+                <Database className="w-4 h-4 mr-2" />
+                {comparisonMode ? 'Comparison: ON' : 'Compare with Keyword'}
+              </Button>
+            </div>
+          </div>
+
+          {/* Search Info */}
+          <Alert>
+            <Database className="w-4 h-4" />
+            <AlertDescription>
+              Vector search uses Modal.com's distributed FAISS implementation for high-performance semantic similarity matching.
+              Enable comparison mode to see differences between vector and traditional keyword search.
+            </AlertDescription>
+          </Alert>
+        </CardContent>
+      </Card>
+
+      {/* Search Results */}
+      {searchResults && (
+        <div className="grid grid-cols-1 gap-6">
+          {/* Vector Search Results */}
+          <Card>
+            <CardHeader>
+              <div className="flex items-center justify-between">
+                <CardTitle className="flex items-center gap-2">
+                  <Zap className="w-5 h-5 text-blue-500" />
+                  Vector Search Results
+                </CardTitle>
+                <div className="flex items-center gap-4 text-sm text-gray-500">
+                  {searchResults.searchTime && (
+                    <span>Search time: {(searchResults.searchTime * 1000).toFixed(0)}ms</span>
+                  )}
+                  <Badge variant="outline">
+                    {searchResults.totalFound} results found
+                  </Badge>
+                </div>
+              </div>
+            </CardHeader>
+            <CardContent>
+              {searchResults.success ? (
+                <div className="space-y-4">
+                  {searchResults.results.length === 0 ? (
+                    <div className="text-center py-8 text-gray-500">
+                      <Database className="w-12 h-12 mx-auto mb-4 opacity-50" />
+                      <p>No results found in vector index.</p>
+                      <p className="text-sm">Try uploading and processing some documents first.</p>
+                    </div>
+                  ) : (
+                    searchResults.results.map((result, index) => (
+                      <div key={result.id} className="border rounded-lg p-4 hover:bg-gray-50 dark:hover:bg-gray-800 transition-colors">
+                        <div className="flex items-start justify-between mb-2">
+                          <div className="flex items-center gap-3">
+                            <Badge variant="outline" className="text-xs">
+                              #{result.rank}
+                            </Badge>
+                            <Badge className={getScoreColor(result.relevanceScore)}>
+                              {formatRelevanceScore(result.relevanceScore)}
+                            </Badge>
+                          </div>
+                          <span className="text-xs text-gray-500">ID: {result.id}</span>
+                        </div>
+                        
+                        <h3 className="font-semibold text-lg mb-2">{result.title}</h3>
+                        
+                        <p className="text-gray-600 dark:text-gray-300 mb-3 leading-relaxed">
+                          {result.snippet}
+                        </p>
+                        
+                        <div className="flex items-center justify-between text-sm text-gray-500">
+                          <span>{result.source}</span>
+                          <ArrowRight className="w-4 h-4" />
+                        </div>
+                      </div>
+                    ))
+                  )}
+                </div>
+              ) : (
+                <Alert variant="destructive">
+                  <AlertDescription>
+                    {searchResults.error || 'Vector search failed'}
+                  </AlertDescription>
+                </Alert>
+              )}
+            </CardContent>
+          </Card>
+
+          {/* Traditional Search Results (if comparison mode) */}
+          {comparisonMode && traditionalResults && (
+            <Card>
+              <CardHeader>
+                <CardTitle className="flex items-center gap-2">
+                  <Search className="w-5 h-5 text-gray-500" />
+                  Traditional Search Results (for comparison)
+                </CardTitle>
+              </CardHeader>
+              <CardContent>
+                {traditionalResults.results && traditionalResults.results.length > 0 ? (
+                  <div className="space-y-4">
+                    {traditionalResults.results.slice(0, maxResults).map((result: any, index: number) => (
+                      <div key={result.id} className="border rounded-lg p-4 opacity-75">
+                        <div className="flex items-start justify-between mb-2">
+                          <Badge variant="outline" className="text-xs">
+                            #{index + 1}
+                          </Badge>
+                          <Badge variant="secondary">
+                            {formatRelevanceScore(result.relevanceScore || 0)}
+                          </Badge>
+                        </div>
+                        
+                        <h3 className="font-semibold text-lg mb-2">{result.title}</h3>
+                        
+                        <p className="text-gray-600 dark:text-gray-300 mb-3 leading-relaxed">
+                          {result.snippet}
+                        </p>
+                        
+                        <div className="text-sm text-gray-500">
+                          {result.source}
+                        </div>
+                      </div>
+                    ))}
+                  </div>
+                ) : (
+                  <div className="text-center py-4 text-gray-500">
+                    <p>No traditional search results found.</p>
+                  </div>
+                )}
+              </CardContent>
+            </Card>
+          )}
+        </div>
+      )}
+
+      {/* Help Text */}
+      {!searchResults && (
+        <Card>
+          <CardContent className="pt-6">
+            <div className="text-center text-gray-500 space-y-2">
+              <Database className="w-16 h-16 mx-auto opacity-50" />
+              <h3 className="text-lg font-medium">Advanced Vector Search</h3>
+              <p className="text-sm max-w-md mx-auto">
+                Search through your documents using semantic similarity powered by Modal.com's distributed FAISS implementation.
+                Upload documents first to build your vector index.
+              </p>
+            </div>
+          </CardContent>
+        </Card>
+      )}
+    </div>
+  );
+}

client/src/pages/knowledge-base.tsx

@@ -5,6 +5,8 @@ import SearchResults from "@/components/knowledge-base/search-results";
 import CitationPanel from "@/components/knowledge-base/citation-panel";
 import SystemFlowDiagram from "@/components/knowledge-base/system-flow-diagram";
 import { KnowledgeGraph } from "@/components/knowledge-base/knowledge-graph";
+import DocumentUpload from "@/components/knowledge-base/document-upload";
+import VectorSearch from "@/components/knowledge-base/vector-search";
 import { ThemeToggle } from "@/components/theme-toggle";
 import { type SearchRequest, type SearchResponse, type Citation } from "@shared/schema";
 import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
@@ -180,9 +182,11 @@ export default function KnowledgeBase() {
         </div>
 
         <Tabs defaultValue="search" className="w-full">
-          <TabsList className="grid w-full grid-cols-3 mb-6">
+          <TabsList className="grid w-full grid-cols-5 mb-6">
             <TabsTrigger value="search">🔍 AI-Enhanced Search</TabsTrigger>
-            <TabsTrigger value="flow">⚡ System Flow</TabsTrigger>
+            <TabsTrigger value="upload">📄 Document Upload</TabsTrigger>
+            <TabsTrigger value="vector">⚡ Vector Search</TabsTrigger>
+            <TabsTrigger value="flow">🔧 System Flow</TabsTrigger>
             <TabsTrigger value="graph">🕸️ Knowledge Graph</TabsTrigger>
           </TabsList>
 
@@ -212,6 +216,15 @@ export default function KnowledgeBase() {
             />
           </TabsContent>
 
+          {/* Document Upload */}
+          <TabsContent value="upload">
+            <DocumentUpload />
+          </TabsContent>
+
+          {/* Vector Search */}
+          <TabsContent value="vector">
+            <VectorSearch />
+          </TabsContent>
 
           <TabsContent value="flow">
             <SystemFlowDiagram />

package-lock.json

@@ -41,9 +41,12 @@
         "@radix-ui/react-tooltip": "^1.2.0",
         "@tailwindcss/typography": "^0.5.15",
         "@tanstack/react-query": "^5.60.5",
+        "@types/better-sqlite3": "^7.6.13",
         "@types/d3": "^7.4.3",
+        "@types/multer": "^1.4.13",
         "@vitejs/plugin-react": "^4.3.2",
         "autoprefixer": "^10.4.20",
+        "better-sqlite3": "^11.10.0",
         "class-variance-authority": "^0.7.1",
         "clsx": "^2.1.1",
         "cmdk": "^1.1.1",
@@ -64,6 +67,7 @@
         "input-otp": "^1.4.2",
         "lucide-react": "^0.453.0",
         "memorystore": "^1.6.7",
+        "multer": "^2.0.1",
         "next-themes": "^0.4.6",
         "openai": "^5.1.0",
         "passport": "^0.7.0",
@@ -3266,11 +3270,18 @@
         "@babel/types": "^7.20.7"
       }
     },
+    "node_modules/@types/better-sqlite3": {
+      "version": "7.6.13",
+      "resolved": "https://registry.npmjs.org/@types/better-sqlite3/-/better-sqlite3-7.6.13.tgz",
+      "integrity": "sha512-NMv9ASNARoKksWtsq/SHakpYAYnhBrQgGD8zkLYk/jaK8jUGn08CfEdTRgYhMypUQAfzSP8W6gNLe0q19/t4VA==",
+      "dependencies": {
+        "@types/node": "*"
+      }
+    },
     "node_modules/@types/body-parser": {
       "version": "1.19.5",
       "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.5.tgz",
       "integrity": "sha512-fB3Zu92ucau0iQ0JMCFQE7b/dv8Ot07NI3KaZIkIUNXq82k4eBAqUaneXfleGY9JWskeS9y+u0nXMyspcuQrCg==",
-      "dev": true,
       "license": "MIT",
       "dependencies": {
         "@types/connect": "*",
@@ -3281,7 +3292,6 @@
       "version": "3.4.38",
       "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz",
       "integrity": "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==",
-      "dev": true,
       "license": "MIT",
       "dependencies": {
         "@types/node": "*"
@@ -3562,7 +3572,6 @@
       "version": "4.17.21",
       "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.21.tgz",
       "integrity": "sha512-ejlPM315qwLpaQlQDTjPdsUFSc6ZsP4AN6AlWnogPjQ7CVi7PYF3YVz+CY3jE2pwYf7E/7HlDAN0rV2GxTG0HQ==",
-      "dev": true,
       "license": "MIT",
       "dependencies": {
         "@types/body-parser": "*",
@@ -3575,7 +3584,6 @@
       "version": "4.19.6",
       "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.19.6.tgz",
       "integrity": "sha512-N4LZ2xG7DatVqhCZzOGb1Yi5lMbXSZcmdLDe9EzSndPV2HpWYWzRbaerl2n27irrm94EPpprqa8KpskPT085+A==",
-      "dev": true,
       "license": "MIT",
       "dependencies": {
         "@types/node": "*",
@@ -3604,16 +3612,22 @@
       "version": "2.0.4",
       "resolved": "https://registry.npmjs.org/@types/http-errors/-/http-errors-2.0.4.tgz",
       "integrity": "sha512-D0CFMMtydbJAegzOyHjtiKPLlvnm3iTZyZRSZoLq2mRhDdmLfIWOCYPfQJ4cu2erKghU++QvjcUjp/5h7hESpA==",
-      "dev": true,
       "license": "MIT"
     },
     "node_modules/@types/mime": {
       "version": "1.3.5",
       "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz",
       "integrity": "sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==",
-      "dev": true,
       "license": "MIT"
     },
+    "node_modules/@types/multer": {
+      "version": "1.4.13",
+      "resolved": "https://registry.npmjs.org/@types/multer/-/multer-1.4.13.tgz",
+      "integrity": "sha512-bhhdtPw7JqCiEfC9Jimx5LqX9BDIPJEh2q/fQ4bqbBPtyEZYr3cvF22NwG0DmPZNYA0CAf2CnqDB4KIGGpJcaw==",
+      "dependencies": {
+        "@types/express": "*"
+      }
+    },
     "node_modules/@types/node": {
       "version": "20.16.11",
       "resolved": "https://registry.npmjs.org/@types/node/-/node-20.16.11.tgz",
@@ -3678,14 +3692,12 @@
       "version": "6.9.16",
       "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.16.tgz",
       "integrity": "sha512-7i+zxXdPD0T4cKDuxCUXJ4wHcsJLwENa6Z3dCu8cfCK743OGy5Nu1RmAGqDPsoTDINVEcdXKRvR/zre+P2Ku1A==",
-      "dev": true,
       "license": "MIT"
     },
     "node_modules/@types/range-parser": {
       "version": "1.2.7",
       "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.7.tgz",
       "integrity": "sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==",
-      "dev": true,
       "license": "MIT"
     },
     "node_modules/@types/react": {
@@ -3713,7 +3725,6 @@
       "version": "0.17.4",
       "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.4.tgz",
       "integrity": "sha512-x2EM6TJOybec7c52BX0ZspPodMsQUd5L6PRwOunVyVUhXiBSKf3AezDL8Dgvgt5o0UfKNfuA0eMLr2wLT4AiBA==",
-      "dev": true,
       "license": "MIT",
       "dependencies": {
         "@types/mime": "^1",
@@ -3724,7 +3735,6 @@
       "version": "1.15.7",
       "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.7.tgz",
       "integrity": "sha512-W8Ym+h8nhuRwaKPaDw34QUkwsGi6Rc4yYqvKFo5rm2FUEhCFbzVWrxXUxuKK8TASjWsysJY0nsmNCGhCOIsrOw==",
-      "dev": true,
       "license": "MIT",
       "dependencies": {
         "@types/http-errors": "*",
@@ -3842,6 +3852,11 @@
         "node": ">= 8"
       }
     },
+    "node_modules/append-field": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/append-field/-/append-field-1.0.0.tgz",
+      "integrity": "sha512-klpgFSWLW1ZEs8svjfb7g4qWY0YS5imI82dTg+QahUvJ8YqAY0P10Uk8tTyh9ZGuYEZEMaeJYCF5BFuX552hsw=="
+    },
     "node_modules/arg": {
       "version": "5.0.2",
       "resolved": "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz",
@@ -3909,6 +3924,35 @@
       "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==",
       "license": "MIT"
     },
+    "node_modules/base64-js": {
+      "version": "1.5.1",
+      "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
+      "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/feross"
+        },
+        {
+          "type": "patreon",
+          "url": "https://www.patreon.com/feross"
+        },
+        {
+          "type": "consulting",
+          "url": "https://feross.org/support"
+        }
+      ]
+    },
+    "node_modules/better-sqlite3": {
+      "version": "11.10.0",
+      "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-11.10.0.tgz",
+      "integrity": "sha512-EwhOpyXiOEL/lKzHz9AW1msWFNzGc/z+LzeB3/jnFJpxu+th2yqvzsSWas1v9jgs9+xiXJcD5A8CJxAG2TaghQ==",
+      "hasInstallScript": true,
+      "dependencies": {
+        "bindings": "^1.5.0",
+        "prebuild-install": "^7.1.1"
+      }
+    },
     "node_modules/bezier-js": {
       "version": "6.1.4",
       "resolved": "https://registry.npmjs.org/bezier-js/-/bezier-js-6.1.4.tgz",
@@ -3931,6 +3975,24 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/bindings": {
+      "version": "1.5.0",
+      "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz",
+      "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==",
+      "dependencies": {
+        "file-uri-to-path": "1.0.0"
+      }
+    },
+    "node_modules/bl": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz",
+      "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==",
+      "dependencies": {
+        "buffer": "^5.5.0",
+        "inherits": "^2.0.4",
+        "readable-stream": "^3.4.0"
+      }
+    },
     "node_modules/body-parser": {
       "version": "1.20.3",
       "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz",
@@ -4023,11 +4085,33 @@
         "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7"
       }
     },
+    "node_modules/buffer": {
+      "version": "5.7.1",
+      "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
+      "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/feross"
+        },
+        {
+          "type": "patreon",
+          "url": "https://www.patreon.com/feross"
+        },
+        {
+          "type": "consulting",
+          "url": "https://feross.org/support"
+        }
+      ],
+      "dependencies": {
+        "base64-js": "^1.3.1",
+        "ieee754": "^1.1.13"
+      }
+    },
     "node_modules/buffer-from": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz",
       "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==",
-      "dev": true,
       "license": "MIT"
     },
     "node_modules/bufferutil": {
@@ -4044,6 +4128,17 @@
         "node": ">=6.14.2"
       }
     },
+    "node_modules/busboy": {
+      "version": "1.6.0",
+      "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz",
+      "integrity": "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==",
+      "dependencies": {
+        "streamsearch": "^1.1.0"
+      },
+      "engines": {
+        "node": ">=10.16.0"
+      }
+    },
     "node_modules/bytes": {
       "version": "3.1.2",
       "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz",
@@ -4148,6 +4243,11 @@
         "node": ">= 6"
       }
     },
+    "node_modules/chownr": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz",
+      "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg=="
+    },
     "node_modules/class-variance-authority": {
       "version": "0.7.1",
       "resolved": "https://registry.npmjs.org/class-variance-authority/-/class-variance-authority-0.7.1.tgz",
@@ -4209,6 +4309,20 @@
         "node": ">= 6"
       }
     },
+    "node_modules/concat-stream": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz",
+      "integrity": "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==",
+      "engines": [
+        "node >= 6.0"
+      ],
+      "dependencies": {
+        "buffer-from": "^1.0.0",
+        "inherits": "^2.0.3",
+        "readable-stream": "^3.0.2",
+        "typedarray": "^0.0.6"
+      }
+    },
     "node_modules/connect-pg-simple": {
       "version": "10.0.0",
       "resolved": "https://registry.npmjs.org/connect-pg-simple/-/connect-pg-simple-10.0.0.tgz",
@@ -4788,6 +4902,28 @@
       "integrity": "sha512-qIMFpTMZmny+MMIitAB6D7iVPEorVw6YQRWkvarTkT4tBeSLLiHzcwj6q0MmYSFCiVpiqPJTJEYIrpcPzVEIvg==",
       "license": "MIT"
     },
+    "node_modules/decompress-response": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz",
+      "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==",
+      "dependencies": {
+        "mimic-response": "^3.1.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/deep-extend": {
+      "version": "0.6.0",
+      "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz",
+      "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==",
+      "engines": {
+        "node": ">=4.0.0"
+      }
+    },
     "node_modules/define-data-property": {
       "version": "1.1.4",
       "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz",
@@ -4837,7 +4973,6 @@
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.3.tgz",
       "integrity": "sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==",
-      "devOptional": true,
       "engines": {
         "node": ">=8"
       }
@@ -5542,6 +5677,14 @@
         "node": ">= 0.8"
       }
     },
+    "node_modules/end-of-stream": {
+      "version": "1.4.4",
+      "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz",
+      "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==",
+      "dependencies": {
+        "once": "^1.4.0"
+      }
+    },
     "node_modules/enhanced-resolve": {
       "version": "5.18.1",
       "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.18.1.tgz",
@@ -5659,6 +5802,14 @@
       "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==",
       "license": "MIT"
     },
+    "node_modules/expand-template": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz",
+      "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==",
+      "engines": {
+        "node": ">=6"
+      }
+    },
     "node_modules/express": {
       "version": "4.21.2",
       "resolved": "https://registry.npmjs.org/express/-/express-4.21.2.tgz",
@@ -5840,6 +5991,11 @@
         "reusify": "^1.0.4"
       }
     },
+    "node_modules/file-uri-to-path": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz",
+      "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw=="
+    },
     "node_modules/fill-range": {
       "version": "7.1.1",
       "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz",
@@ -5999,6 +6155,11 @@
         "node": ">= 0.6"
       }
     },
+    "node_modules/fs-constants": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz",
+      "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow=="
+    },
     "node_modules/fsevents": {
       "version": "2.3.3",
       "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz",
@@ -6071,6 +6232,11 @@
         "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1"
       }
     },
+    "node_modules/github-from-package": {
+      "version": "0.0.0",
+      "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz",
+      "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw=="
+    },
     "node_modules/glob": {
       "version": "10.4.5",
       "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.5.tgz",
@@ -6214,6 +6380,25 @@
         "node": ">=0.10.0"
       }
     },
+    "node_modules/ieee754": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz",
+      "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/feross"
+        },
+        {
+          "type": "patreon",
+          "url": "https://www.patreon.com/feross"
+        },
+        {
+          "type": "consulting",
+          "url": "https://feross.org/support"
+        }
+      ]
+    },
     "node_modules/index-array-by": {
       "version": "1.4.2",
       "resolved": "https://registry.npmjs.org/index-array-by/-/index-array-by-1.4.2.tgz",
@@ -6229,6 +6414,11 @@
       "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==",
       "license": "ISC"
     },
+    "node_modules/ini": {
+      "version": "1.3.8",
+      "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz",
+      "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew=="
+    },
     "node_modules/input-otp": {
       "version": "1.4.2",
       "resolved": "https://registry.npmjs.org/input-otp/-/input-otp-1.4.2.tgz",
@@ -6827,6 +7017,17 @@
         "node": ">= 0.6"
       }
     },
+    "node_modules/mimic-response": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz",
+      "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==",
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
     "node_modules/minimatch": {
       "version": "9.0.5",
       "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz",
@@ -6846,7 +7047,6 @@
       "version": "1.2.8",
       "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz",
       "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==",
-      "dev": true,
       "funding": {
         "url": "https://github.com/sponsors/ljharb"
       }
@@ -6866,6 +7066,22 @@
       "integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==",
       "license": "MIT"
     },
+    "node_modules/mkdirp": {
+      "version": "0.5.6",
+      "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz",
+      "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==",
+      "dependencies": {
+        "minimist": "^1.2.6"
+      },
+      "bin": {
+        "mkdirp": "bin/cmd.js"
+      }
+    },
+    "node_modules/mkdirp-classic": {
+      "version": "0.5.3",
+      "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz",
+      "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A=="
+    },
     "node_modules/modern-screenshot": {
       "version": "4.6.0",
       "resolved": "https://registry.npmjs.org/modern-screenshot/-/modern-screenshot-4.6.0.tgz",
@@ -6888,6 +7104,23 @@
       "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
       "license": "MIT"
     },
+    "node_modules/multer": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/multer/-/multer-2.0.1.tgz",
+      "integrity": "sha512-Ug8bXeTIUlxurg8xLTEskKShvcKDZALo1THEX5E41pYCD2sCVub5/kIRIGqWNoqV6szyLyQKV6mD4QUrWE5GCQ==",
+      "dependencies": {
+        "append-field": "^1.0.0",
+        "busboy": "^1.6.0",
+        "concat-stream": "^2.0.0",
+        "mkdirp": "^0.5.6",
+        "object-assign": "^4.1.1",
+        "type-is": "^1.6.18",
+        "xtend": "^4.0.2"
+      },
+      "engines": {
+        "node": ">= 10.16.0"
+      }
+    },
     "node_modules/mz": {
       "version": "2.7.0",
       "resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz",
@@ -6916,6 +7149,11 @@
         "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1"
       }
     },
+    "node_modules/napi-build-utils": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-2.0.0.tgz",
+      "integrity": "sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA=="
+    },
     "node_modules/negotiator": {
       "version": "0.6.3",
       "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz",
@@ -6972,6 +7210,28 @@
       "integrity": "sha512-4EUeAGbB2HWX9njd6bP6tciN6ByJfoaAvmVL9QTaZSeXrW46eNGA9GajiXiPBbvFqxUWFkEbyo6x5qsACUuVfA==",
       "license": "BSD-3-Clause"
     },
+    "node_modules/node-abi": {
+      "version": "3.75.0",
+      "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.75.0.tgz",
+      "integrity": "sha512-OhYaY5sDsIka7H7AtijtI9jwGYLyl29eQn/W623DiN/MIv5sUqc4g7BIDThX+gb7di9f6xK02nkp8sdfFWZLTg==",
+      "dependencies": {
+        "semver": "^7.3.5"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/node-abi/node_modules/semver": {
+      "version": "7.7.2",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.2.tgz",
+      "integrity": "sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA==",
+      "bin": {
+        "semver": "bin/semver.js"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
     "node_modules/node-gyp-build": {
       "version": "4.8.3",
       "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.3.tgz",
@@ -7065,6 +7325,14 @@
         "node": ">= 0.8"
       }
     },
+    "node_modules/once": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
+      "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==",
+      "dependencies": {
+        "wrappy": "1"
+      }
+    },
     "node_modules/openai": {
       "version": "5.1.0",
       "resolved": "https://registry.npmjs.org/openai/-/openai-5.1.0.tgz",
@@ -7587,6 +7855,31 @@
         "url": "https://opencollective.com/preact"
       }
     },
+    "node_modules/prebuild-install": {
+      "version": "7.1.3",
+      "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz",
+      "integrity": "sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==",
+      "dependencies": {
+        "detect-libc": "^2.0.0",
+        "expand-template": "^2.0.3",
+        "github-from-package": "0.0.0",
+        "minimist": "^1.2.3",
+        "mkdirp-classic": "^0.5.3",
+        "napi-build-utils": "^2.0.0",
+        "node-abi": "^3.3.0",
+        "pump": "^3.0.0",
+        "rc": "^1.2.7",
+        "simple-get": "^4.0.0",
+        "tar-fs": "^2.0.0",
+        "tunnel-agent": "^0.6.0"
+      },
+      "bin": {
+        "prebuild-install": "bin.js"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
     "node_modules/prop-types": {
       "version": "15.8.1",
       "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz",
@@ -7621,6 +7914,15 @@
       "integrity": "sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ==",
       "license": "ISC"
     },
+    "node_modules/pump": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.2.tgz",
+      "integrity": "sha512-tUPXtzlGM8FE3P0ZL6DVs/3P58k9nk8/jZeQCurTJylQA8qFYzHFfhBJkuqyE0FifOsQ0uKWekiZ5g8wtr28cw==",
+      "dependencies": {
+        "end-of-stream": "^1.1.0",
+        "once": "^1.3.1"
+      }
+    },
     "node_modules/qs": {
       "version": "6.13.0",
       "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz",
@@ -7689,6 +7991,20 @@
         "node": ">= 0.8"
       }
     },
+    "node_modules/rc": {
+      "version": "1.2.8",
+      "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz",
+      "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==",
+      "dependencies": {
+        "deep-extend": "^0.6.0",
+        "ini": "~1.3.0",
+        "minimist": "^1.2.0",
+        "strip-json-comments": "~2.0.1"
+      },
+      "bin": {
+        "rc": "cli.js"
+      }
+    },
     "node_modules/react": {
       "version": "18.3.1",
       "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz",
@@ -7926,6 +8242,19 @@
         "pify": "^2.3.0"
       }
     },
+    "node_modules/readable-stream": {
+      "version": "3.6.2",
+      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
+      "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==",
+      "dependencies": {
+        "inherits": "^2.0.3",
+        "string_decoder": "^1.1.1",
+        "util-deprecate": "^1.0.1"
+      },
+      "engines": {
+        "node": ">= 6"
+      }
+    },
     "node_modules/readdirp": {
       "version": "3.6.0",
       "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz",
@@ -8273,6 +8602,49 @@
         "url": "https://github.com/sponsors/isaacs"
       }
     },
+    "node_modules/simple-concat": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz",
+      "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/feross"
+        },
+        {
+          "type": "patreon",
+          "url": "https://www.patreon.com/feross"
+        },
+        {
+          "type": "consulting",
+          "url": "https://feross.org/support"
+        }
+      ]
+    },
+    "node_modules/simple-get": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz",
+      "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/feross"
+        },
+        {
+          "type": "patreon",
+          "url": "https://www.patreon.com/feross"
+        },
+        {
+          "type": "consulting",
+          "url": "https://feross.org/support"
+        }
+      ],
+      "dependencies": {
+        "decompress-response": "^6.0.0",
+        "once": "^1.3.1",
+        "simple-concat": "^1.0.0"
+      }
+    },
     "node_modules/source-map": {
       "version": "0.6.1",
       "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
@@ -8321,6 +8693,22 @@
         "node": ">= 0.8"
       }
     },
+    "node_modules/streamsearch": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz",
+      "integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==",
+      "engines": {
+        "node": ">=10.0.0"
+      }
+    },
+    "node_modules/string_decoder": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",
+      "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==",
+      "dependencies": {
+        "safe-buffer": "~5.2.0"
+      }
+    },
     "node_modules/string-width": {
       "version": "5.1.2",
       "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz",
@@ -8417,6 +8805,14 @@
         "node": ">=8"
       }
     },
+    "node_modules/strip-json-comments": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz",
+      "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/sucrase": {
       "version": "3.35.0",
       "resolved": "https://registry.npmjs.org/sucrase/-/sucrase-3.35.0.tgz",
@@ -8513,6 +8909,32 @@
         "node": ">=6"
       }
     },
+    "node_modules/tar-fs": {
+      "version": "2.1.3",
+      "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.3.tgz",
+      "integrity": "sha512-090nwYJDmlhwFwEW3QQl+vaNnxsO2yVsd45eTKRBzSzu+hlb1w2K9inVq5b0ngXuLVqQ4ApvsUHHnu/zQNkWAg==",
+      "dependencies": {
+        "chownr": "^1.1.1",
+        "mkdirp-classic": "^0.5.2",
+        "pump": "^3.0.0",
+        "tar-stream": "^2.1.4"
+      }
+    },
+    "node_modules/tar-stream": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz",
+      "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==",
+      "dependencies": {
+        "bl": "^4.0.3",
+        "end-of-stream": "^1.4.1",
+        "fs-constants": "^1.0.0",
+        "inherits": "^2.0.3",
+        "readable-stream": "^3.1.1"
+      },
+      "engines": {
+        "node": ">=6"
+      }
+    },
     "node_modules/thenify": {
       "version": "3.3.1",
       "resolved": "https://registry.npmjs.org/thenify/-/thenify-3.3.1.tgz",
@@ -9096,6 +9518,17 @@
         "@esbuild/win32-x64": "0.23.1"
       }
     },
+    "node_modules/tunnel-agent": {
+      "version": "0.6.0",
+      "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz",
+      "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==",
+      "dependencies": {
+        "safe-buffer": "^5.0.1"
+      },
+      "engines": {
+        "node": "*"
+      }
+    },
     "node_modules/tw-animate-css": {
       "version": "1.2.5",
       "resolved": "https://registry.npmjs.org/tw-animate-css/-/tw-animate-css-1.2.5.tgz",
@@ -9117,6 +9550,11 @@
         "node": ">= 0.6"
       }
     },
+    "node_modules/typedarray": {
+      "version": "0.0.6",
+      "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz",
+      "integrity": "sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA=="
+    },
     "node_modules/typescript": {
       "version": "5.6.3",
       "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.6.3.tgz",
@@ -9889,6 +10327,11 @@
         "node": ">=8"
       }
     },
+    "node_modules/wrappy": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
+      "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="
+    },
     "node_modules/ws": {
       "version": "8.18.0",
       "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz",

package.json

@@ -43,9 +43,12 @@
     "@radix-ui/react-tooltip": "^1.2.0",
     "@tailwindcss/typography": "^0.5.15",
     "@tanstack/react-query": "^5.60.5",
+    "@types/better-sqlite3": "^7.6.13",
     "@types/d3": "^7.4.3",
+    "@types/multer": "^1.4.13",
     "@vitejs/plugin-react": "^4.3.2",
     "autoprefixer": "^10.4.20",
+    "better-sqlite3": "^11.10.0",
     "class-variance-authority": "^0.7.1",
     "clsx": "^2.1.1",
     "cmdk": "^1.1.1",
@@ -66,6 +69,7 @@
     "input-otp": "^1.4.2",
     "lucide-react": "^0.453.0",
     "memorystore": "^1.6.7",
+    "multer": "^2.0.1",
     "next-themes": "^0.4.6",
     "openai": "^5.1.0",
     "passport": "^0.7.0",

server/document-processor.ts

@@ -0,0 +1,465 @@
+import fs from 'fs';
+import path from 'path';
+import { modalClient } from './modal-client';
+import { nebiusClient } from './nebius-client';
+import { FileProcessor } from './file-upload';
+import { type Document, type InsertDocument } from '@shared/schema';
+
+export interface ProcessingResult {
+  success: boolean;
+  extractedText?: string;
+  embeddings?: number[];
+  modalTaskId?: string;
+  error?: string;
+  processingTime: number;
+}
+
+export interface BatchProcessingResult {
+  success: boolean;
+  processedCount: number;
+  failedCount: number;
+  results: Array<{
+    documentId: number;
+    success: boolean;
+    extractedText?: string;
+    embeddings?: number[];
+    error?: string;
+  }>;
+  totalProcessingTime: number;
+}
+
+export class DocumentProcessor {
+  private static instance: DocumentProcessor;
+
+  static getInstance(): DocumentProcessor {
+    if (!DocumentProcessor.instance) {
+      DocumentProcessor.instance = new DocumentProcessor();
+    }
+    return DocumentProcessor.instance;
+  }
+
+  /**
+   * Process a single document using Modal for heavy workloads
+   */
+  async processDocument(
+    document: Document,
+    operations: Array<'extract_text' | 'generate_embedding' | 'build_index'> = ['extract_text']
+  ): Promise<ProcessingResult> {
+    const startTime = Date.now();
+    
+    try {
+      let extractedText = document.content;
+      let embeddings: number[] | undefined;
+      let modalTaskId: string | undefined;
+
+      // Step 1: Extract text if needed (for PDFs and images)
+      if (operations.includes('extract_text') && document.filePath) {
+        const textResult = await this.extractText(document);
+        if (textResult.success) {
+          extractedText = textResult.extractedText || document.content;
+          modalTaskId = textResult.modalTaskId;
+        } else {
+          console.warn(`Text extraction failed for document ${document.id}: ${textResult.error}`);
+        }
+      }
+
+      // Step 2: Generate embeddings if requested
+      if (operations.includes('generate_embedding') && extractedText) {
+        const embeddingResult = await this.generateEmbeddings(extractedText);
+        if (embeddingResult.success) {
+          embeddings = embeddingResult.embeddings;
+        } else {
+          console.warn(`Embedding generation failed for document ${document.id}: ${embeddingResult.error}`);
+        }
+      }
+
+      const processingTime = Date.now() - startTime;
+
+      return {
+        success: true,
+        extractedText,
+        embeddings,
+        modalTaskId,
+        processingTime
+      };
+
+    } catch (error) {
+      const processingTime = Date.now() - startTime;
+      return {
+        success: false,
+        error: error instanceof Error ? error.message : String(error),
+        processingTime
+      };
+    }
+  }
+
+  /**
+   * Process multiple documents in batch using Modal's distributed computing
+   */
+  async batchProcessDocuments(
+    documents: Document[],
+    operations: Array<'extract_text' | 'generate_embedding' | 'build_index'> = ['extract_text']
+  ): Promise<BatchProcessingResult> {
+    const startTime = Date.now();
+    const results: BatchProcessingResult['results'] = [];
+    
+    try {
+      // Separate documents by processing requirements
+      const documentsForModal = documents.filter(doc => 
+        doc.filePath && FileProcessor.requiresOCR(doc.mimeType || '')
+      );
+      
+      const documentsForLocal = documents.filter(doc => 
+        !doc.filePath || !FileProcessor.requiresOCR(doc.mimeType || '')
+      );
+
+      // Process Modal-required documents in batch
+      if (documentsForModal.length > 0 && operations.includes('extract_text')) {
+        try {
+          const modalResults = await this.batchExtractTextModal(documentsForModal);
+          results.push(...modalResults);
+        } catch (error) {
+          console.error('Modal batch processing failed:', error);
+          // Fall back to individual processing
+          for (const doc of documentsForModal) {
+            const result = await this.processDocument(doc, operations);
+            results.push({
+              documentId: doc.id,
+              success: result.success,
+              extractedText: result.extractedText,
+              embeddings: result.embeddings,
+              error: result.error
+            });
+          }
+        }
+      }
+
+      // Process local documents
+      for (const doc of documentsForLocal) {
+        const result = await this.processDocument(doc, operations);
+        results.push({
+          documentId: doc.id,
+          success: result.success,
+          extractedText: result.extractedText,
+          embeddings: result.embeddings,
+          error: result.error
+        });
+      }
+
+      const totalProcessingTime = Date.now() - startTime;
+      const successCount = results.filter(r => r.success).length;
+      const failedCount = results.length - successCount;
+
+      return {
+        success: true,
+        processedCount: successCount,
+        failedCount,
+        results,
+        totalProcessingTime
+      };
+
+    } catch (error) {
+      const totalProcessingTime = Date.now() - startTime;
+      return {
+        success: false,
+        processedCount: 0,
+        failedCount: documents.length,
+        results: documents.map(doc => ({
+          documentId: doc.id,
+          success: false,
+          error: error instanceof Error ? error.message : String(error)
+        })),
+        totalProcessingTime
+      };
+    }
+  }
+
+  /**
+   * Extract text from a document using Modal for PDFs/images or direct reading for text files
+   */
+  private async extractText(document: Document): Promise<{
+    success: boolean;
+    extractedText?: string;
+    modalTaskId?: string;
+    error?: string;
+  }> {
+    if (!document.filePath) {
+      return { success: true, extractedText: document.content };
+    }
+
+    const mimeType = document.mimeType || '';
+
+    try {
+      // For text files, read directly
+      if (FileProcessor.isTextFile(mimeType)) {
+        const content = await FileProcessor.readTextFile(document.filePath);
+        return { success: true, extractedText: content };
+      }
+
+      // For PDFs and images, use Modal
+      if (FileProcessor.requiresOCR(mimeType)) {
+        return await this.extractTextModal(document);
+      }
+
+      // Fallback: return existing content
+      return { success: true, extractedText: document.content };
+
+    } catch (error) {
+      return {
+        success: false,
+        error: error instanceof Error ? error.message : String(error)
+      };
+    }
+  }
+
+  /**
+   * Extract text using Modal for OCR-required files
+   */
+  private async extractTextModal(document: Document): Promise<{
+    success: boolean;
+    extractedText?: string;
+    modalTaskId?: string;
+    error?: string;
+  }> {
+    try {
+      if (!document.filePath) {
+        throw new Error('No file path provided for Modal processing');
+      }
+
+      // Read file and convert to base64
+      const fileBuffer = await fs.promises.readFile(document.filePath);
+      const base64Content = fileBuffer.toString('base64');
+
+      // Prepare document for Modal
+      const modalDocument = {
+        id: document.id.toString(),
+        content: base64Content,
+        contentType: document.mimeType || 'application/octet-stream'
+      };
+
+      // Call Modal extract-text endpoint
+      const result = await modalClient.extractTextFromDocuments([modalDocument]);
+      
+      if (result.status === 'completed' && result.results?.length > 0) {
+        const extractionResult = result.results[0];
+        if (extractionResult.status === 'completed') {
+          return {
+            success: true,
+            extractedText: extractionResult.extracted_text,
+            modalTaskId: result.task_id
+          };
+        } else {
+          return {
+            success: false,
+            error: extractionResult.error || 'Modal extraction failed'
+          };
+        }
+      } else {
+        return {
+          success: false,
+          error: result.error || 'Modal processing failed'
+        };
+      }
+
+    } catch (error) {
+      console.error('Modal text extraction failed:', error);
+      return {
+        success: false,
+        error: error instanceof Error ? error.message : String(error)
+      };
+    }
+  }
+
+  /**
+   * Batch extract text using Modal
+   */
+  private async batchExtractTextModal(documents: Document[]): Promise<Array<{
+    documentId: number;
+    success: boolean;
+    extractedText?: string;
+    error?: string;
+  }>> {
+    const modalDocuments = await Promise.all(
+      documents.map(async (doc) => {
+        if (!doc.filePath) return null;
+        
+        try {
+          const fileBuffer = await fs.promises.readFile(doc.filePath);
+          return {
+            id: doc.id.toString(),
+            content: fileBuffer.toString('base64'),
+            contentType: doc.mimeType || 'application/octet-stream'
+          };
+        } catch (error) {
+          console.error(`Failed to read file for document ${doc.id}:`, error);
+          return null;
+        }
+      })
+    );
+
+    const validDocuments = modalDocuments.filter(doc => doc !== null) as any[];
+    
+    if (validDocuments.length === 0) {
+      return documents.map(doc => ({
+        documentId: doc.id,
+        success: false,
+        error: 'No valid documents for processing'
+      }));
+    }
+
+    try {
+      const batchResult = await modalClient.batchProcessDocuments({
+        documents: validDocuments,
+        modelName: 'text-embedding-3-small',
+        batchSize: Math.min(validDocuments.length, 10)
+      });
+
+      if (batchResult.status === 'completed' && batchResult.extraction_results) {
+        return batchResult.extraction_results.map((result: any) => ({
+          documentId: parseInt(result.id),
+          success: result.status === 'completed',
+          extractedText: result.extracted_text,
+          error: result.error
+        }));
+      } else {
+        throw new Error(batchResult.error || 'Batch processing failed');
+      }
+
+    } catch (error) {
+      console.error('Modal batch processing failed:', error);
+      return documents.map(doc => ({
+        documentId: doc.id,
+        success: false,
+        error: error instanceof Error ? error.message : String(error)
+      }));
+    }
+  }
+
+  /**
+   * Generate embeddings using Nebius AI
+   */
+  private async generateEmbeddings(text: string): Promise<{
+    success: boolean;
+    embeddings?: number[];
+    error?: string;
+  }> {
+    try {
+      // Truncate text if too long (most embedding models have token limits)
+      const maxLength = 8000; // Conservative limit
+      const truncatedText = text.length > maxLength ? text.substring(0, maxLength) : text;
+      
+      const result = await nebiusClient.generateEmbeddings(truncatedText);
+      
+      if (result.success && result.embeddings) {
+        return {
+          success: true,
+          embeddings: result.embeddings
+        };
+      } else {
+        return {
+          success: false,
+          error: result.error || 'Embedding generation failed'
+        };
+      }
+
+    } catch (error) {
+      return {
+        success: false,
+        error: error instanceof Error ? error.message : String(error)
+      };
+    }
+  }
+
+  /**
+   * Build vector index using Modal
+   */
+  async buildVectorIndex(
+    documents: Document[],
+    indexName = 'main_index'
+  ): Promise<{
+    success: boolean;
+    indexName?: string;
+    documentCount?: number;
+    error?: string;
+  }> {
+    try {
+      const modalDocuments = documents.map(doc => ({
+        id: doc.id.toString(),
+        content: doc.content,
+        title: doc.title,
+        source: doc.source
+      }));
+
+      const result = await modalClient.buildVectorIndex(modalDocuments, {
+        indexName,
+        dimension: 1536, // Standard OpenAI embedding dimension
+        indexType: 'IVF',
+        nlist: Math.min(100, Math.max(10, Math.floor(documents.length / 10)))
+      });
+
+      if (result.status === 'completed') {
+        return {
+          success: true,
+          indexName: result.index_name,
+          documentCount: result.document_count
+        };
+      } else {
+        return {
+          success: false,
+          error: result.error || 'Index building failed'
+        };
+      }
+
+    } catch (error) {
+      return {
+        success: false,
+        error: error instanceof Error ? error.message : String(error)
+      };
+    }
+  }
+
+  /**
+   * Search vector index using Modal
+   */
+  async searchVectorIndex(
+    query: string,
+    indexName = 'main_index',
+    maxResults = 10
+  ): Promise<{
+    success: boolean;
+    results?: Array<{
+      id: string;
+      title: string;
+      content: string;
+      source: string;
+      relevanceScore: number;
+      rank: number;
+      snippet: string;
+    }>;
+    error?: string;
+  }> {
+    try {
+      const result = await modalClient.vectorSearch(query, indexName, maxResults);
+      
+      if (result.status === 'completed') {
+        return {
+          success: true,
+          results: result.results
+        };
+      } else {
+        return {
+          success: false,
+          error: result.error || 'Vector search failed'
+        };
+      }
+
+    } catch (error) {
+      return {
+        success: false,
+        error: error instanceof Error ? error.message : String(error)
+      };
+    }
+  }
+}
+
+export const documentProcessor = DocumentProcessor.getInstance();

server/document-routes.ts

@@ -0,0 +1,449 @@
+import { Router } from 'express';
+import { upload, validateUpload, FileProcessor } from './file-upload';
+import { documentProcessor } from './document-processor';
+import { storage } from './storage';
+import { fileUploadSchema, documentProcessingSchema, batchProcessingSchema } from '@shared/schema';
+import path from 'path';
+
+const router = Router();
+
+/**
+ * Upload documents (multiple files supported)
+ */
+router.post('/upload', upload.array('files', 10), validateUpload, async (req, res) => {
+  try {
+    const files = req.files as Express.Multer.File[];
+    const uploadedDocuments = [];
+
+    for (const file of files) {
+      // Extract title from filename or use provided title
+      const title = req.body.title || path.basename(file.originalname, path.extname(file.originalname));
+      const source = req.body.source || `Uploaded file: ${file.originalname}`;
+
+      // Determine source type based on MIME type
+      let sourceType = 'document';
+      if (FileProcessor.isPdfFile(file.mimetype)) {
+        sourceType = 'pdf';
+      } else if (FileProcessor.isImageFile(file.mimetype)) {
+        sourceType = 'image';
+      } else if (file.mimetype.includes('text') || file.mimetype.includes('json')) {
+        sourceType = 'text';
+      }
+
+      // Read text content for text files
+      let content = 'Processing...';
+      if (FileProcessor.isTextFile(file.mimetype)) {
+        try {
+          content = await FileProcessor.readTextFile(file.path);
+        } catch (error) {
+          console.warn(`Failed to read text file ${file.originalname}:`, error);
+          content = 'Failed to read file content';
+        }
+      }
+
+      // Create document record
+      const document = await storage.createDocument({
+        title,
+        content,
+        source,
+        sourceType,
+        url: null,
+        metadata: {
+          originalName: file.originalname,
+          uploadedAt: new Date().toISOString(),
+          mimeType: file.mimetype,
+          size: file.size
+        },
+        embedding: null,
+        filePath: file.path,
+        fileName: file.originalname,
+        fileSize: file.size,
+        mimeType: file.mimetype,
+        processingStatus: FileProcessor.requiresOCR(file.mimetype) ? 'pending' : 'completed'
+      } as any);
+
+      uploadedDocuments.push(document);
+    }
+
+    res.status(201).json({
+      success: true,
+      message: `Successfully uploaded ${uploadedDocuments.length} document(s)`,
+      documents: uploadedDocuments
+    });
+
+  } catch (error) {
+    console.error('File upload error:', error);
+    res.status(500).json({
+      success: false,
+      error: 'File upload failed',
+      message: error instanceof Error ? error.message : 'Unknown error occurred'
+    });
+  }
+});
+
+/**
+ * Process a single document
+ */
+router.post('/process/:id', async (req, res) => {
+  try {
+    const documentId = parseInt(req.params.id);
+    const { operations = ['extract_text'], indexName } = documentProcessingSchema.parse(req.body);
+
+    const document = await storage.getDocument(documentId);
+    if (!document) {
+      return res.status(404).json({
+        success: false,
+        error: 'Document not found'
+      });
+    }
+
+    // Update status to processing
+    await storage.updateDocument(documentId, { 
+      processingStatus: 'processing' 
+    } as any);
+
+    // Process the document
+    const result = await documentProcessor.processDocument(document, operations);
+
+    if (result.success) {
+      // Update document with results
+      const updateData: any = {
+        processingStatus: 'completed',
+        processedAt: new Date()
+      };
+
+      if (result.extractedText && result.extractedText !== document.content) {
+        updateData.content = result.extractedText;
+      }
+
+      if (result.embeddings) {
+        updateData.embedding = JSON.stringify(result.embeddings);
+      }
+
+      if (result.modalTaskId) {
+        updateData.modalTaskId = result.modalTaskId;
+      }
+
+      const updatedDocument = await storage.updateDocument(documentId, updateData);
+
+      res.json({
+        success: true,
+        message: 'Document processed successfully',
+        document: updatedDocument,
+        processingTime: result.processingTime
+      });
+
+    } else {
+      // Update status to failed
+      await storage.updateDocument(documentId, { 
+        processingStatus: 'failed' 
+      } as any);
+
+      res.status(500).json({
+        success: false,
+        error: 'Document processing failed',
+        message: result.error,
+        processingTime: result.processingTime
+      });
+    }
+
+  } catch (error) {
+    console.error('Document processing error:', error);
+    res.status(500).json({
+      success: false,
+      error: 'Processing request failed',
+      message: error instanceof Error ? error.message : 'Unknown error occurred'
+    });
+  }
+});
+
+/**
+ * Batch process multiple documents
+ */
+router.post('/process/batch', async (req, res) => {
+  try {
+    const { documentIds, operations = ['extract_text'], indexName } = batchProcessingSchema.parse(req.body);
+
+    // Fetch all documents
+    const documents = await Promise.all(
+      documentIds.map(id => storage.getDocument(id))
+    );
+
+    const validDocuments = documents.filter(doc => doc !== undefined) as any[];
+    
+    if (validDocuments.length === 0) {
+      return res.status(404).json({
+        success: false,
+        error: 'No valid documents found'
+      });
+    }
+
+    // Update all documents to processing status
+    await Promise.all(
+      validDocuments.map(doc => 
+        storage.updateDocument(doc.id, { processingStatus: 'processing' } as any)
+      )
+    );
+
+    // Process documents in batch
+    const batchResult = await documentProcessor.batchProcessDocuments(validDocuments, operations);
+
+    // Update documents with results
+    const updatePromises = batchResult.results.map(async (result) => {
+      const updateData: any = {
+        processingStatus: result.success ? 'completed' : 'failed',
+        processedAt: new Date()
+      };
+
+      if (result.success) {
+        if (result.extractedText) {
+          updateData.content = result.extractedText;
+        }
+        if (result.embeddings) {
+          updateData.embedding = JSON.stringify(result.embeddings);
+        }
+      }
+
+      return storage.updateDocument(result.documentId, updateData);
+    });
+
+    await Promise.all(updatePromises);
+
+    res.json({
+      success: true,
+      message: `Batch processing completed: ${batchResult.processedCount} successful, ${batchResult.failedCount} failed`,
+      processedCount: batchResult.processedCount,
+      failedCount: batchResult.failedCount,
+      results: batchResult.results,
+      totalProcessingTime: batchResult.totalProcessingTime
+    });
+
+  } catch (error) {
+    console.error('Batch processing error:', error);
+    res.status(500).json({
+      success: false,
+      error: 'Batch processing failed',
+      message: error instanceof Error ? error.message : 'Unknown error occurred'
+    });
+  }
+});
+
+/**
+ * Build vector index from documents
+ */
+router.post('/index/build', async (req, res) => {
+  try {
+    const { documentIds, indexName = 'main_index' } = req.body;
+
+    let documents;
+    if (documentIds && Array.isArray(documentIds)) {
+      // Build index from specific documents
+      const fetchedDocs = await Promise.all(
+        documentIds.map((id: number) => storage.getDocument(id))
+      );
+      documents = fetchedDocs.filter(doc => doc !== undefined) as any[];
+    } else {
+      // Build index from all completed documents
+      documents = await storage.getDocuments(1000, 0);
+      documents = documents.filter(doc => doc.processingStatus === 'completed');
+    }
+
+    if (documents.length === 0) {
+      return res.status(400).json({
+        success: false,
+        error: 'No processed documents available for indexing'
+      });
+    }
+
+    const result = await documentProcessor.buildVectorIndex(documents, indexName);
+
+    if (result.success) {
+      res.json({
+        success: true,
+        message: 'Vector index built successfully',
+        indexName: result.indexName,
+        documentCount: result.documentCount
+      });
+    } else {
+      res.status(500).json({
+        success: false,
+        error: 'Index building failed',
+        message: result.error
+      });
+    }
+
+  } catch (error) {
+    console.error('Index building error:', error);
+    res.status(500).json({
+      success: false,
+      error: 'Index building request failed',
+      message: error instanceof Error ? error.message : 'Unknown error occurred'
+    });
+  }
+});
+
+/**
+ * Search vector index
+ */
+router.post('/search/vector', async (req, res) => {
+  try {
+    const { query, indexName = 'main_index', maxResults = 10 } = req.body;
+
+    if (!query || typeof query !== 'string') {
+      return res.status(400).json({
+        success: false,
+        error: 'Query parameter is required and must be a string'
+      });
+    }
+
+    const result = await documentProcessor.searchVectorIndex(query, indexName, maxResults);
+
+    if (result.success) {
+      res.json({
+        success: true,
+        query,
+        indexName,
+        results: result.results,
+        totalFound: result.results?.length || 0
+      });
+    } else {
+      res.status(500).json({
+        success: false,
+        error: 'Vector search failed',
+        message: result.error
+      });
+    }
+
+  } catch (error) {
+    console.error('Vector search error:', error);
+    res.status(500).json({
+      success: false,
+      error: 'Vector search request failed',
+      message: error instanceof Error ? error.message : 'Unknown error occurred'
+    });
+  }
+});
+
+/**
+ * Get document processing status
+ */
+router.get('/status/:id', async (req, res) => {
+  try {
+    const documentId = parseInt(req.params.id);
+    const document = await storage.getDocument(documentId);
+
+    if (!document) {
+      return res.status(404).json({
+        success: false,
+        error: 'Document not found'
+      });
+    }
+
+    res.json({
+      success: true,
+      document: {
+        id: document.id,
+        title: document.title,
+        processingStatus: (document as any).processingStatus,
+        modalTaskId: (document as any).modalTaskId,
+        createdAt: document.createdAt,
+        processedAt: (document as any).processedAt,
+        fileSize: (document as any).fileSize,
+        mimeType: (document as any).mimeType
+      }
+    });
+
+  } catch (error) {
+    console.error('Status check error:', error);
+    res.status(500).json({
+      success: false,
+      error: 'Status check failed',
+      message: error instanceof Error ? error.message : 'Unknown error occurred'
+    });
+  }
+});
+
+/**
+ * Get all documents with filtering
+ */
+router.get('/list', async (req, res) => {
+  try {
+    const { 
+      limit = 50, 
+      offset = 0, 
+      sourceType, 
+      processingStatus 
+    } = req.query;
+
+    let documents;
+    
+    if (sourceType) {
+      documents = await storage.getDocumentsBySourceType(sourceType as string);
+    } else if (processingStatus && 'getDocumentsByProcessingStatus' in storage) {
+      documents = await (storage as any).getDocumentsByProcessingStatus(processingStatus as string);
+    } else {
+      documents = await storage.getDocuments(Number(limit), Number(offset));
+    }
+
+    res.json({
+      success: true,
+      documents,
+      totalCount: documents.length
+    });
+
+  } catch (error) {
+    console.error('Document list error:', error);
+    res.status(500).json({
+      success: false,
+      error: 'Failed to retrieve documents',
+      message: error instanceof Error ? error.message : 'Unknown error occurred'
+    });
+  }
+});
+
+/**
+ * Delete a document and its file
+ */
+router.delete('/:id', async (req, res) => {
+  try {
+    const documentId = parseInt(req.params.id);
+    const document = await storage.getDocument(documentId);
+
+    if (!document) {
+      return res.status(404).json({
+        success: false,
+        error: 'Document not found'
+      });
+    }
+
+    // Delete file if it exists
+    if ((document as any).filePath) {
+      await FileProcessor.deleteFile((document as any).filePath);
+    }
+
+    // Delete document record
+    const deleted = await storage.deleteDocument(documentId);
+
+    if (deleted) {
+      res.json({
+        success: true,
+        message: 'Document deleted successfully'
+      });
+    } else {
+      res.status(500).json({
+        success: false,
+        error: 'Failed to delete document'
+      });
+    }
+
+  } catch (error) {
+    console.error('Document deletion error:', error);
+    res.status(500).json({
+      success: false,
+      error: 'Document deletion failed',
+      message: error instanceof Error ? error.message : 'Unknown error occurred'
+    });
+  }
+});
+
+export default router;

server/file-upload.ts

@@ -0,0 +1,166 @@
+import multer from 'multer';
+import path from 'path';
+import fs from 'fs';
+import crypto from 'crypto';
+import { Request } from 'express';
+
+// Ensure uploads directory exists
+const uploadsDir = path.join(process.cwd(), 'uploads');
+if (!fs.existsSync(uploadsDir)) {
+  fs.mkdirSync(uploadsDir, { recursive: true });
+}
+
+// Configure multer for file uploads
+const storage = multer.diskStorage({
+  destination: (req, file, cb) => {
+    // Create subdirectories by date for organization
+    const dateDir = new Date().toISOString().split('T')[0];
+    const fullPath = path.join(uploadsDir, dateDir);
+    
+    if (!fs.existsSync(fullPath)) {
+      fs.mkdirSync(fullPath, { recursive: true });
+    }
+    
+    cb(null, fullPath);
+  },
+  filename: (req, file, cb) => {
+    // Generate unique filename with timestamp and random string
+    const timestamp = Date.now();
+    const randomString = crypto.randomBytes(8).toString('hex');
+    const ext = path.extname(file.originalname);
+    const baseName = path.basename(file.originalname, ext);
+    
+    // Sanitize filename
+    const sanitizedBaseName = baseName.replace(/[^a-zA-Z0-9-_]/g, '_');
+    const filename = `${timestamp}_${randomString}_${sanitizedBaseName}${ext}`;
+    
+    cb(null, filename);
+  }
+});
+
+// File filter to accept specific file types
+const fileFilter = (req: Request, file: Express.Multer.File, cb: multer.FileFilterCallback) => {
+  const allowedMimeTypes = [
+    'application/pdf',
+    'image/jpeg',
+    'image/jpg', 
+    'image/png',
+    'image/gif',
+    'image/webp',
+    'text/plain',
+    'text/markdown',
+    'application/msword',
+    'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+    'application/json'
+  ];
+
+  if (allowedMimeTypes.includes(file.mimetype)) {
+    cb(null, true);
+  } else {
+    cb(new Error(`Unsupported file type: ${file.mimetype}. Allowed types: PDF, images (JPEG, PNG, GIF, WebP), text files, Word documents, JSON`));
+  }
+};
+
+// Configure multer with size limits
+export const upload = multer({
+  storage,
+  fileFilter,
+  limits: {
+    fileSize: 50 * 1024 * 1024, // 50MB limit
+    files: 10 // Maximum 10 files per upload
+  }
+});
+
+// File processing utilities
+export class FileProcessor {
+  static async getFileInfo(filePath: string): Promise<{
+    size: number;
+    mimeType: string;
+    exists: boolean;
+  }> {
+    try {
+      const stats = await fs.promises.stat(filePath);
+      return {
+        size: stats.size,
+        mimeType: await this.getMimeType(filePath),
+        exists: true
+      };
+    } catch (error) {
+      return {
+        size: 0,
+        mimeType: '',
+        exists: false
+      };
+    }
+  }
+
+  static async getMimeType(filePath: string): Promise<string> {
+    const ext = path.extname(filePath).toLowerCase();
+    const mimeTypes: Record<string, string> = {
+      '.pdf': 'application/pdf',
+      '.jpg': 'image/jpeg',
+      '.jpeg': 'image/jpeg',
+      '.png': 'image/png',
+      '.gif': 'image/gif',
+      '.webp': 'image/webp',
+      '.txt': 'text/plain',
+      '.md': 'text/markdown',
+      '.doc': 'application/msword',
+      '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+      '.json': 'application/json'
+    };
+    
+    return mimeTypes[ext] || 'application/octet-stream';
+  }
+
+  static async readTextFile(filePath: string): Promise<string> {
+    try {
+      const content = await fs.promises.readFile(filePath, 'utf-8');
+      return content;
+    } catch (error) {
+      throw new Error(`Failed to read text file: ${error}`);
+    }
+  }
+
+  static async deleteFile(filePath: string): Promise<boolean> {
+    try {
+      await fs.promises.unlink(filePath);
+      return true;
+    } catch (error) {
+      console.error(`Failed to delete file ${filePath}:`, error);
+      return false;
+    }
+  }
+
+  static isTextFile(mimeType: string): boolean {
+    return mimeType.startsWith('text/') || 
+           mimeType === 'application/json' ||
+           mimeType.includes('document');
+  }
+
+  static isImageFile(mimeType: string): boolean {
+    return mimeType.startsWith('image/');
+  }
+
+  static isPdfFile(mimeType: string): boolean {
+    return mimeType === 'application/pdf';
+  }
+
+  static requiresOCR(mimeType: string): boolean {
+    return this.isImageFile(mimeType) || this.isPdfFile(mimeType);
+  }
+}
+
+// Upload validation middleware
+export const validateUpload = (req: Request, res: any, next: any) => {
+  if (!req.files || (Array.isArray(req.files) && req.files.length === 0)) {
+    return res.status(400).json({
+      error: 'No files uploaded',
+      message: 'Please select at least one file to upload'
+    });
+  }
+
+  next();
+};
+
+export default upload;

server/routes.ts

@@ -6,6 +6,7 @@ import { searchRequestSchema } from "@shared/schema";
 import { smartIngestionService } from "./smart-ingestion";
 import { nebiusClient } from "./nebius-client";
 import { modalClient } from "./modal-client";
+import documentRoutes from "./document-routes";
 
 interface GitHubRepo {
   id: number;
@@ -1139,6 +1140,9 @@ Provide a brief, engaging explanation (2-3 sentences) that would be pleasant to
     }
   });
 
+  // Register document routes
+  app.use("/api/documents", documentRoutes);
+
   const httpServer = createServer(app);
   return httpServer;
 }

server/sqlite-storage.ts

@@ -0,0 +1,473 @@
+import Database from 'better-sqlite3';
+import path from 'path';
+import fs from 'fs';
+import { 
+  type Document, 
+  type InsertDocument,
+  type SearchQuery,
+  type InsertSearchQuery,
+  type SearchResult,
+  type InsertSearchResult,
+  type Citation,
+  type InsertCitation,
+  type SearchRequest,
+  type SearchResponse,
+  type DocumentWithContext
+} from "@shared/schema";
+import { IStorage } from './storage';
+
+export class SQLiteStorage implements IStorage {
+  private db: Database.Database;
+
+  constructor(dbPath = './data/knowledgebridge.db') {
+    // Ensure data directory exists
+    const dir = path.dirname(dbPath);
+    if (!fs.existsSync(dir)) {
+      fs.mkdirSync(dir, { recursive: true });
+    }
+
+    this.db = new Database(dbPath);
+    this.initializeTables();
+  }
+
+  private initializeTables() {
+    // Enable foreign keys
+    this.db.pragma('foreign_keys = ON');
+
+    // Create documents table
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS documents (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        title TEXT NOT NULL,
+        content TEXT NOT NULL,
+        source TEXT NOT NULL,
+        source_type TEXT NOT NULL,
+        url TEXT,
+        metadata TEXT, -- JSON string
+        embedding TEXT, -- JSON string
+        file_path TEXT,
+        file_name TEXT,
+        file_size INTEGER,
+        mime_type TEXT,
+        processing_status TEXT NOT NULL DEFAULT 'pending',
+        modal_task_id TEXT,
+        created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+        processed_at DATETIME
+      )
+    `);
+
+    // Create search_queries table
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS search_queries (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        query TEXT NOT NULL,
+        search_type TEXT NOT NULL DEFAULT 'semantic',
+        filters TEXT, -- JSON string
+        results_count INTEGER DEFAULT 0,
+        search_time REAL,
+        created_at DATETIME DEFAULT CURRENT_TIMESTAMP
+      )
+    `);
+
+    // Create search_results table
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS search_results (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        query_id INTEGER NOT NULL,
+        document_id INTEGER NOT NULL,
+        relevance_score REAL NOT NULL,
+        snippet TEXT NOT NULL,
+        rank INTEGER NOT NULL,
+        FOREIGN KEY (query_id) REFERENCES search_queries(id),
+        FOREIGN KEY (document_id) REFERENCES documents(id)
+      )
+    `);
+
+    // Create citations table
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS citations (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        document_id INTEGER NOT NULL,
+        citation_text TEXT NOT NULL,
+        page_number INTEGER,
+        section TEXT,
+        created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+        FOREIGN KEY (document_id) REFERENCES documents(id)
+      )
+    `);
+
+    // Create indexes for better performance
+    this.db.exec(`
+      CREATE INDEX IF NOT EXISTS idx_documents_source_type ON documents(source_type);
+      CREATE INDEX IF NOT EXISTS idx_documents_processing_status ON documents(processing_status);
+      CREATE INDEX IF NOT EXISTS idx_search_results_query_id ON search_results(query_id);
+      CREATE INDEX IF NOT EXISTS idx_search_results_document_id ON search_results(document_id);
+      CREATE INDEX IF NOT EXISTS idx_citations_document_id ON citations(document_id);
+    `);
+  }
+
+  async getDocument(id: number): Promise<Document | undefined> {
+    const stmt = this.db.prepare('SELECT * FROM documents WHERE id = ?');
+    const row = stmt.get(id) as any;
+    return row ? this.mapDocumentRow(row) : undefined;
+  }
+
+  async getDocuments(limit = 50, offset = 0): Promise<Document[]> {
+    const stmt = this.db.prepare('SELECT * FROM documents ORDER BY created_at DESC LIMIT ? OFFSET ?');
+    const rows = stmt.all(limit, offset) as any[];
+    return rows.map(row => this.mapDocumentRow(row));
+  }
+
+  async createDocument(insertDocument: InsertDocument): Promise<Document> {
+    const stmt = this.db.prepare(`
+      INSERT INTO documents (
+        title, content, source, source_type, url, metadata, embedding,
+        file_path, file_name, file_size, mime_type, processing_status, modal_task_id
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `);
+
+    const result = stmt.run(
+      insertDocument.title,
+      insertDocument.content,
+      insertDocument.source,
+      insertDocument.sourceType,
+      insertDocument.url || null,
+      insertDocument.metadata ? JSON.stringify(insertDocument.metadata) : null,
+      insertDocument.embedding || null,
+      (insertDocument as any).filePath || null,
+      (insertDocument as any).fileName || null,
+      (insertDocument as any).fileSize || null,
+      (insertDocument as any).mimeType || null,
+      (insertDocument as any).processingStatus || 'pending',
+      (insertDocument as any).modalTaskId || null
+    );
+
+    const created = await this.getDocument(result.lastInsertRowid as number);
+    if (!created) throw new Error('Failed to create document');
+    return created;
+  }
+
+  async updateDocument(id: number, updateData: Partial<InsertDocument & { processingStatus?: string; modalTaskId?: string; processedAt?: Date }>): Promise<Document | undefined> {
+    const existing = await this.getDocument(id);
+    if (!existing) return undefined;
+
+    const fields: string[] = [];
+    const values: any[] = [];
+
+    Object.entries(updateData).forEach(([key, value]) => {
+      if (value !== undefined) {
+        switch (key) {
+          case 'sourceType':
+            fields.push('source_type = ?');
+            break;
+          case 'processingStatus':
+            fields.push('processing_status = ?');
+            break;
+          case 'modalTaskId':
+            fields.push('modal_task_id = ?');
+            break;
+          case 'filePath':
+            fields.push('file_path = ?');
+            break;
+          case 'fileName':
+            fields.push('file_name = ?');
+            break;
+          case 'fileSize':
+            fields.push('file_size = ?');
+            break;
+          case 'mimeType':
+            fields.push('mime_type = ?');
+            break;
+          case 'processedAt':
+            fields.push('processed_at = ?');
+            value = value instanceof Date ? value.toISOString() : value;
+            break;
+          case 'metadata':
+            fields.push('metadata = ?');
+            value = value ? JSON.stringify(value) : null;
+            break;
+          default:
+            fields.push(`${key} = ?`);
+        }
+        values.push(value);
+      }
+    });
+
+    if (fields.length === 0) return existing;
+
+    values.push(id);
+    const stmt = this.db.prepare(`UPDATE documents SET ${fields.join(', ')} WHERE id = ?`);
+    stmt.run(...values);
+
+    return await this.getDocument(id);
+  }
+
+  async deleteDocument(id: number): Promise<boolean> {
+    const stmt = this.db.prepare('DELETE FROM documents WHERE id = ?');
+    const result = stmt.run(id);
+    return result.changes > 0;
+  }
+
+  async searchDocuments(request: SearchRequest): Promise<SearchResponse> {
+    const startTime = Date.now();
+    
+    let sql = `
+      SELECT * FROM documents 
+      WHERE (title LIKE ? OR content LIKE ?)
+    `;
+    const params: any[] = [`%${request.query}%`, `%${request.query}%`];
+
+    // Add source type filter if specified
+    if (request.filters?.sourceTypes?.length) {
+      const placeholders = request.filters.sourceTypes.map(() => '?').join(',');
+      sql += ` AND source_type IN (${placeholders})`;
+      params.push(...request.filters.sourceTypes);
+    }
+
+    sql += ` ORDER BY 
+      CASE 
+        WHEN title LIKE ? THEN 1 
+        WHEN content LIKE ? THEN 2 
+        ELSE 3 
+      END,
+      created_at DESC
+      LIMIT ? OFFSET ?
+    `;
+    
+    params.push(`%${request.query}%`, `%${request.query}%`, request.limit, request.offset);
+
+    const stmt = this.db.prepare(sql);
+    const rows = stmt.all(...params) as any[];
+    
+    const results = rows.map((row, index) => {
+      const doc = this.mapDocumentRow(row);
+      return {
+        ...doc,
+        relevanceScore: this.calculateRelevanceScore(doc, request.query),
+        snippet: this.extractSnippet(doc.content, request.query),
+        rank: index + 1
+      };
+    });
+
+    const searchTime = (Date.now() - startTime) / 1000;
+
+    // Save search query
+    const searchQuery = await this.createSearchQuery({
+      query: request.query,
+      searchType: request.searchType,
+      filters: request.filters,
+      resultsCount: results.length,
+      searchTime
+    });
+
+    // Save search results
+    for (const doc of results) {
+      await this.createSearchResult({
+        queryId: searchQuery.id,
+        documentId: doc.id,
+        relevanceScore: doc.relevanceScore,
+        snippet: doc.snippet,
+        rank: doc.rank
+      });
+    }
+
+    return {
+      results,
+      totalCount: results.length,
+      searchTime,
+      query: request.query,
+      queryId: searchQuery.id
+    };
+  }
+
+  private calculateRelevanceScore(doc: Document, query: string): number {
+    const queryLower = query.toLowerCase();
+    const titleLower = doc.title.toLowerCase();
+    const contentLower = doc.content.toLowerCase();
+    
+    let score = 0;
+    
+    // Exact title match gets highest score
+    if (titleLower === queryLower) score += 1.0;
+    else if (titleLower.includes(queryLower)) score += 0.8;
+    
+    // Content matches
+    if (contentLower.includes(queryLower)) score += 0.3;
+    
+    // Word-by-word scoring
+    const queryWords = queryLower.split(' ');
+    queryWords.forEach(word => {
+      if (titleLower.includes(word)) score += 0.2;
+      if (contentLower.includes(word)) score += 0.1;
+    });
+
+    return Math.min(score, 1.0);
+  }
+
+  private extractSnippet(content: string, query: string, maxLength = 200): string {
+    const queryLower = query.toLowerCase();
+    const contentLower = content.toLowerCase();
+    
+    const index = contentLower.indexOf(queryLower);
+    if (index === -1) {
+      return content.substring(0, maxLength) + (content.length > maxLength ? '...' : '');
+    }
+
+    const start = Math.max(0, index - 50);
+    const end = Math.min(content.length, index + queryLower.length + 150);
+    
+    let snippet = content.substring(start, end);
+    if (start > 0) snippet = '...' + snippet;
+    if (end < content.length) snippet = snippet + '...';
+    
+    return snippet;
+  }
+
+  async getDocumentsBySourceType(sourceType: string): Promise<Document[]> {
+    const stmt = this.db.prepare('SELECT * FROM documents WHERE source_type = ? ORDER BY created_at DESC');
+    const rows = stmt.all(sourceType) as any[];
+    return rows.map(row => this.mapDocumentRow(row));
+  }
+
+  async getDocumentsByProcessingStatus(status: string): Promise<Document[]> {
+    const stmt = this.db.prepare('SELECT * FROM documents WHERE processing_status = ? ORDER BY created_at DESC');
+    const rows = stmt.all(status) as any[];
+    return rows.map(row => this.mapDocumentRow(row));
+  }
+
+  async createSearchQuery(insertQuery: InsertSearchQuery): Promise<SearchQuery> {
+    const stmt = this.db.prepare(`
+      INSERT INTO search_queries (query, search_type, filters, results_count, search_time)
+      VALUES (?, ?, ?, ?, ?)
+    `);
+
+    const result = stmt.run(
+      insertQuery.query,
+      insertQuery.searchType || 'semantic',
+      insertQuery.filters ? JSON.stringify(insertQuery.filters) : null,
+      insertQuery.resultsCount || null,
+      insertQuery.searchTime || null
+    );
+
+    const created = this.db.prepare('SELECT * FROM search_queries WHERE id = ?').get(result.lastInsertRowid) as any;
+    return this.mapSearchQueryRow(created);
+  }
+
+  async getSearchQueries(limit = 50): Promise<SearchQuery[]> {
+    const stmt = this.db.prepare('SELECT * FROM search_queries ORDER BY created_at DESC LIMIT ?');
+    const rows = stmt.all(limit) as any[];
+    return rows.map(row => this.mapSearchQueryRow(row));
+  }
+
+  async createSearchResult(insertResult: InsertSearchResult): Promise<SearchResult> {
+    const stmt = this.db.prepare(`
+      INSERT INTO search_results (query_id, document_id, relevance_score, snippet, rank)
+      VALUES (?, ?, ?, ?, ?)
+    `);
+
+    const result = stmt.run(
+      insertResult.queryId,
+      insertResult.documentId,
+      insertResult.relevanceScore,
+      insertResult.snippet,
+      insertResult.rank
+    );
+
+    const created = this.db.prepare('SELECT * FROM search_results WHERE id = ?').get(result.lastInsertRowid) as any;
+    return this.mapSearchResultRow(created);
+  }
+
+  async getSearchResults(queryId: number): Promise<SearchResult[]> {
+    const stmt = this.db.prepare('SELECT * FROM search_results WHERE query_id = ? ORDER BY rank');
+    const rows = stmt.all(queryId) as any[];
+    return rows.map(row => this.mapSearchResultRow(row));
+  }
+
+  async createCitation(insertCitation: InsertCitation): Promise<Citation> {
+    const stmt = this.db.prepare(`
+      INSERT INTO citations (document_id, citation_text, page_number, section)
+      VALUES (?, ?, ?, ?)
+    `);
+
+    const result = stmt.run(
+      insertCitation.documentId,
+      insertCitation.citationText,
+      insertCitation.pageNumber || null,
+      insertCitation.section || null
+    );
+
+    const created = this.db.prepare('SELECT * FROM citations WHERE id = ?').get(result.lastInsertRowid) as any;
+    return this.mapCitationRow(created);
+  }
+
+  async getCitationsByDocument(documentId: number): Promise<Citation[]> {
+    const stmt = this.db.prepare('SELECT * FROM citations WHERE document_id = ? ORDER BY created_at DESC');
+    const rows = stmt.all(documentId) as any[];
+    return rows.map(row => this.mapCitationRow(row));
+  }
+
+  async deleteCitation(id: number): Promise<boolean> {
+    const stmt = this.db.prepare('DELETE FROM citations WHERE id = ?');
+    const result = stmt.run(id);
+    return result.changes > 0;
+  }
+
+  private mapDocumentRow(row: any): Document {
+    return {
+      id: row.id,
+      title: row.title,
+      content: row.content,
+      source: row.source,
+      sourceType: row.source_type,
+      url: row.url,
+      metadata: row.metadata ? JSON.parse(row.metadata) : null,
+      embedding: row.embedding,
+      createdAt: new Date(row.created_at),
+      filePath: row.file_path,
+      fileName: row.file_name,
+      fileSize: row.file_size,
+      mimeType: row.mime_type,
+      processingStatus: row.processing_status,
+      modalTaskId: row.modal_task_id,
+      processedAt: row.processed_at ? new Date(row.processed_at) : null,
+    } as Document;
+  }
+
+  private mapSearchQueryRow(row: any): SearchQuery {
+    return {
+      id: row.id,
+      query: row.query,
+      searchType: row.search_type,
+      filters: row.filters ? JSON.parse(row.filters) : null,
+      resultsCount: row.results_count,
+      searchTime: row.search_time,
+      createdAt: new Date(row.created_at)
+    };
+  }
+
+  private mapSearchResultRow(row: any): SearchResult {
+    return {
+      id: row.id,
+      queryId: row.query_id,
+      documentId: row.document_id,
+      relevanceScore: row.relevance_score,
+      snippet: row.snippet,
+      rank: row.rank
+    };
+  }
+
+  private mapCitationRow(row: any): Citation {
+    return {
+      id: row.id,
+      documentId: row.document_id,
+      citationText: row.citation_text,
+      pageNumber: row.page_number,
+      section: row.section,
+      createdAt: new Date(row.created_at)
+    };
+  }
+
+  close() {
+    this.db.close();
+  }
+}

server/storage.ts

@@ -433,4 +433,7 @@ export class MemStorage implements IStorage {
   }
 }
 
-export const storage = new MemStorage();
+import { SQLiteStorage } from './sqlite-storage';
+
+// Use SQLite storage in production, keep MemStorage for testing
+export const storage = process.env.NODE_ENV === 'test' ? new MemStorage() : new SQLiteStorage();

shared/schema.ts

@@ -7,11 +7,18 @@ export const documents = pgTable("documents", {
   title: text("title").notNull(),
   content: text("content").notNull(),
   source: text("source").notNull(),
-  sourceType: text("source_type").notNull(), // pdf, web, code, academic
+  sourceType: text("source_type").notNull(), // pdf, web, code, academic, image
   url: text("url"),
   metadata: jsonb("metadata"), // author, date, tags, etc.
   embedding: text("embedding"), // vector embedding as JSON string
+  filePath: text("file_path"), // local file path for uploaded files
+  fileName: text("file_name"), // original file name
+  fileSize: integer("file_size"), // file size in bytes
+  mimeType: text("mime_type"), // MIME type of uploaded file
+  processingStatus: text("processing_status").notNull().default("pending"), // pending, processing, completed, failed
+  modalTaskId: text("modal_task_id"), // Modal processing task ID
   createdAt: timestamp("created_at").defaultNow().notNull(),
+  processedAt: timestamp("processed_at"),
 });
 
 export const searchQueries = pgTable("search_queries", {
@@ -114,3 +121,32 @@ export interface DocumentWithContext extends Document {
     pageNumber?: number;
   }>;
 }
+
+// File upload schemas
+export const fileUploadSchema = z.object({
+  fileName: z.string().min(1),
+  fileSize: z.number().min(1),
+  mimeType: z.string().min(1),
+  title: z.string().optional(),
+  source: z.string().optional(),
+});
+
+export type FileUpload = z.infer<typeof fileUploadSchema>;
+
+// Document processing schemas
+export const documentProcessingSchema = z.object({
+  documentId: z.number(),
+  operations: z.array(z.enum(["extract_text", "build_index", "generate_embedding"])).default(["extract_text"]),
+  indexName: z.string().optional(),
+});
+
+export type DocumentProcessing = z.infer<typeof documentProcessingSchema>;
+
+// Batch processing schemas
+export const batchProcessingSchema = z.object({
+  documentIds: z.array(z.number()).min(1),
+  operations: z.array(z.enum(["extract_text", "build_index", "generate_embedding"])).default(["extract_text"]),
+  indexName: z.string().optional(),
+});
+
+export type BatchProcessing = z.infer<typeof batchProcessingSchema>;