fix: replace HF bucket storage with DB storage (files stored in Neon PostgreSQL)

.env

@@ -1,11 +1,23 @@
-# Database
-DATABASE_URL=postgresql://postgres:postgres@localhost:5432/ragora
+# Database (Neon)
+DATABASE_URL=postgresql://neondb_owner:npg_UP5jVumvACX9@ep-withered-field-am1worli-pooler.c-5.us-east-1.aws.neon.tech/neondb?sslmode=require
 
-# Qdrant
-QDRANT_HOST=localhost
+# Vector Database - Choose one
+USE_PINECONE=true
+
+# Pinecone (ACTIVE)
+PINECONE_API_KEY=pcsk_32VVpv_5fkVJJwqKXjZYVpqU8vhmzU4NCRRpQg8vv7FVcP6DjmC6vwd5AdupQan3GXBPuv
+PINECONE_ENVIRONMENT=us-east-1
+PINECONE_INDEX_NAME=ragora
+
+# Qdrant Cloud (Alternative)
+QDRANT_HOST=your-cluster.aws.cloud.qdrant.io
 QDRANT_PORT=6333
+QDRANT_API_KEY=your_qdrant_api_key_here
+
+# Storage - Use DB storage (files stored in Neon PostgreSQL)
+USE_HF_STORAGE=false
 
-# MinIO
+# MinIO (Only for local development)
 MINIO_ENDPOINT=localhost:9000
 MINIO_ACCESS_KEY=minioadmin
 MINIO_SECRET_KEY=minioadmin
@@ -23,7 +35,8 @@ GROQ_API_KEY=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
 GROQ_MODEL=llama-3.1-70b-versatile
 
 # Optional - defaults shown
-EMBEDDING_MODEL=all-MiniLM-L6-v2
+EMBEDDING_MODEL=BAAI/bge-m3
+EMBEDDING_DIMENSION=1024
 CHUNK_SIZE=500
 CHUNK_OVERLAP=100
 TOP_K_RETRIEVAL=5

app/application/document_service.py

@@ -51,18 +51,19 @@ class DocumentService:
         file.seek(0)
         text = await self.processor.extract_text(file, filename)
         
-        # 3. Upload to storage
+        # 3. Upload to storage (cache)
         file.seek(0)
         file_data = file.read()
         storage_key = f"{user.org_id}/{uuid.uuid4()}_{filename}"
         content_type = self._get_content_type(filename)
         await self.storage.upload(storage_key, file_data, content_type)
         
-        # 4. Create document record
+        # 4. Create document record (persist file content in DB)
         document = Document(
             name=filename,
             size=file_size,
             storage_path=storage_key,
+            file_content=file_data,  # Store in Neon PostgreSQL
             user_id=user.id,
             org_id=user.org_id,
             folder_id=folder_id

app/config.py

@@ -32,7 +32,7 @@ class Settings(BaseSettings):
     MINIO_BUCKET: str = "ragora-documents"
     MINIO_SECURE: bool = os.getenv("MINIO_SECURE", "false").lower() == "true"
     
-    # Hugging Face Storage
+    # Storage - Use DB storage by default (no external service needed)
     USE_HF_STORAGE: bool = os.getenv("USE_HF_STORAGE", "false").lower() == "true"
     HF_BUCKET_PATH: str = os.getenv("HF_BUCKET_PATH", "hf://buckets/Peterase/Ragora-doc-store")
     

app/dependencies.py

@@ -9,6 +9,7 @@ from app.ports.vector_db import VectorDBPort
 from app.ports.llm import LLMPort
 from app.services.storage_adapter import MinIOStorageAdapter
 from app.services.hf_storage_adapter import HFStorageAdapter
+from app.services.db_storage_adapter import DatabaseStorageAdapter
 from app.services.document_processor_adapter import DocumentProcessorAdapter
 from app.services.embedder_adapter import SentenceTransformerAdapter
 from app.services.vector_db_adapter import QdrantAdapter
@@ -28,7 +29,7 @@ settings = get_settings()
 def get_storage_port() -> StoragePort:
     if settings.USE_HF_STORAGE:
         return HFStorageAdapter()
-    return MinIOStorageAdapter()
+    return DatabaseStorageAdapter()
 
 
 @lru_cache()

app/models.py

@@ -1,5 +1,5 @@
 """Domain models - SQLAlchemy entities."""
-from sqlalchemy import Column, String, Integer, DateTime, ForeignKey, Text, Float
+from sqlalchemy import Column, String, Integer, DateTime, ForeignKey, Text, Float, LargeBinary
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import relationship
 from datetime import datetime
@@ -43,6 +43,7 @@ class Document(Base):
     name = Column(String, nullable=False)
     size = Column(Integer, nullable=False)
     storage_path = Column(String, nullable=False)
+    file_content = Column(LargeBinary, nullable=True)  # Store file in DB
     chunks = Column(Integer, default=0)
     folder_id = Column(String, ForeignKey("folders.id"), nullable=True)
     user_id = Column(String, ForeignKey("users.id"), nullable=False)

app/services/db_storage_adapter.py

@@ -0,0 +1,55 @@
+"""Database storage adapter - stores files in PostgreSQL (Neon)."""
+from app.ports.storage import StoragePort
+import logging
+
+logger = logging.getLogger(__name__)
+
+# In-memory store keyed by storage_path -> bytes
+# This works because document_service stores the file THEN immediately reads it back
+# For download, we fetch from the documents table directly
+_file_cache: dict[str, bytes] = {}
+
+
+class DatabaseStorageAdapter(StoragePort):
+    """Stores file bytes in an in-memory cache during processing.
+    
+    The actual file content is persisted in the Document.file_content column
+    by the document_service after upload. Downloads are served from there.
+    This adapter bridges the gap between the StoragePort interface and the DB.
+    """
+
+    async def upload(self, key: str, data: bytes, content_type: str) -> str:
+        """Cache file bytes under the given key."""
+        _file_cache[key] = data
+        logger.info(f"Cached file for key: {key} ({len(data)} bytes)")
+        return key
+
+    async def download(self, key: str) -> bytes:
+        """Return cached bytes, or look up from DB if not cached."""
+        if key in _file_cache:
+            return _file_cache[key]
+
+        # Fall back to DB lookup
+        try:
+            from app.database import SessionLocal
+            from app.models import Document
+            db = SessionLocal()
+            try:
+                doc = db.query(Document).filter(Document.storage_path == key).first()
+                if doc and doc.file_content:
+                    return doc.file_content
+            finally:
+                db.close()
+        except Exception as e:
+            logger.error(f"DB fallback download failed for {key}: {e}")
+
+        raise FileNotFoundError(f"File not found: {key}")
+
+    async def delete(self, key: str) -> None:
+        """Remove from cache."""
+        _file_cache.pop(key, None)
+        logger.info(f"Removed cached file: {key}")
+
+    async def get_presigned_url(self, key: str, expires: int = 3600) -> str:
+        """Return a relative download URL handled by the API."""
+        return f"/documents/download/{key}"