ok

app/api/v1/vector_stores.py

@@ -1,11 +1,9 @@
 from __future__ import annotations
 
 import asyncio
-import os
-import tempfile
 import time
 
-from fastapi import APIRouter, Depends, File, Form, HTTPException, UploadFile, status
+from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, UploadFile, status
 
 from app.api.deps import get_vector_store_service, require_auth
 from app.core.logger import get_logger
@@ -15,6 +13,7 @@ from app.models.schemas import (
     DeleteResponse,
     DocumentIngestRequest,
     DocumentIngestResponse,
+    DocumentIngestUrlRequest,
     SearchRequest,
     SearchResponse,
     VectorStoreCreate,
@@ -22,12 +21,28 @@ from app.models.schemas import (
     VectorStoreResponse,
 )
 from app.services.converter_service import ConverterService
+from app.services.text_cleaner_service import TextCleanerService
 from app.services.vector_store_service import VectorStoreService
 
 router = APIRouter()
 logger = get_logger(__name__)
 
 
+async def _process_pdf_bytes(raw: bytes, source: str, clean_content: bool) -> str | ConversionError:
+    if len(raw) < 5 or raw[:5] != b"%PDF-":
+        return ConversionError(source=source, error_type="ValueError", message="Not a valid PDF", duration_ms=0)
+    loop = asyncio.get_running_loop()
+    converter = ConverterService()
+    result = await loop.run_in_executor(None, converter.convert_stream, raw, source)
+    if isinstance(result, ConversionError):
+        return result
+    text = result.markdown
+    if clean_content:
+        text_cleaner = TextCleanerService()
+        text = await loop.run_in_executor(None, text_cleaner.clean, text)
+    return text
+
+
 @router.post(
     "/vector-stores",
     response_model=VectorStoreResponse,
@@ -196,6 +211,7 @@ async def ingest_pdf_document(
     doc_id: str = Form(..., min_length=1, max_length=256),
     chunk_size: int = Form(512, ge=64, le=4096),
     chunk_overlap: int = Form(64, ge=0, le=512),
+    clean_content: bool = Query(True, description="Clean markdown text after PDF conversion"),
     token: str = Depends(require_auth),
     vector_store_service: VectorStoreService = Depends(get_vector_store_service),
 ) -> DocumentIngestResponse:
@@ -208,67 +224,131 @@ async def ingest_pdf_document(
     if not file.filename or not file.filename.lower().endswith(".pdf"):
         raise HTTPException(status_code=400, detail="Only .pdf files are accepted")
 
-    tmp_path = None
     try:
         raw = await file.read()
-        if len(raw) < 5 or raw[:5] != b"%PDF-":
-            raise HTTPException(status_code=400, detail="File is not a valid PDF")
-
-        suffix = ".pdf"
-        with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp:
-            tmp.write(raw)
-            tmp_path = tmp.name
-
-        converter = ConverterService()
-        loop = asyncio.get_running_loop()
-        result = await loop.run_in_executor(None, converter.convert_file, tmp_path)
-
-        if isinstance(result, ConversionError):
-            logger.error("PDF conversion failed: %s", result.message)
-            return DocumentIngestResponse(
-                success=False,
-                vector_store_id=store_id,
-                doc_id=doc_id,
-                chunks_ingested=0,
-                time_ms=0,
-                error=result.message,
-            )
-
-        text = result.markdown
-        source = file.filename
+    finally:
+        await file.close()
+
+    text = await _process_pdf_bytes(raw, file.filename, clean_content)
+    if isinstance(text, ConversionError):
+        return DocumentIngestResponse(
+            success=False,
+            vector_store_id=store_id,
+            doc_id=doc_id,
+            chunks_ingested=0,
+            time_ms=0,
+            error=text.message,
+        )
 
+    try:
         chunks, elapsed = await vector_store_service.ingest_document(
             store_id=store_id,
             doc_id=doc_id,
             text=text,
-            source=source,
+            source=file.filename,
             chunk_size=chunk_size,
             chunk_overlap=chunk_overlap,
         )
-
+    except Exception as exc:
+        logger.error("PDF ingest failed for store %s: %s", store_id, exc)
         return DocumentIngestResponse(
-            success=True,
+            success=False,
             vector_store_id=store_id,
             doc_id=doc_id,
-            chunks_ingested=chunks,
-            time_ms=round(elapsed, 3),
+            chunks_ingested=0,
+            time_ms=0,
+            error=str(exc),
+        )
+
+    return DocumentIngestResponse(
+        success=True,
+        vector_store_id=store_id,
+        doc_id=doc_id,
+        chunks_ingested=chunks,
+        time_ms=round(elapsed, 3),
+    )
+
+
+@router.post(
+    "/vector-stores/{store_id}/documents/upload-url",
+    response_model=DocumentIngestResponse,
+    summary="Ingest a PDF from a URL into the vector store",
+)
+async def ingest_pdf_url(
+    store_id: str,
+    body: DocumentIngestUrlRequest,
+    clean_content: bool = Query(True, description="Clean markdown text after PDF conversion"),
+    token: str = Depends(require_auth),
+    vector_store_service: VectorStoreService = Depends(get_vector_store_service),
+) -> DocumentIngestResponse:
+    record = vector_store_service.get_store(store_id)
+    if record is None:
+        raise HTTPException(status_code=404, detail=f"Vector store {store_id} not found")
+
+    import httpx
+
+    try:
+        async with httpx.AsyncClient(timeout=30.0, follow_redirects=True) as client:
+            resp = await client.get(body.url)
+            resp.raise_for_status()
+            raw = resp.content
+    except httpx.HTTPStatusError as exc:
+        return DocumentIngestResponse(
+            success=False,
+            vector_store_id=store_id,
+            doc_id=body.doc_id,
+            chunks_ingested=0,
+            time_ms=0,
+            error=f"Failed to fetch PDF from URL: HTTP {exc.response.status_code}",
+        )
+    except httpx.RequestError as exc:
+        return DocumentIngestResponse(
+            success=False,
+            vector_store_id=store_id,
+            doc_id=body.doc_id,
+            chunks_ingested=0,
+            time_ms=0,
+            error=f"Failed to fetch PDF from URL: {exc}",
+        )
+
+    text = await _process_pdf_bytes(raw, body.url, clean_content)
+    if isinstance(text, ConversionError):
+        return DocumentIngestResponse(
+            success=False,
+            vector_store_id=store_id,
+            doc_id=body.doc_id,
+            chunks_ingested=0,
+            time_ms=0,
+            error=text.message,
+        )
+
+    try:
+        chunks, elapsed = await vector_store_service.ingest_document(
+            store_id=store_id,
+            doc_id=body.doc_id,
+            text=text,
+            source=body.url,
+            chunk_size=body.chunk_size,
+            chunk_overlap=body.chunk_overlap,
         )
-    except HTTPException:
-        raise
     except Exception as exc:
-        logger.error("PDF ingest failed for store %s: %s", store_id, exc)
+        logger.error("PDF URL ingest failed for store %s: %s", store_id, exc)
         return DocumentIngestResponse(
             success=False,
             vector_store_id=store_id,
-            doc_id=doc_id,
+            doc_id=body.doc_id,
             chunks_ingested=0,
             time_ms=0,
             error=str(exc),
         )
-    finally:
-        if tmp_path and os.path.exists(tmp_path):
-            os.unlink(tmp_path)
-        await file.close()
+
+    return DocumentIngestResponse(
+        success=True,
+        vector_store_id=store_id,
+        doc_id=body.doc_id,
+        chunks_ingested=chunks,
+        time_ms=round(elapsed, 3),
+    )
 
 
 @router.post(

app/models/schemas.py

@@ -611,6 +611,13 @@ class DocumentIngestResponse(BaseModel):
     error: Optional[str] = None
 
 
+class DocumentIngestUrlRequest(BaseModel):
+    url: str = Field(..., min_length=1, description="URL of the PDF file to ingest")
+    doc_id: str = Field(..., min_length=1, max_length=256)
+    chunk_size: int = Field(512, ge=64, le=4096)
+    chunk_overlap: int = Field(64, ge=0, le=512)
+
+
 class SearchRequest(BaseModel):
     query: str = Field(..., min_length=1, max_length=5000, description="Natural language query")
     top_k: int = Field(default=10, ge=1, le=100, description="Max results to return")

app/services/vector_store_service.py

@@ -358,10 +358,6 @@ class VectorStoreService:
 
     # --- Async public API ---
 
-    async def _run_in_thread(self, fn, *args, **kwargs):
-        loop = asyncio.get_running_loop()
-        return await loop.run_in_executor(self._thread_pool, _run_sync, lambda: fn(*args, **kwargs))
-
     async def _run_sync_fn(self, fn):
         loop = asyncio.get_running_loop()
         return await loop.run_in_executor(self._thread_pool, fn)