added return json

app/api/v1/batch.py

@@ -2,13 +2,15 @@ from __future__ import annotations
 
 import asyncio
 import concurrent.futures
+import json as json_mod
 import os
 import time
-from typing import Annotated, List
+from typing import Annotated, List, Optional
 
-from fastapi import APIRouter, Depends, File, HTTPException, UploadFile
+import httpx
+from fastapi import APIRouter, Depends, File, Form, HTTPException, UploadFile
 
-from app.api.deps import get_converter_service, require_auth
+from app.api.deps import get_converter_service, get_extraction_service, require_auth
 from app.config import get_settings
 from app.core.logger import get_logger
 from app.models.domain import ConversionError
@@ -19,6 +21,7 @@ from app.models.schemas import (
     ConversionMetadata,
 )
 from app.services.converter_service import ConverterService
+from app.services.extraction_service import ExtractionService
 
 router = APIRouter()
 _logger = get_logger(__name__)
@@ -68,14 +71,24 @@ def _batch_result_from_ok(result) -> BatchFileResult:
 )
 async def batch_files(
     files: Annotated[List[UploadFile], File(description="Files to convert")],
+    return_json: bool = Form(False),
+    mappings: Optional[str] = Form(None, description="JSON string with field mappings"),
     token: str = Depends(require_auth),
     converter_service: ConverterService = Depends(get_converter_service),
+    extraction_service: ExtractionService = Depends(get_extraction_service),
 ):
     if not files:
         raise HTTPException(status_code=400, detail={"success": False, "message": "No files provided."})
     if len(files) > _MAX_BATCH_FILES:
         raise HTTPException(status_code=400, detail={"success": False, "message": f"Maximum {_MAX_BATCH_FILES} files per batch."})
 
+    parsed_mappings = None
+    if mappings:
+        try:
+            parsed_mappings = json_mod.loads(mappings)
+        except json_mod.JSONDecodeError:
+            raise HTTPException(status_code=400, detail={"success": False, "message": "Invalid JSON in mappings parameter."})
+
     batch_start = time.perf_counter()
 
     async def process_single_file(f: UploadFile) -> BatchFileResult:
@@ -92,11 +105,23 @@ async def batch_files(
             )
         loop = asyncio.get_running_loop()
         outcome = await loop.run_in_executor(_thread_pool, converter_service.convert_stream, raw, f.filename or "upload")
-        return (
-            _batch_result_from_error(f.filename or "unknown", outcome)
-            if isinstance(outcome, ConversionError)
-            else _batch_result_from_ok(outcome)
-        )
+        if isinstance(outcome, ConversionError):
+            return _batch_result_from_error(f.filename or "unknown", outcome)
+
+        result = _batch_result_from_ok(outcome)
+        if return_json and f.filename:
+            loop = asyncio.get_running_loop()
+            json_result = await loop.run_in_executor(
+                _thread_pool,
+                extraction_service.extract_structured,
+                f.filename,
+                outcome.markdown,
+                parsed_mappings,
+                raw,
+            )
+            result.json_content = json_result if "error" not in json_result else None
+            result.error = json_result.get("error") if "error" in json_result else None
+        return result
 
     tasks = [process_single_file(f) for f in files]
     results = await asyncio.gather(*tasks)
@@ -122,11 +147,50 @@ async def batch_urls(
     body: BatchUrlRequest,
     token: str = Depends(require_auth),
     converter_service: ConverterService = Depends(get_converter_service),
+    extraction_service: ExtractionService = Depends(get_extraction_service),
 ):
     batch_start = time.perf_counter()
 
     async def process_single_url(url: str) -> BatchFileResult:
         _logger.info("Batch processing URL: %s", url)
+        from urllib.parse import urlparse
+        parsed = urlparse(url)
+        filename = parsed.path.split("/")[-1] or "url_content"
+
+        if body.return_json:
+            try:
+                async with httpx.AsyncClient(timeout=30.0, follow_redirects=True) as client:
+                    resp = await client.get(url)
+                    resp.raise_for_status()
+                raw_data = resp.content
+                if len(raw_data) > _MAX_UPLOAD_BYTES:
+                    return BatchFileResult(
+                        filename=filename, success=False, time_ms=0,
+                        error=f"File exceeds {_settings.max_upload_mb} MB limit.",
+                    )
+                loop = asyncio.get_running_loop()
+                outcome = await loop.run_in_executor(_thread_pool, converter_service.convert_stream, raw_data, filename)
+                if isinstance(outcome, ConversionError):
+                    return _batch_result_from_error(url, outcome)
+                result = _batch_result_from_ok(outcome)
+                loop = asyncio.get_running_loop()
+                json_result = await loop.run_in_executor(
+                    _thread_pool,
+                    extraction_service.extract_structured,
+                    filename,
+                    outcome.markdown,
+                    body.mappings,
+                    raw_data,
+                )
+                result.json_content = json_result if "error" not in json_result else None
+                result.error = json_result.get("error") if "error" in json_result else None
+                return result
+            except httpx.HTTPError as exc:
+                return BatchFileResult(
+                    filename=filename, success=False, time_ms=0,
+                    error=f"Failed to fetch URL: {exc}",
+                )
+
         loop = asyncio.get_running_loop()
         outcome = await loop.run_in_executor(_thread_pool, converter_service.convert_url, url)
         return (

app/models/schemas.py

@@ -43,6 +43,8 @@ class UrlRequest(BaseModel):
 
 class BatchUrlRequest(BaseModel):
     urls: List[str]
+    return_json: bool = False
+    mappings: Optional[Dict[str, Dict[str, Any]]] = None
 
     @field_validator("urls")
     @classmethod
@@ -60,6 +62,7 @@ class BatchFileResult(BaseModel):
     success: bool
     time_ms: float
     content: Optional[str] = None
+    json_content: Optional[Any] = None
     error: Optional[str] = None
     metadata: Optional[ConversionMetadata] = None