ok

app/api/v1/batch.py

@@ -8,9 +8,9 @@ import time
 from typing import Annotated, List, Optional
 
 import httpx
-from fastapi import APIRouter, Depends, File, Form, HTTPException, UploadFile
+from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, UploadFile
 
-from app.api.deps import get_converter_service, get_extraction_service, require_auth
+from app.api.deps import get_converter_service, get_extraction_service, get_text_cleaner_service, require_auth
 from app.config import get_settings
 from app.core.logger import get_logger
 from app.models.domain import ConversionError
@@ -22,6 +22,7 @@ from app.models.schemas import (
 )
 from app.services.converter_service import ConverterService
 from app.services.extraction_service import ExtractionService
+from app.services.text_cleaner_service import TextCleanerService
 
 router = APIRouter()
 _logger = get_logger(__name__)
@@ -72,10 +73,12 @@ def _batch_result_from_ok(result) -> BatchFileResult:
 async def batch_files(
     files: Annotated[List[UploadFile], File(description="Files to convert")],
     return_json: bool = Form(False),
+    clean_content: bool = Form(False),
     mappings: Optional[str] = Form(None, description="JSON string with field mappings"),
     token: str = Depends(require_auth),
     converter_service: ConverterService = Depends(get_converter_service),
     extraction_service: ExtractionService = Depends(get_extraction_service),
+    text_cleaner_service: TextCleanerService = Depends(get_text_cleaner_service),
 ):
     if not files:
         raise HTTPException(status_code=400, detail={"success": False, "message": "No files provided."})
@@ -108,7 +111,14 @@ async def batch_files(
         if isinstance(outcome, ConversionError):
             return _batch_result_from_error(f.filename or "unknown", outcome)
 
+        content = outcome.markdown
+        if clean_content:
+            loop2 = asyncio.get_running_loop()
+            content = await loop2.run_in_executor(_thread_pool, text_cleaner_service.clean, content)
+
         result = _batch_result_from_ok(outcome)
+        if clean_content:
+            result.content = content
         if return_json and f.filename:
             loop = asyncio.get_running_loop()
             json_result = await loop.run_in_executor(
@@ -145,9 +155,11 @@ async def batch_files(
 )
 async def batch_urls(
     body: BatchUrlRequest,
+    clean_content: bool = Query(False),
     token: str = Depends(require_auth),
     converter_service: ConverterService = Depends(get_converter_service),
     extraction_service: ExtractionService = Depends(get_extraction_service),
+    text_cleaner_service: TextCleanerService = Depends(get_text_cleaner_service),
 ):
     batch_start = time.perf_counter()
 
@@ -173,6 +185,9 @@ async def batch_urls(
                 if isinstance(outcome, ConversionError):
                     return _batch_result_from_error(url, outcome)
                 result = _batch_result_from_ok(outcome)
+                if clean_content:
+                    loop2 = asyncio.get_running_loop()
+                    result.content = await loop2.run_in_executor(_thread_pool, text_cleaner_service.clean, outcome.markdown)
                 loop = asyncio.get_running_loop()
                 json_result = await loop.run_in_executor(
                     _thread_pool,
@@ -193,11 +208,12 @@ async def batch_urls(
 
         loop = asyncio.get_running_loop()
         outcome = await loop.run_in_executor(_thread_pool, converter_service.convert_url, url)
-        return (
-            _batch_result_from_error(url, outcome)
-            if isinstance(outcome, ConversionError)
-            else _batch_result_from_ok(outcome)
-        )
+        if isinstance(outcome, ConversionError):
+            return _batch_result_from_error(url, outcome)
+        result = _batch_result_from_ok(outcome)
+        if clean_content:
+            result.content = await loop.run_in_executor(_thread_pool, text_cleaner_service.clean, outcome.markdown)
+        return result
 
     tasks = [process_single_url(url) for url in body.urls]
     results = await asyncio.gather(*tasks)

app/api/v1/convert.py

@@ -8,7 +8,7 @@ from typing import Annotated, Any, Dict, Optional
 from urllib.parse import urlparse
 
 import httpx
-from fastapi import APIRouter, Depends, File, Form, HTTPException, UploadFile, status
+from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, UploadFile, status
 from pydantic import BaseModel
 
 from app.config import get_settings
@@ -195,9 +195,11 @@ async def convert_file(
 )
 async def convert_url(
     body: UrlRequest,
+    clean_content: bool = Query(False),
     token: str = Depends(require_auth),
     converter_service: ConverterService = Depends(get_converter_service),
     extraction_service: ExtractionService = Depends(get_extraction_service),
+    text_cleaner_service: TextCleanerService = Depends(get_text_cleaner_service),
 ):
     _logger.info("Received request to convert URL: %s", body.url)
     parsed = urlparse(body.url)
@@ -227,6 +229,8 @@ async def convert_url(
                 raw_data=raw_data,
                 mappings=body.mappings,
                 extraction_service=extraction_service,
+                clean_content=clean_content,
+                text_cleaner_service=text_cleaner_service,
             )
             _logger.info("Request completed for URL %s", body.url)
             return response
@@ -247,6 +251,8 @@ async def convert_url(
         filename=filename,
         mappings=body.mappings,
         extraction_service=extraction_service,
+        clean_content=clean_content,
+        text_cleaner_service=text_cleaner_service,
     )
     _logger.info("Request completed for URL %s", body.url)
     return response