ok

.gitignore

@@ -0,0 +1,119 @@
+__pycache__/
+*.py[cod]
+*$py.class
+
+*.so
+
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+*.manifest
+*.spec
+
+pip-log.txt
+pip-delete-this-directory.txt
+
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+*.mo
+*.pot
+
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+instance/
+.webassets-cache
+
+.scrapy
+
+docs/_build/
+
+target/
+
+.ipynb_checkpoints
+
+profile_default/
+ipython_config.py
+
+.python-version
+
+__pypackages__/
+
+celerybeat-schedule
+celerybeat.pid
+
+*.sage.py
+
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+.spyderproject
+.spyproject
+
+.ropeproject
+
+/site
+
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+.pyre/
+
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+.DS_Store
+.DS_Store?
+._*
+.Spotlight-V100
+.Trashes
+ehthumbs.db
+Thumbs.db
+
+logs/
+*.log
+
+*.db
+*.sqlite3
+
+*.tmp
+*.temp

Dockerfile

@@ -0,0 +1,40 @@
+FROM python:3.12-slim
+
+LABEL maintainer="reconciliation-non-ai-extractor"
+LABEL description="Enterprise document extraction API with Microsoft MarkItDown, RapidOCR, and spaCy"
+LABEL version="1.0.0"
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    curl \
+    ffmpeg \
+    libmagic1 \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN groupadd --gid 1000 appuser && \
+    useradd --uid 1000 --gid appuser --shell /bin/bash --create-home appuser
+
+WORKDIR /app
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir --upgrade pip && \
+    pip install --no-cache-dir -r requirements.txt && \
+    python -m spacy download en_core_web_sm
+
+COPY --chown=appuser:appuser . .
+
+RUN mkdir -p /app/logs && \
+    chown -R appuser:appuser /app/logs
+
+RUN chmod +x /app/start.sh
+
+USER appuser
+
+ENV PYTHONPATH=/app
+ENV PYTHONUNBUFFERED=1
+
+EXPOSE 7860
+
+HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 \
+    CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:7860/health')" || exit 1
+
+CMD ["/bin/bash", "/app/start.sh"]

README.md

@@ -0,0 +1,9 @@
+---
+title: MarkItDown API — v2.1.0
+emoji: ⚡
+colorFrom: red
+colorTo: purple
+sdk: docker
+pinned: false
+app_port: 7860
+---

__init__.py

@@ -0,0 +1,4 @@
+from __future__ import annotations
+
+__version__ = "1.0.0"
+__app_name__ = "reconciliation-non-ai-extractor"

app/__init__.py

@@ -0,0 +1 @@
+from __future__ import annotations

app/api/__init__.py

@@ -0,0 +1 @@
+from __future__ import annotations

app/api/deps.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+from fastapi import Depends
+
+from app.core.security import require_api_key
+from app.services.auth_service import AuthService
+from app.services.converter_service import ConverterService
+from app.services.extraction_service import ExtractionService
+from app.services.ocr_service import OCRService
+
+
+def get_auth_service() -> AuthService:
+    return AuthService()
+
+
+def get_converter_service() -> ConverterService:
+    return ConverterService()
+
+
+def get_ocr_service() -> OCRService:
+    return OCRService()
+
+
+def get_extraction_service() -> ExtractionService:
+    return ExtractionService()
+
+
+def require_auth(token: str = Depends(require_api_key)) -> str:
+    return token

app/api/server.py

@@ -0,0 +1,92 @@
+from __future__ import annotations
+
+import asyncio
+import os
+import time
+from contextlib import asynccontextmanager
+from datetime import datetime, timezone
+
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.middleware.gzip import GZipMiddleware
+
+from app.config import get_settings
+from app.core.banner import print_banner
+from app.core.constants import SUPPORTED_EXTENSIONS
+from app.core.logger import get_logger
+from app.api.v1.router import api_v1_router
+
+_logger = get_logger(__name__)
+_settings = get_settings()
+_START_TIME = time.time()
+
+
+async def _self_ping():
+    import httpx
+    health_url = f"http://{_settings.host}:{_settings.port}/health"
+    while True:
+        try:
+            async with httpx.AsyncClient(timeout=30.0) as client:
+                response = await client.get(health_url)
+                if response.status_code == 200:
+                    _logger.info("Self-ping successful: %s", health_url)
+                else:
+                    _logger.warning("Self-ping returned: %s - %s", health_url, response.status_code)
+        except Exception as exc:
+            _logger.error("Self-ping error: %s", exc)
+        await asyncio.sleep(3600)
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    print_banner()
+    _logger.info("Server ready at http://%s:%s", _settings.host, _settings.port)
+    _logger.info("Swagger UI : http://%s:%s/docs", _settings.host, _settings.port)
+    _logger.info("ReDoc      : http://%s:%s/redoc", _settings.host, _settings.port)
+    _logger.info("Started at : %s", datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC"))
+    asyncio.create_task(_self_ping())
+    _logger.info("Self-ping task started (every hour)")
+    yield
+    _logger.info("Application shutting down gracefully")
+
+
+def create_application() -> FastAPI:
+    app = FastAPI(
+        title=_settings.app_name,
+        description="Document-to-Markdown and structured data extraction API powered by Microsoft MarkItDown, RapidOCR, and spaCy.",
+        version=_settings.app_version,
+        docs_url="/docs",
+        redoc_url="/redoc",
+        openapi_tags=[
+            {"name": "Convert", "description": "Single-file and single-URL conversion"},
+            {"name": "Batch", "description": "Bulk conversion of files and URLs"},
+            {"name": "System", "description": "Health, info, and supported formats"},
+        ],
+        lifespan=lifespan,
+    )
+
+    app.add_middleware(GZipMiddleware, minimum_size=1000)
+    app.add_middleware(
+        CORSMiddleware,
+        allow_origins=["*"],
+        allow_methods=["*"],
+        allow_headers=["*"],
+    )
+
+    app.include_router(api_v1_router, prefix="/v1")
+
+    @app.get("/health", include_in_schema=False)
+    async def root_health():
+        return {"status": "ok", "version": _settings.app_version}
+
+    @app.get("/ping", include_in_schema=False)
+    async def ping():
+        return {"message": f"{_settings.app_name} is running..."}
+
+    _max_workers = min(32, (os.cpu_count() or 1) + 4)
+    _logger.info("Initialized thread pool with %s workers", _max_workers)
+
+    return app
+
+
+app = create_application()

app/api/v1/__init__.py

@@ -0,0 +1 @@
+from __future__ import annotations

app/api/v1/batch.py

@@ -0,0 +1,150 @@
+from __future__ import annotations
+
+import asyncio
+import concurrent.futures
+import os
+import time
+from typing import Annotated, List
+
+from fastapi import APIRouter, Depends, File, HTTPException, UploadFile
+
+from app.api.deps import get_converter_service, require_auth
+from app.config import get_settings
+from app.core.logger import get_logger
+from app.models.domain import ConversionError
+from app.models.schemas import (
+    BatchFileResult,
+    BatchResponse,
+    BatchUrlRequest,
+    ConversionMetadata,
+)
+from app.services.converter_service import ConverterService
+
+router = APIRouter()
+_logger = get_logger(__name__)
+_settings = get_settings()
+_MAX_UPLOAD_BYTES = _settings.max_upload_bytes
+_MAX_BATCH_FILES = _settings.max_batch_files
+_MAX_WORKERS = min(32, (os.cpu_count() or 1) + 4)
+_thread_pool = concurrent.futures.ThreadPoolExecutor(max_workers=_MAX_WORKERS)
+
+
+def _build_metadata(result) -> ConversionMetadata:
+    return ConversionMetadata(
+        source=result.source,
+        char_count=result.char_count,
+        word_count=result.word_count,
+        line_count=result.line_count,
+        file_size_bytes=result.file_size_bytes,
+        mime_type=result.mime_type,
+        content_hash=result.content_hash,
+        token_estimate=result.token_estimate,
+    )
+
+
+def _batch_result_from_error(name: str, err: ConversionError) -> BatchFileResult:
+    return BatchFileResult(
+        filename=name,
+        success=False,
+        time_ms=round(err.duration_ms, 3),
+        error=err.message,
+    )
+
+
+def _batch_result_from_ok(result) -> BatchFileResult:
+    return BatchFileResult(
+        filename=result.source,
+        success=True,
+        time_ms=round(result.duration_ms, 3),
+        content=result.markdown,
+        metadata=_build_metadata(result),
+    )
+
+
+@router.post(
+    "/batch/files",
+    response_model=BatchResponse,
+    summary="Convert multiple files (up to 10)",
+)
+async def batch_files(
+    files: Annotated[List[UploadFile], File(description="Files to convert")],
+    token: str = require_auth,
+    converter_service: ConverterService = Depends(get_converter_service),
+):
+    if not files:
+        raise HTTPException(status_code=400, detail={"success": False, "message": "No files provided."})
+    if len(files) > _MAX_BATCH_FILES:
+        raise HTTPException(status_code=400, detail={"success": False, "message": f"Maximum {_MAX_BATCH_FILES} files per batch."})
+
+    batch_start = time.perf_counter()
+
+    async def process_single_file(f: UploadFile) -> BatchFileResult:
+        if f is None:
+            return BatchFileResult(filename="unknown", success=False, time_ms=0, error="File object is None.")
+        _logger.info("Batch processing file: %s", f.filename)
+        raw = await f.read()
+        if len(raw) > _MAX_UPLOAD_BYTES:
+            return BatchFileResult(
+                filename=f.filename or "unknown",
+                success=False,
+                time_ms=0,
+                error=f"File exceeds {_settings.max_upload_mb} MB limit.",
+            )
+        loop = asyncio.get_running_loop()
+        outcome = await loop.run_in_executor(_thread_pool, converter_service.convert_stream, raw, f.filename or "upload")
+        return (
+            _batch_result_from_error(f.filename or "unknown", outcome)
+            if isinstance(outcome, ConversionError)
+            else _batch_result_from_ok(outcome)
+        )
+
+    tasks = [process_single_file(f) for f in files]
+    results = await asyncio.gather(*tasks)
+
+    total_ms = round((time.perf_counter() - batch_start) * 1000, 3)
+    succeeded = sum(1 for r in results if r.success)
+    _logger.info("Batch files completed. Succeeded: %s/%s", succeeded, len(results))
+    return BatchResponse(
+        total=len(results),
+        succeeded=succeeded,
+        failed=len(results) - succeeded,
+        total_time_ms=total_ms,
+        results=results,
+    )
+
+
+@router.post(
+    "/batch/urls",
+    response_model=BatchResponse,
+    summary="Convert multiple URLs (up to 20)",
+)
+async def batch_urls(
+    body: BatchUrlRequest,
+    token: str = require_auth,
+    converter_service: ConverterService = Depends(get_converter_service),
+):
+    batch_start = time.perf_counter()
+
+    async def process_single_url(url: str) -> BatchFileResult:
+        _logger.info("Batch processing URL: %s", url)
+        loop = asyncio.get_running_loop()
+        outcome = await loop.run_in_executor(_thread_pool, converter_service.convert_url, url)
+        return (
+            _batch_result_from_error(url, outcome)
+            if isinstance(outcome, ConversionError)
+            else _batch_result_from_ok(outcome)
+        )
+
+    tasks = [process_single_url(url) for url in body.urls]
+    results = await asyncio.gather(*tasks)
+
+    total_ms = round((time.perf_counter() - batch_start) * 1000, 3)
+    succeeded = sum(1 for r in results if r.success)
+    _logger.info("Batch URLs completed. Succeeded: %s/%s", succeeded, len(results))
+    return BatchResponse(
+        total=len(results),
+        succeeded=succeeded,
+        failed=len(results) - succeeded,
+        total_time_ms=total_ms,
+        results=results,
+    )

app/api/v1/convert.py

@@ -0,0 +1,217 @@
+from __future__ import annotations
+
+import asyncio
+import concurrent.futures
+import json as json_mod
+import os
+from typing import Annotated, Any, Dict, Optional
+from urllib.parse import urlparse
+
+import httpx
+from fastapi import APIRouter, Depends, File, Form, HTTPException, UploadFile, status
+from pydantic import BaseModel
+
+from app.config import get_settings
+from app.api.deps import (
+    get_converter_service,
+    get_extraction_service,
+    require_auth,
+)
+from app.core.logger import get_logger
+from app.models.domain import ConversionError
+from app.models.schemas import ConversionMetadata, ConversionResponse, UrlRequest
+from app.services.converter_service import ConverterService
+from app.services.extraction_service import ExtractionService
+
+router = APIRouter()
+_logger = get_logger(__name__)
+_settings = get_settings()
+_MAX_UPLOAD_BYTES = _settings.max_upload_bytes
+_MAX_WORKERS = min(32, (os.cpu_count() or 1) + 4)
+_thread_pool = concurrent.futures.ThreadPoolExecutor(max_workers=_MAX_WORKERS)
+
+
+def _build_metadata(result) -> ConversionMetadata:
+    return ConversionMetadata(
+        source=result.source,
+        char_count=result.char_count,
+        word_count=result.word_count,
+        line_count=result.line_count,
+        file_size_bytes=result.file_size_bytes,
+        mime_type=result.mime_type,
+        content_hash=result.content_hash,
+        token_estimate=result.token_estimate,
+    )
+
+
+async def _build_response(
+    result,
+    *,
+    return_json: bool = False,
+    filename: Optional[str] = None,
+    raw_data: Optional[bytes] = None,
+    mappings: Optional[Dict[str, Dict[str, Any]]] = None,
+    extraction_service: ExtractionService = None,
+) -> ConversionResponse:
+    json_content: Optional[Any] = None
+    error_message: Optional[str] = None
+
+    if return_json and filename and extraction_service:
+        loop = asyncio.get_running_loop()
+        json_result = await loop.run_in_executor(
+            _thread_pool,
+            extraction_service.extract_structured,
+            filename,
+            result.markdown,
+            mappings,
+            raw_data,
+        )
+        if "error" in json_result:
+            error_message = json_result["error"]
+        else:
+            json_content = json_result
+
+    return ConversionResponse(
+        success=True,
+        time_ms=round(result.duration_ms, 3),
+        content=result.markdown,
+        return_json=return_json,
+        json_content=json_content,
+        metadata=_build_metadata(result),
+        error_message=error_message,
+    )
+
+
+def _raise_for_error(outcome: ConversionError) -> None:
+    status_map = {
+        "FileNotFoundError": status.HTTP_404_NOT_FOUND,
+        "ValueError": status.HTTP_422_UNPROCESSABLE_ENTITY,
+        "PermissionError": status.HTTP_403_FORBIDDEN,
+    }
+    code = status_map.get(outcome.error_type, status.HTTP_500_INTERNAL_SERVER_ERROR)
+    raise HTTPException(
+        status_code=code,
+        detail={
+            "success": False,
+            "error_type": outcome.error_type,
+            "message": outcome.message,
+            "time_ms": round(outcome.duration_ms, 3),
+        },
+    )
+
+
+@router.post(
+    "/convert/file",
+    response_model=ConversionResponse,
+    summary="Convert uploaded file to Markdown",
+)
+async def convert_file(
+    file: Annotated[UploadFile, File(description="File to convert")],
+    plain_text: bool = Form(False),
+    return_json: bool = Form(False),
+    mappings: Optional[str] = Form(None, description="JSON string with field mappings"),
+    token: str = require_auth,
+    converter_service: ConverterService = Depends(get_converter_service),
+    extraction_service: ExtractionService = Depends(get_extraction_service),
+):
+    if file is None:
+        raise HTTPException(status_code=400, detail={"success": False, "message": "No file provided."})
+
+    parsed_mappings = None
+    if mappings:
+        try:
+            parsed_mappings = json_mod.loads(mappings)
+        except json_mod.JSONDecodeError:
+            raise HTTPException(status_code=400, detail={"success": False, "message": "Invalid JSON in mappings parameter."})
+
+    _logger.info("Received request to convert file: %s", file.filename)
+    raw = await file.read()
+    if len(raw) > _MAX_UPLOAD_BYTES:
+        _logger.error("File %s exceeds %s MB limit", file.filename, _settings.max_upload_mb)
+        raise HTTPException(status_code=413, detail={"success": False, "message": f"File exceeds {_settings.max_upload_mb} MB limit."})
+
+    loop = asyncio.get_running_loop()
+    outcome = await loop.run_in_executor(_thread_pool, converter_service.convert_stream, raw, file.filename or "upload")
+    if isinstance(outcome, ConversionError):
+        _logger.error("Conversion failed for %s: %s", file.filename, outcome.message)
+        _raise_for_error(outcome)
+
+    _logger.info("Conversion successful for %s", file.filename)
+    if plain_text:
+        from fastapi.responses import PlainTextResponse
+        return PlainTextResponse(outcome.markdown)
+
+    response = await _build_response(
+        outcome,
+        return_json=return_json,
+        filename=file.filename,
+        raw_data=raw,
+        mappings=parsed_mappings,
+        extraction_service=extraction_service,
+    )
+    _logger.info("Request completed for %s", file.filename)
+    return response
+
+
+@router.post(
+    "/convert/url",
+    response_model=ConversionResponse,
+    summary="Convert a URL to Markdown",
+)
+async def convert_url(
+    body: UrlRequest,
+    token: str = require_auth,
+    converter_service: ConverterService = Depends(get_converter_service),
+    extraction_service: ExtractionService = Depends(get_extraction_service),
+):
+    _logger.info("Received request to convert URL: %s", body.url)
+    parsed = urlparse(body.url)
+    filename = parsed.path.split("/")[-1] or "url_content"
+
+    if body.return_json:
+        try:
+            _logger.info("Fetching URL for JSON extraction: %s", body.url)
+            async with httpx.AsyncClient(timeout=30.0, follow_redirects=True) as client:
+                resp = await client.get(body.url)
+                resp.raise_for_status()
+            raw_data = resp.content
+            if len(raw_data) > _MAX_UPLOAD_BYTES:
+                raise HTTPException(status_code=413, detail={"success": False, "message": f"File exceeds {_settings.max_upload_mb} MB limit."})
+
+            loop = asyncio.get_running_loop()
+            outcome = await loop.run_in_executor(_thread_pool, converter_service.convert_stream, raw_data, filename)
+            if isinstance(outcome, ConversionError):
+                _logger.error("Conversion failed for URL %s: %s", body.url, outcome.message)
+                _raise_for_error(outcome)
+
+            _logger.info("Conversion successful for URL %s", body.url)
+            response = await _build_response(
+                outcome,
+                return_json=body.return_json,
+                filename=filename,
+                raw_data=raw_data,
+                mappings=body.mappings,
+                extraction_service=extraction_service,
+            )
+            _logger.info("Request completed for URL %s", body.url)
+            return response
+        except httpx.HTTPError as exc:
+            _logger.error("Failed to fetch URL %s: %s", body.url, exc)
+            raise HTTPException(status_code=400, detail={"success": False, "message": f"Failed to fetch URL: {exc}"})
+
+    loop = asyncio.get_running_loop()
+    outcome = await loop.run_in_executor(_thread_pool, converter_service.convert_url, body.url)
+    if isinstance(outcome, ConversionError):
+        _logger.error("Conversion failed for URL %s: %s", body.url, outcome.message)
+        _raise_for_error(outcome)
+
+    _logger.info("Conversion successful for URL %s", body.url)
+    response = await _build_response(
+        outcome,
+        return_json=body.return_json,
+        filename=filename,
+        mappings=body.mappings,
+        extraction_service=extraction_service,
+    )
+    _logger.info("Request completed for URL %s", body.url)
+    return response

app/api/v1/router.py

@@ -0,0 +1,10 @@
+from __future__ import annotations
+
+from fastapi import APIRouter
+
+from app.api.v1 import batch, convert, system
+
+api_v1_router = APIRouter()
+api_v1_router.include_router(convert.router, tags=["Convert"])
+api_v1_router.include_router(batch.router, tags=["Batch"])
+api_v1_router.include_router(system.router, tags=["System"])

app/api/v1/system.py

@@ -0,0 +1,110 @@
+from __future__ import annotations
+
+import platform
+import time
+from datetime import datetime, timezone
+
+from fastapi import APIRouter, Depends
+
+from app.api.deps import get_extraction_service, require_auth
+from app.config import get_settings
+from app.core.constants import (
+    ARCHIVE_EXTENSIONS,
+    AUDIO_EXTENSIONS,
+    DOCUMENT_EXTENSIONS,
+    IMAGE_EXTENSIONS,
+    OFFICE_EXTENSIONS,
+    SUPPORTED_EXTENSIONS,
+    TEXT_EXTENSIONS,
+    WEB_EXTENSIONS,
+)
+from app.models.schemas import (
+    HealthResponse,
+    InfoResponse,
+    SpacyLabelsResponse,
+    SupportedFormatsResponse,
+)
+from app.services.extraction_service import ExtractionService
+
+router = APIRouter()
+_settings = get_settings()
+_START_TIME = time.time()
+
+
+@router.get("/health", response_model=HealthResponse, summary="Health check")
+async def health(
+    token: str = require_auth,
+):
+    return HealthResponse(
+        success=True,
+        status="ok",
+        version=_settings.app_version,
+        uptime_seconds=round(time.time() - _START_TIME, 2),
+        timestamp=datetime.now(timezone.utc).isoformat(),
+    )
+
+
+@router.get("/info", response_model=InfoResponse, summary="Server and environment information")
+async def info(
+    token: str = require_auth,
+):
+    return InfoResponse(
+        success=True,
+        app=_settings.app_name,
+        version=_settings.app_version,
+        python_version=platform.python_version(),
+        platform=platform.system(),
+        uptime_seconds=round(time.time() - _START_TIME, 2),
+        max_upload_mb=_settings.max_upload_mb,
+        supported_extensions=len(SUPPORTED_EXTENSIONS),
+        timestamp=datetime.now(timezone.utc).isoformat(),
+    )
+
+
+@router.get("/formats", response_model=SupportedFormatsResponse, summary="List supported file formats")
+async def list_formats(
+    token: str = require_auth,
+):
+    by_category = {
+        "documents": [e for e in SUPPORTED_EXTENSIONS if e in DOCUMENT_EXTENSIONS],
+        "office": [e for e in SUPPORTED_EXTENSIONS if e in OFFICE_EXTENSIONS],
+        "data": [e for e in SUPPORTED_EXTENSIONS if e in {".csv", ".json", ".xml"}],
+        "web": [e for e in SUPPORTED_EXTENSIONS if e in WEB_EXTENSIONS],
+        "text": [e for e in SUPPORTED_EXTENSIONS if e in TEXT_EXTENSIONS],
+        "images": [e for e in SUPPORTED_EXTENSIONS if e in IMAGE_EXTENSIONS],
+        "audio": [e for e in SUPPORTED_EXTENSIONS if e in AUDIO_EXTENSIONS],
+        "archives": [e for e in SUPPORTED_EXTENSIONS if e in ARCHIVE_EXTENSIONS],
+    }
+    return SupportedFormatsResponse(
+        success=True,
+        total_count=len(SUPPORTED_EXTENSIONS),
+        all_extensions=sorted(SUPPORTED_EXTENSIONS),
+        by_category={k: sorted(v) for k, v in by_category.items()},
+    )
+
+
+@router.get("/spacy-labels", response_model=SpacyLabelsResponse, summary="List available spaCy NER labels")
+async def list_spacy_labels(
+    token: str = require_auth,
+    extraction_service: ExtractionService = Depends(get_extraction_service),
+):
+    return SpacyLabelsResponse(
+        success=True,
+        spacy_labels=extraction_service.get_spacy_labels(),
+        source_types={
+            "entity": "Extract using spaCy NER labels (ORG, PERSON, DATE, etc.)",
+            "regex": "Extract using custom regular expressions",
+            "token_attr": "Extract using token attributes (text, pos_, tag_, etc.)",
+        },
+        example_mappings={
+            "company": {"source_type": "entity", "label": "ORG"},
+            "person": {"source_type": "entity", "label": "PERSON"},
+            "date": {"source_type": "entity", "label": "DATE"},
+            "money": {"source_type": "entity", "label": "MONEY"},
+            "email": {
+                "source_type": "regex",
+                "pattern": r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b",
+            },
+            "phone": {"source_type": "regex", "pattern": r"\b\d{3}-\d{3}-\d{4}\b"},
+        },
+    )

app/config.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+from functools import lru_cache
+from typing import Optional
+
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(
+        env_file=".env",
+        env_file_encoding="utf-8",
+        extra="ignore",
+    )
+
+    app_name: str = "reconciliation-non-ai-extractor"
+    app_version: str = "1.0.0"
+    environment: str = "production"
+    host: str = "0.0.0.0"
+    port: int = 7860
+    workers: int = 1
+    log_level: str = "INFO"
+    enable_colors: bool = True
+
+    api_key: str = "changeme"
+    max_upload_bytes: int = 100 * 1024 * 1024
+    max_batch_files: int = 10
+    max_batch_urls: int = 20
+
+    ocr_det_cuda: bool = False
+    ocr_det_dml: bool = False
+    ocr_cls_cuda: bool = False
+    ocr_cls_dml: bool = False
+    ocr_rec_cuda: bool = False
+    ocr_rec_dml: bool = False
+
+    spacy_model: str = "en_core_web_sm"
+
+    @property
+    def max_upload_mb(self) -> int:
+        return self.max_upload_bytes // (1024 * 1024)
+
+
+@lru_cache(maxsize=1)
+def get_settings() -> Settings:
+    return Settings()

app/core/__init__.py

@@ -0,0 +1 @@
+from __future__ import annotations

app/core/banner.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+import pyfiglet
+from rich.console import Console
+from rich.rule import Rule
+from rich.text import Text
+
+from app.config import get_settings
+
+_console = Console()
+_settings = get_settings()
+
+
+def print_banner() -> None:
+    art = pyfiglet.figlet_format("ValidOps", font="slant")
+    _console.print(Text(art, style="bold cyan"))
+    _console.print(f"  [bold white]{'Service:':<14}[/bold white]  [cyan]{_settings.app_name}[/cyan]")
+    _console.print(f"  [bold white]{'Version:':<14}[/bold white]  [cyan]v{_settings.app_version}[/cyan]")
+    _console.print(f"  [bold white]{'Environment:':<14}[/bold white]  [cyan]{_settings.environment}[/cyan]")
+    _console.print(Rule(style="dim cyan"))
+    _console.print()

app/core/constants.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+SUPPORTED_EXTENSIONS = {
+    ".pdf", ".docx", ".doc", ".pptx", ".ppt",
+    ".xlsx", ".xls", ".csv", ".json", ".xml",
+    ".html", ".htm", ".txt", ".md", ".rst",
+    ".jpg", ".jpeg", ".png", ".gif", ".bmp", ".webp", ".tiff",
+    ".mp3", ".wav", ".ogg", ".flac",
+    ".zip", ".epub",
+}
+
+IMAGE_EXTENSIONS = {
+    ".jpg", ".jpeg", ".png", ".gif", ".bmp", ".webp", ".tiff",
+}
+
+IMAGE_MIME_PREFIXES = {"image/"}
+
+TABULAR_EXTENSIONS = {".csv", ".xls", ".xlsx"}
+
+AUDIO_EXTENSIONS = {".mp3", ".wav", ".ogg", ".flac"}
+
+DOCUMENT_EXTENSIONS = {".pdf", ".docx", ".doc", ".epub"}
+
+OFFICE_EXTENSIONS = {".pptx", ".ppt", ".xlsx", ".xls"}
+
+WEB_EXTENSIONS = {".html", ".htm"}
+
+TEXT_EXTENSIONS = {".txt", ".md", ".rst"}
+
+ARCHIVE_EXTENSIONS = {".zip"}
+
+MAX_CSV_ROWS = 100_000
+MAX_EXCEL_ROWS = 50_000
+MAX_MEMORY_CELLS = 2_000_000
+
+OCR_TEXT_SCORE = 0.5
+OCR_DPI = 150

app/core/logger.py

@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+import logging
+import sys
+from pathlib import Path
+from typing import Optional
+
+from rich.console import Console
+from rich.logging import RichHandler
+
+from app.config import get_settings
+
+_settings = get_settings()
+_console = Console()
+
+
+class AppLogger:
+    def __init__(self) -> None:
+        self._loggers: dict[str, logging.Logger] = {}
+
+    def get_logger(self, name: str) -> logging.Logger:
+        if name in self._loggers:
+            return self._loggers[name]
+
+        logger = logging.getLogger(name)
+        logger.setLevel(getattr(logging, _settings.log_level.upper(), logging.INFO))
+        logger.handlers.clear()
+        logger.propagate = False
+
+        rich_handler = RichHandler(
+            console=_console,
+            rich_tracebacks=True,
+            show_path=False,
+            show_time=True,
+        )
+        rich_handler.setFormatter(logging.Formatter("%(message)s"))
+        logger.addHandler(rich_handler)
+
+        file_handler = self._build_file_handler()
+        if file_handler:
+            logger.addHandler(file_handler)
+
+        self._loggers[name] = logger
+        return logger
+
+    def _build_file_handler(self) -> Optional[logging.FileHandler]:
+        log_dir = Path("logs")
+        log_dir.mkdir(parents=True, exist_ok=True)
+        file_handler = logging.FileHandler(
+            log_dir / f"{_settings.app_name}.log",
+            encoding="utf-8",
+        )
+        file_handler.setFormatter(
+            logging.Formatter(
+                fmt="%(asctime)s | %(levelname)-8s | %(name)s | %(message)s",
+                datefmt="%Y-%m-%d %H:%M:%S",
+            )
+        )
+        return file_handler
+
+
+_logger_factory = AppLogger()
+
+
+def get_logger(name: str) -> logging.Logger:
+    return _logger_factory.get_logger(name)

app/core/security.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+from fastapi import HTTPException, Security, status
+from fastapi.security import APIKeyHeader
+
+from app.config import get_settings
+from app.core.logger import get_logger
+
+_logger = get_logger(__name__)
+_security = APIKeyHeader(name="x-api-key", auto_error=False)
+
+
+async def require_api_key(
+    api_key: str = Security(_security),
+) -> str:
+    settings = get_settings()
+    token = settings.API_KEY
+
+    if not api_key:
+        _logger.warning("Missing x-api-key header")
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Missing x-api-key header",
+        )
+
+    if api_key != token:
+        _logger.warning("Invalid API key provided")
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Invalid API key",
+        )
+
+    return api_key

app/models/__init__.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from app.models.domain import ConversionError, ConversionResult
+from app.models.schemas import (
+    BatchFileResult,
+    BatchResponse,
+    BatchUrlRequest,
+    ConversionMetadata,
+    ConversionResponse,
+    HealthResponse,
+    InfoResponse,
+    SpacyLabelsResponse,
+    SupportedFormatsResponse,
+    UrlRequest,
+)
+
+__all__ = [
+    "ConversionError",
+    "ConversionResult",
+    "ConversionMetadata",
+    "ConversionResponse",
+    "UrlRequest",
+    "BatchUrlRequest",
+    "BatchFileResult",
+    "BatchResponse",
+    "HealthResponse",
+    "InfoResponse",
+    "SupportedFormatsResponse",
+    "SpacyLabelsResponse",
+]

app/models/domain.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass, field
+from typing import Any, Dict, Optional
+
+
+@dataclass(frozen=True)
+class ConversionResult:
+    source: str
+    markdown: str
+    char_count: int
+    word_count: int
+    line_count: int
+    duration_ms: float
+    file_size_bytes: int
+    mime_type: str
+    content_hash: str
+    metadata: dict = field(default_factory=dict)
+
+    @property
+    def token_estimate(self) -> int:
+        return max(1, self.word_count * 4 // 3)
+
+
+@dataclass(frozen=True)
+class ConversionError:
+    source: str
+    error_type: str
+    message: str
+    duration_ms: float

app/models/schemas.py

@@ -0,0 +1,106 @@
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel, Field, field_validator
+
+
+class ConversionMetadata(BaseModel):
+    source: str
+    char_count: int
+    word_count: int
+    line_count: int
+    file_size_bytes: int
+    mime_type: str
+    content_hash: str
+    token_estimate: int
+
+
+class ConversionResponse(BaseModel):
+    success: bool
+    time_ms: float
+    content: str
+    return_json: bool = False
+    json_content: Optional[Any] = None
+    metadata: Optional[ConversionMetadata] = None
+    error_message: Optional[str] = None
+
+
+class UrlRequest(BaseModel):
+    url: str
+    return_json: bool = False
+    mappings: Optional[Dict[str, Dict[str, Any]]] = None
+
+    model_config = {"populate_by_name": True}
+
+    @field_validator("url")
+    @classmethod
+    def validate_scheme(cls, v: str) -> str:
+        if not v.startswith(("http://", "https://")):
+            raise ValueError("Only http/https URLs are supported.")
+        return v
+
+
+class BatchUrlRequest(BaseModel):
+    urls: List[str]
+
+    @field_validator("urls")
+    @classmethod
+    def validate_urls(cls, v: List[str]) -> List[str]:
+        for url in v:
+            if not url.startswith(("http://", "https://")):
+                raise ValueError(f"Invalid URL scheme: {url}")
+        if len(v) > 20:
+            raise ValueError("Maximum 20 URLs per batch request.")
+        return v
+
+
+class BatchFileResult(BaseModel):
+    filename: str
+    success: bool
+    time_ms: float
+    content: Optional[str] = None
+    error: Optional[str] = None
+    metadata: Optional[ConversionMetadata] = None
+
+
+class BatchResponse(BaseModel):
+    total: int
+    succeeded: int
+    failed: int
+    total_time_ms: float
+    results: List[BatchFileResult]
+
+
+class HealthResponse(BaseModel):
+    success: bool
+    status: str
+    version: str
+    uptime_seconds: float
+    timestamp: str
+
+
+class InfoResponse(BaseModel):
+    success: bool
+    app: str
+    version: str
+    python_version: str
+    platform: str
+    uptime_seconds: float
+    max_upload_mb: int
+    supported_extensions: int
+    timestamp: str
+
+
+class SupportedFormatsResponse(BaseModel):
+    success: bool
+    total_count: int
+    all_extensions: List[str]
+    by_category: Dict[str, List[str]]
+
+
+class SpacyLabelsResponse(BaseModel):
+    success: bool
+    spacy_labels: Dict[str, str]
+    source_types: Dict[str, str]
+    example_mappings: Dict[str, Dict[str, Any]]

app/services/__init__.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+from app.services.auth_service import AuthService
+from app.services.converter_service import ConverterService
+from app.services.extraction_service import ExtractionService
+from app.services.ocr_service import OCRService
+
+__all__ = [
+    "AuthService",
+    "ConverterService",
+    "ExtractionService",
+    "OCRService",
+]

app/services/auth_service.py

@@ -0,0 +1,11 @@
+from __future__ import annotations
+
+from app.config import get_settings
+
+
+class AuthService:
+    def __init__(self) -> None:
+        self._settings = get_settings()
+
+    def validate_token(self, token: str) -> bool:
+        return token == self._settings.api_bearer_token

app/services/converter_service.py

@@ -0,0 +1,145 @@
+from __future__ import annotations
+
+import hashlib
+import io
+import mimetypes
+import time
+from pathlib import Path
+from typing import Optional
+from urllib.parse import urlparse
+
+from markitdown import MarkItDown
+
+from app.config import get_settings
+from app.core.constants import IMAGE_EXTENSIONS, IMAGE_MIME_PREFIXES
+from app.core.logger import get_logger
+from app.models.domain import ConversionError, ConversionResult
+from app.services.ocr_service import ocr_image, ocr_pdf
+
+_logger = get_logger(__name__)
+_settings = get_settings()
+
+
+def _is_image(ext: str, mime: str) -> bool:
+    return ext.lower() in IMAGE_EXTENSIONS or any(mime.startswith(p) for p in IMAGE_MIME_PREFIXES)
+
+
+def _build_result(
+    source: str,
+    markdown: str,
+    file_size: int,
+    mime_type: str,
+    elapsed: float,
+) -> ConversionResult:
+    lines = markdown.splitlines()
+    words = markdown.split()
+    content_hash = hashlib.sha256(markdown.encode()).hexdigest()
+    return ConversionResult(
+        source=source,
+        markdown=markdown,
+        char_count=len(markdown),
+        word_count=len(words),
+        line_count=len(lines),
+        duration_ms=elapsed,
+        file_size_bytes=file_size,
+        mime_type=mime_type,
+        content_hash=content_hash,
+    )
+
+
+class ConverterService:
+    def __init__(self, enable_plugins: bool = False) -> None:
+        kwargs: dict = {"enable_plugins": enable_plugins}
+        self._engine = MarkItDown(**kwargs)
+
+    def convert_file(self, path: str | Path) -> ConversionResult | ConversionError:
+        path = Path(path).resolve()
+        start = time.perf_counter()
+
+        if not path.exists():
+            return ConversionError(
+                source=str(path),
+                error_type="FileNotFoundError",
+                message=f"File does not exist: {path}",
+                duration_ms=0.0,
+            )
+
+        file_size = path.stat().st_size
+        mime_type, _ = mimetypes.guess_type(str(path))
+        mime_type = mime_type or "application/octet-stream"
+
+        try:
+            if _is_image(path.suffix, mime_type):
+                markdown = ocr_image(str(path))
+            else:
+                markdown = self._engine.convert(str(path)).text_content
+                if not markdown.strip() and path.suffix.lower() == ".pdf":
+                    _logger.info("No text from PDF, falling back to OCR")
+                    markdown = ocr_pdf(str(path))
+            elapsed = (time.perf_counter() - start) * 1000
+            return _build_result(str(path), markdown, file_size, mime_type, elapsed)
+        except Exception as exc:
+            elapsed = (time.perf_counter() - start) * 1000
+            return ConversionError(
+                source=str(path),
+                error_type=type(exc).__name__,
+                message=str(exc),
+                duration_ms=elapsed,
+            )
+
+    def convert_url(self, url: str) -> ConversionResult | ConversionError:
+        parsed = urlparse(url)
+        if parsed.scheme not in {"http", "https"}:
+            return ConversionError(
+                source=url,
+                error_type="ValueError",
+                message=f"Unsupported URL scheme: {parsed.scheme!r}",
+                duration_ms=0.0,
+            )
+
+        start = time.perf_counter()
+        try:
+            url_ext = Path(urlparse(url).path).suffix.lower()
+            if url_ext in IMAGE_EXTENSIONS:
+                markdown = ocr_image(url)
+                mime_type = mimetypes.guess_type(url)[0] or "image/jpeg"
+            else:
+                result = self._engine.convert(url)
+                markdown = result.text_content
+                mime_type = "text/html"
+            elapsed = (time.perf_counter() - start) * 1000
+            return _build_result(url, markdown, 0, mime_type, elapsed)
+        except Exception as exc:
+            elapsed = (time.perf_counter() - start) * 1000
+            return ConversionError(
+                source=url,
+                error_type=type(exc).__name__,
+                message=str(exc),
+                duration_ms=elapsed,
+            )
+
+    def convert_stream(self, data: bytes, filename: str) -> ConversionResult | ConversionError:
+        start = time.perf_counter()
+        mime_type, _ = mimetypes.guess_type(filename)
+        mime_type = mime_type or "application/octet-stream"
+        ext = Path(filename).suffix.lower()
+
+        try:
+            if _is_image(ext, mime_type):
+                markdown = ocr_image(data)
+            else:
+                result = self._engine.convert_stream(io.BytesIO(data), file_extension=ext)
+                markdown = result.text_content
+                if not markdown.strip() and ext == ".pdf":
+                    _logger.info("No text from PDF stream, falling back to OCR")
+                    markdown = ocr_pdf(data)
+            elapsed = (time.perf_counter() - start) * 1000
+            return _build_result(filename, markdown, len(data), mime_type, elapsed)
+        except Exception as exc:
+            elapsed = (time.perf_counter() - start) * 1000
+            return ConversionError(
+                source=filename,
+                error_type=type(exc).__name__,
+                message=str(exc),
+                duration_ms=elapsed,
+            )

app/services/extraction_service.py

@@ -0,0 +1,537 @@
+from __future__ import annotations
+
+import io
+import re
+import threading
+import warnings
+from pathlib import Path
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union
+
+import pandas as pd
+
+from app.config import get_settings
+from app.core.constants import (
+    MAX_CSV_ROWS,
+    MAX_EXCEL_ROWS,
+    MAX_MEMORY_CELLS,
+    TABULAR_EXTENSIONS,
+)
+from app.core.logger import get_logger
+
+_logger = get_logger(__name__)
+_settings = get_settings()
+
+VALID_SPACY_LABELS: Dict[str, str] = {
+    "ORG": "Companies, agencies, institutions",
+    "PERSON": "People, including fictional",
+    "DATE": "Absolute or relative dates or periods",
+    "MONEY": "Monetary values, including unit",
+    "GPE": "Countries, cities, states",
+    "LOC": "Non-GPE locations, mountain ranges, bodies of water",
+    "PRODUCT": "Objects, vehicles, foods, etc.",
+    "EVENT": "Named hurricanes, battles, wars, sports events",
+    "CARDINAL": "Numerals that do not fall under another type",
+    "PERCENT": "Percentage, including '%'",
+    "QUANTITY": "Measurements, as of weight or distance",
+    "TIME": "Times smaller than a day",
+    "NORP": "Nationalities or religious or political groups",
+    "FAC": "Buildings, airports, highways, bridges",
+    "WORK_OF_ART": "Titles of books, songs, etc.",
+    "LAW": "Named documents made into laws",
+    "LANGUAGE": "Any named language",
+    "ORDINAL": "'first', 'second', etc.",
+}
+
+SchemaNode = Dict[str, Any]
+ResultNode = Union[str, List[Any], Dict[str, Any], None]
+
+_norm_lock = threading.Lock()
+_NORMALIZERS: Dict[str, Callable[[str], str]] = {
+    "strip": lambda s: s.strip(),
+    "upper": lambda s: s.upper(),
+    "lower": lambda s: s.lower(),
+    "remove_commas": lambda s: s.replace(",", ""),
+    "remove_spaces": lambda s: s.replace(" ", ""),
+    "remove_newlines": lambda s: s.replace("\n", " ").replace("\r", ""),
+    "collapse_whitespace": lambda s: re.sub(r"\s+", " ", s).strip(),
+    "remove_currency": lambda s: re.sub(r"[$\u20ac\u00a3\u00a5\u20b9]", "", s),
+    "remove_non_numeric": lambda s: re.sub(r"[^\d.]", "", s),
+    "normalize_date_sep": lambda s: re.sub(r"[/.]", "-", s),
+}
+
+_resolver_lock = threading.Lock()
+_RESOLVERS: Dict[str, Callable[[Dict[str, Any], Any, str], Optional[str]]] = {}
+
+_nlp_lock = threading.Lock()
+_nlp = None
+
+
+def _get_nlp():
+    global _nlp
+    if _nlp is not None:
+        return _nlp
+    with _nlp_lock:
+        if _nlp is None:
+            import spacy
+            _nlp = spacy.load(
+                _settings.spacy_model,
+                exclude=["tagger", "parser", "lemmatizer", "attribute_ruler"],
+            )
+            _logger.info("spaCy %s loaded", _settings.spacy_model)
+    return _nlp
+
+
+def _validate_file_size(size: int) -> Optional[str]:
+    max_size = _settings.max_upload_bytes
+    if size > max_size:
+        return f"File size {size} bytes exceeds limit of {max_size} bytes"
+    return None
+
+
+def _check_memory_usage(rows: int, cols: int) -> Optional[str]:
+    approx_mb = (rows * cols * 50) / (1024 * 1024)
+    if rows * cols > MAX_MEMORY_CELLS:
+        return f"Data size too large (approx {approx_mb:.1f} MB). Too many cells: {rows}x{cols}"
+    return None
+
+
+def _build_flags(rule: Dict[str, Any]) -> re.RegexFlag:
+    flags = re.RegexFlag(0)
+    for name in rule.get("flags", []):
+        obj = getattr(re, name.upper(), None)
+        if obj is None:
+            _logger.warning("Unknown re flag %s", name)
+            continue
+        flags |= obj
+    return flags
+
+
+def _apply_normalizers(value: Optional[str], normalize: Any) -> Optional[str]:
+    if not isinstance(value, str):
+        return None
+    if not normalize:
+        return value
+    if isinstance(normalize, str):
+        normalize = [normalize]
+    for key in normalize:
+        with _norm_lock:
+            fn = _NORMALIZERS.get(key)
+        if fn is None:
+            _logger.warning("Unknown normalizer %s", key)
+            continue
+        try:
+            value = fn(value)
+        except Exception as exc:
+            _logger.error("Normalizer %s raised on value %s: %s", key, value, exc)
+    return value if value else None
+
+
+def _try_group(match: re.Match, capture_group: Any) -> Tuple[bool, Optional[str]]:
+    try:
+        return True, match.group(capture_group)
+    except (IndexError, re.error):
+        _logger.warning("Group %s does not exist in pattern", capture_group)
+        return False, None
+
+
+def _resolve_regex(rule: Dict[str, Any], text: str) -> Optional[str]:
+    primary = rule.get("pattern", "")
+    if not primary:
+        _logger.warning("Regex rule missing pattern")
+        return None
+    flags = _build_flags(rule)
+    capture_group = rule.get("capture_group", 0)
+    match_index = rule.get("match_index", 0)
+    normalize = rule.get("normalize", "")
+    strip_chars = rule.get("strip_chars", "")
+    fallbacks = rule.get("fallback_patterns", [])
+
+    for pat in [primary, *fallbacks]:
+        try:
+            matches = list(re.finditer(pat, text, flags))
+        except re.error as exc:
+            _logger.error("Invalid regex %s: %s", pat, exc)
+            continue
+        if not matches:
+            continue
+        try:
+            target_matches = [matches[match_index]]
+        except IndexError:
+            target_matches = [matches[-1]]
+        for m in target_matches:
+            exists, result = _try_group(m, capture_group)
+            if not exists or result is None:
+                break
+            result = _apply_normalizers(result, normalize)
+            if result is None:
+                break
+            result = result.strip(strip_chars) if strip_chars else result.strip()
+            return result or None
+    return None
+
+
+def _resolve_regex_all(rule: Dict[str, Any], text: str) -> List[Optional[str]]:
+    primary = rule.get("pattern", "")
+    if not primary:
+        _logger.warning("Regex-array rule missing pattern")
+        return []
+    flags = _build_flags(rule)
+    capture_group = rule.get("capture_group", 0)
+    normalize = rule.get("normalize", "")
+    strip_chars = rule.get("strip_chars", "")
+    max_items = rule.get("max_items")
+
+    try:
+        matches = list(re.finditer(primary, text, flags))
+    except re.error as exc:
+        _logger.error("Invalid regex %s: %s", primary, exc)
+        return []
+
+    results: List[Optional[str]] = []
+    for m in matches:
+        exists, result = _try_group(m, capture_group)
+        if not exists or result is None:
+            continue
+        result = _apply_normalizers(result, normalize)
+        if result is None:
+            continue
+        result = result.strip(strip_chars) if strip_chars else result.strip()
+        if result:
+            results.append(result)
+        if max_items is not None and len(results) >= max_items:
+            break
+    return results
+
+
+def _resolve_entity(rule: Dict[str, Any], doc: Any) -> Optional[str]:
+    if doc is None:
+        _logger.warning("Entity resolver received None doc")
+        return None
+    labels = rule.get("label")
+    if isinstance(labels, str):
+        labels = [labels]
+    labels = set(labels or [])
+    match_index = rule.get("match_index", 0)
+    min_length = rule.get("min_length", 1)
+    exclude_pat = rule.get("exclude_pattern", "")
+    exclude_flags = _build_flags({"flags": rule.get("exclude_flags", [])})
+    normalize = rule.get("normalize", "")
+
+    candidates = [
+        ent.text for ent in doc.ents
+        if ent.label_ in labels
+        and len(ent.text) >= min_length
+        and not (exclude_pat and re.search(exclude_pat, ent.text, exclude_flags))
+    ]
+    if not candidates:
+        return None
+    try:
+        result = candidates[match_index]
+    except IndexError:
+        result = candidates[-1]
+    return _apply_normalizers(result, normalize)
+
+
+def _resolve_entity_all(rule: Dict[str, Any], doc: Any) -> List[Optional[str]]:
+    if doc is None:
+        _logger.warning("Entity-array resolver received None doc")
+        return []
+    labels = rule.get("label")
+    if isinstance(labels, str):
+        labels = [labels]
+    labels = set(labels or [])
+    min_length = rule.get("min_length", 1)
+    exclude_pat = rule.get("exclude_pattern", "")
+    exclude_flags = _build_flags({"flags": rule.get("exclude_flags", [])})
+    normalize = rule.get("normalize", "")
+    max_items = rule.get("max_items")
+    unique = rule.get("unique", False)
+
+    results: List[str] = []
+    seen: set = set()
+    for ent in doc.ents:
+        if ent.label_ not in labels:
+            continue
+        if len(ent.text) < min_length:
+            continue
+        if exclude_pat and re.search(exclude_pat, ent.text, exclude_flags):
+            continue
+        value = _apply_normalizers(ent.text, normalize)
+        if not value:
+            continue
+        if unique:
+            if value in seen:
+                continue
+            seen.add(value)
+        results.append(value)
+        if max_items is not None and len(results) >= max_items:
+            break
+    return results
+
+
+def _resolve_token_attr(rule: Dict[str, Any], doc: Any) -> Optional[str]:
+    if doc is None:
+        _logger.warning("Token-attr resolver received None doc")
+        return None
+    attr = rule.get("attr", "")
+    match_index = rule.get("match_index", 0)
+    normalize = rule.get("normalize", "")
+
+    candidates = [t.text for t in doc if getattr(t, attr, False)]
+    if not candidates:
+        return None
+    try:
+        result = candidates[match_index]
+    except IndexError:
+        result = candidates[-1]
+    return _apply_normalizers(result, normalize)
+
+
+def _register_builtin_resolvers() -> None:
+    with _resolver_lock:
+        _RESOLVERS["regex"] = lambda rule, doc, text: _resolve_regex(rule, text)
+        _RESOLVERS["entity"] = lambda rule, doc, text: _resolve_entity(rule, doc)
+        _RESOLVERS["token_attr"] = lambda rule, doc, text: _resolve_token_attr(rule, doc)
+        _RESOLVERS["regex_all"] = lambda rule, doc, text: _resolve_regex_all(rule, text)
+        _RESOLVERS["entity_all"] = lambda rule, doc, text: _resolve_entity_all(rule, doc)
+
+
+_register_builtin_resolvers()
+
+
+def _resolve_scalar_field(rule: Dict[str, Any], doc: Any, text: str) -> Optional[str]:
+    src = rule.get("source_type")
+    with _resolver_lock:
+        fn = _RESOLVERS.get(src)
+    if fn is None:
+        _logger.warning("Unknown source_type %s", src)
+        return None
+    return fn(rule, doc, text)
+
+
+def _resolve_node(node: SchemaNode, doc: Any, text: str) -> ResultNode:
+    node_type = node.get("type")
+    if node_type == "object":
+        return _resolve_object_node(node, doc, text)
+    if node_type == "array":
+        return _resolve_array_node(node, doc, text)
+    return _resolve_scalar_field(node, doc, text)
+
+
+def _resolve_object_node(node: SchemaNode, doc: Any, text: str) -> Dict[str, ResultNode]:
+    fields: Dict[str, SchemaNode] = node.get("fields", {})
+    result: Dict[str, ResultNode] = {}
+    for field_name, child_node in fields.items():
+        try:
+            result[field_name] = _resolve_node(child_node, doc, text)
+        except Exception as exc:
+            _logger.error("Object field %s raised: %s", field_name, exc)
+            result[field_name] = None
+    return result
+
+
+def _resolve_array_node(node: SchemaNode, doc: Any, text: str) -> List[ResultNode]:
+    item_schema: SchemaNode = node.get("items", {})
+    split_pat: Optional[str] = node.get("split_pattern")
+    split_flags_rule = {"flags": node.get("split_flags", [])}
+    max_items: Optional[int] = node.get("max_items")
+    results: List[ResultNode] = []
+
+    if split_pat:
+        try:
+            flags = _build_flags(split_flags_rule)
+            segments = re.split(split_pat, text, flags=flags)
+        except re.error as exc:
+            _logger.error("Invalid split_pattern %s: %s", split_pat, exc)
+            return []
+        nlp = _get_nlp()
+        try:
+            segment_docs = list(nlp.pipe(segments))
+        except Exception as exc:
+            _logger.error("spaCy pipe failed on array segments: %s", exc)
+            segment_docs = [None] * len(segments)
+        for seg_doc, seg_text in zip(segment_docs, segments):
+            if not seg_text.strip():
+                continue
+            try:
+                item_result = _resolve_node(item_schema, seg_doc, seg_text)
+            except Exception as exc:
+                _logger.error("Array item resolve raised: %s", exc)
+                item_result = None
+            results.append(item_result)
+            if max_items is not None and len(results) >= max_items:
+                break
+    else:
+        try:
+            raw = _resolve_node(item_schema, doc, text)
+        except Exception as exc:
+            _logger.error("Array item resolve raised: %s", exc)
+            return []
+        if isinstance(raw, list):
+            results = raw
+        elif raw is not None:
+            results = [raw]
+        if max_items is not None:
+            results = results[:max_items]
+    return results
+
+
+def _safe_resolve_node(path: str, node: SchemaNode, doc: Any, text: str) -> ResultNode:
+    try:
+        return _resolve_node(node, doc, text)
+    except Exception as exc:
+        _logger.error("Schema path %s raised: %s", path, exc)
+        return None
+
+
+def _extract_spacy_fields(text: str, fields: Dict[str, SchemaNode]) -> Dict[str, ResultNode]:
+    nlp = _get_nlp()
+    try:
+        doc = next(iter(nlp.pipe([text])))
+    except Exception as exc:
+        _logger.error("spaCy pipe failed: %s", exc)
+        doc = None
+    return {
+        field: _safe_resolve_node(field, node, doc, text)
+        for field, node in fields.items()
+    }
+
+
+def _extract_tabular(file_path: Union[str, Path], file_data: Optional[bytes] = None) -> Dict[str, Any]:
+    ext = Path(file_path).suffix.lower()
+
+    if file_data is not None:
+        size_error = _validate_file_size(len(file_data))
+        if size_error:
+            return {"error": size_error, "file_type": ext}
+    elif Path(file_path).exists():
+        size_error = _validate_file_size(Path(file_path).stat().st_size)
+        if size_error:
+            return {"error": size_error, "file_type": ext}
+
+    try:
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", UserWarning)
+            if ext == ".csv":
+                if file_data:
+                    df = pd.read_csv(io.BytesIO(file_data), nrows=MAX_CSV_ROWS + 1, low_memory=False)
+                else:
+                    df = pd.read_csv(file_path, nrows=MAX_CSV_ROWS + 1, low_memory=False)
+            else:
+                if file_data:
+                    df = pd.read_excel(
+                        io.BytesIO(file_data),
+                        engine="openpyxl" if ext == ".xlsx" else "xlrd",
+                    )
+                else:
+                    df = pd.read_excel(
+                        file_path,
+                        engine="openpyxl" if ext == ".xlsx" else "xlrd",
+                    )
+
+        max_rows = MAX_EXCEL_ROWS if ext != ".csv" else MAX_CSV_ROWS
+        if len(df) > max_rows:
+            return {
+                "error": f"File contains {len(df)} rows, exceeds limit of {max_rows}",
+                "file_type": ext,
+                "row_count": len(df),
+            }
+
+        mem_error = _check_memory_usage(len(df), len(df.columns))
+        if mem_error:
+            return {"error": mem_error, "file_type": ext}
+
+        result = {
+            "success": True,
+            "file_type": ext,
+            "data": {
+                "columns": list(df.columns),
+                "rows": df.where(pd.notnull(df), None).to_dict(orient="records"),
+                "shape": [len(df), len(df.columns)],
+                "dtypes": {col: str(dtype) for col, dtype in df.dtypes.items()},
+            },
+        }
+        _logger.info("Extracted JSON from %s: %d rows, %d cols", ext, len(df), len(df.columns))
+        return result
+
+    except pd.errors.EmptyDataError:
+        return {"error": "File is empty or has no data", "file_type": ext}
+    except MemoryError:
+        return {"error": "Out of memory processing file", "file_type": ext}
+    except Exception as exc:
+        _logger.exception("JSON extraction failed for %s", ext)
+        return {
+            "error": f"Processing failed: {exc}",
+            "file_type": ext,
+            "exception_type": type(exc).__name__,
+        }
+
+
+class ExtractionService:
+    def __init__(self) -> None:
+        self._spacy_labels = VALID_SPACY_LABELS
+
+    def extract_structured(
+        self,
+        filename: Union[str, Path],
+        markdown_text: str,
+        mappings: Optional[Dict[str, Dict[str, Any]]] = None,
+        file_data: Optional[bytes] = None,
+    ) -> Dict[str, Any]:
+        ext = Path(filename).suffix.lower()
+
+        if ext in TABULAR_EXTENSIONS:
+            result = _extract_tabular(filename, file_data)
+            if "error" not in result:
+                result["extractor"] = "pandas"
+            return result
+
+        if not mappings:
+            return {
+                "error": (
+                    f"Cannot extract JSON from '{ext}' files without field mappings. "
+                    "Provide a 'mappings' object with field extraction rules."
+                ),
+                "file_type": ext,
+            }
+
+        valid, mapper_error = self.validate_mappings(mappings)
+        if not valid:
+            return {
+                "error": "invalid_spacy_labels",
+                "label_mapper": mapper_error,
+                "file_type": ext,
+            }
+
+        try:
+            data = _extract_spacy_fields(markdown_text, mappings)
+            _logger.info("spaCy extraction completed for %s: %d fields", ext, len(data))
+            return {
+                "success": True,
+                "extractor": "spacy",
+                "file_type": ext,
+                "data": data,
+            }
+        except Exception as exc:
+            _logger.exception("spaCy extraction failed for %s", ext)
+            return {
+                "error": f"spaCy extraction failed: {exc}",
+                "file_type": ext,
+                "exception_type": type(exc).__name__,
+            }
+
+    def validate_mappings(self, mappings: Dict[str, Dict[str, Any]]) -> Tuple[bool, Optional[str]]:
+        for key, rule in mappings.items():
+            source_type = rule.get("source_type")
+            if source_type == "entity":
+                label = rule.get("label")
+                if isinstance(label, str) and label not in self._spacy_labels:
+                    return False, f"Invalid spaCy label '{label}' in field '{key}'"
+                if isinstance(label, list):
+                    for lbl in label:
+                        if lbl not in self._spacy_labels:
+                            return False, f"Invalid spaCy label '{lbl}' in field '{key}'"
+        return True, None
+
+    def get_spacy_labels(self) -> Dict[str, str]:
+        return dict(self._spacy_labels)

app/services/ocr_service.py

@@ -0,0 +1,127 @@
+from __future__ import annotations
+
+import io
+import threading
+from typing import Union
+from urllib.parse import urlparse
+
+import numpy as np
+from PIL import Image
+
+from app.config import get_settings
+from app.core.logger import get_logger
+
+_logger = get_logger(__name__)
+_settings = get_settings()
+
+_lock = threading.Lock()
+_engine = None
+
+
+def _get_engine():
+    global _engine
+    if _engine is None:
+        with _lock:
+            if _engine is None:
+                from rapidocr_onnxruntime import RapidOCR
+                _engine = RapidOCR(
+                    Det={"use_cuda": _settings.ocr_det_cuda, "use_dml": _settings.ocr_det_dml},
+                    Cls={"use_cuda": _settings.ocr_cls_cuda, "use_dml": _settings.ocr_cls_dml},
+                    Rec={"use_cuda": _settings.ocr_rec_cuda, "use_dml": _settings.ocr_rec_dml},
+                    print_verbose=False,
+                )
+    return _engine
+
+
+def _to_numpy(source) -> Union[np.ndarray, str]:
+    if isinstance(source, Image.Image):
+        img = source
+        if img.mode not in ("RGB", "L", "RGBA"):
+            img = img.convert("RGB")
+        return np.array(img)
+
+    if isinstance(source, (bytes, bytearray)):
+        img = Image.open(io.BytesIO(source))
+        if img.mode not in ("RGB", "L", "RGBA"):
+            img = img.convert("RGB")
+        return np.array(img)
+
+    if isinstance(source, str):
+        parsed = urlparse(source)
+        if parsed.scheme in {"http", "https"}:
+            import httpx
+            resp = httpx.get(source, follow_redirects=True, timeout=30)
+            resp.raise_for_status()
+            img = Image.open(io.BytesIO(resp.content))
+            if img.mode not in ("RGB", "L", "RGBA"):
+                img = img.convert("RGB")
+            return np.array(img)
+        return source
+
+    if isinstance(source, np.ndarray):
+        return source
+
+    raise TypeError(
+        f"ocr_image expects bytes, str, numpy.ndarray or PIL.Image; got {type(source).__name__}"
+    )
+
+
+def ocr_image(
+    source,
+    *,
+    use_det: bool = True,
+    use_cls: bool = True,
+    use_rec: bool = True,
+    text_score: float = 0.5,
+) -> str:
+    engine = _get_engine()
+    img = _to_numpy(source)
+    result, _ = engine(
+        img,
+        use_det=use_det,
+        use_cls=use_cls,
+        use_rec=use_rec,
+        text_score=text_score,
+    )
+    if not result:
+        return ""
+    lines = [item[1] for item in result if len(item) > 1 and item[1]]
+    return "\n".join(lines)
+
+
+def ocr_pdf(source: Union[str, bytes], *, dpi: int = 150) -> str:
+    try:
+        import pypdfium2 as pdfium
+    except ImportError:
+        _logger.error("pypdfium2 not installed")
+        return ""
+
+    try:
+        pdf = pdfium.PdfDocument(source)
+        scale = dpi / 72.0
+        page_texts: list[str] = []
+
+        for page_index in range(len(pdf)):
+            page = pdf[page_index]
+            bitmap = page.render(scale=scale, rotation=0)
+            pil_image = bitmap.to_pil()
+            page_text = ocr_image(pil_image)
+            if page_text:
+                page_texts.append(page_text)
+
+        pdf.close()
+        return "\n\n".join(page_texts)
+    except Exception as exc:
+        _logger.error("Failed to OCR PDF: %s", exc)
+        return ""
+
+
+class OCRService:
+    def __init__(self) -> None:
+        self._engine = _get_engine()
+
+    def image_to_text(self, source, text_score: float = 0.5) -> str:
+        return ocr_image(source, text_score=text_score)
+
+    def pdf_to_text(self, source: Union[str, bytes], dpi: int = 150) -> str:
+        return ocr_pdf(source, dpi=dpi)

main.py

@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+import uvicorn
+
+from app.core.logger import get_logger
+
+logger = get_logger(__name__)
+
+if __name__ == "__main__":
+    logger.info("Starting %s server", "reconciliation-non-ai-extractor")
+    uvicorn.run(
+        "app.api.server:app",
+        host="0.0.0.0",
+        port=7860,
+        reload=False,
+        workers=1,
+        log_level="info",
+        access_log=True,
+    )

postman_collection.json

@@ -0,0 +1,321 @@
+{
+    "info": {
+        "_postman_id": "reconciliation-not-ai-extractor-service",
+        "name": "Reconciliation-not-ai-extractor-service",
+        "description": "Production-ready Postman collection for the Document-to-Markdown and structured data extraction API powered by Microsoft MarkItDown, RapidOCR, and spaCy.\n\n**Base URL:** `{{base_url}}`\n**Authentication:** Bearer token via `{{api_bearer_token}}`\n\n## Endpoints\n- **System:** Health, info, supported formats, spaCy labels\n- **Convert:** Single file and single URL to Markdown\n- **Batch:** Bulk conversion of files and URLs",
+        "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
+    },
+    "item": [
+        {
+            "name": "System",
+            "description": "Health checks, server metadata, supported formats, and spaCy NER labels.",
+            "item": [
+                {
+                    "name": "Root Health (No Auth)",
+                    "request": {
+                        "method": "GET",
+                        "header": [],
+                        "url": {
+                            "raw": "{{base_url}}/health",
+                            "host": ["{{base_url}}"],
+                            "path": ["health"]
+                        },
+                        "description": "Quick health check without authentication. Returns `{\"status\": \"ok\", \"version\": \"1.0.0\"}`."
+                    },
+                    "response": []
+                },
+                {
+                    "name": "Health",
+                    "request": {
+                        "method": "GET",
+                        "header": [
+                            {
+                                "key": "Authorization",
+                                "value": "Bearer {{api_bearer_token}}",
+                                "type": "text"
+                            }
+                        ],
+                        "url": {
+                            "raw": "{{base_url}}/v1/health",
+                            "host": ["{{base_url}}"],
+                            "path": ["v1", "health"]
+                        },
+                        "description": "Authenticated health check with uptime and timestamp."
+                    },
+                    "response": []
+                },
+                {
+                    "name": "Info",
+                    "request": {
+                        "method": "GET",
+                        "header": [
+                            {
+                                "key": "Authorization",
+                                "value": "Bearer {{api_bearer_token}}",
+                                "type": "text"
+                            }
+                        ],
+                        "url": {
+                            "raw": "{{base_url}}/v1/info",
+                            "host": ["{{base_url}}"],
+                            "path": ["v1", "info"]
+                        },
+                        "description": "Server and environment information including app name, version, Python version, platform, max upload size, and supported extension count."
+                    },
+                    "response": []
+                },
+                {
+                    "name": "Supported Formats",
+                    "request": {
+                        "method": "GET",
+                        "header": [
+                            {
+                                "key": "Authorization",
+                                "value": "Bearer {{api_bearer_token}}",
+                                "type": "text"
+                            }
+                        ],
+                        "url": {
+                            "raw": "{{base_url}}/v1/formats",
+                            "host": ["{{base_url}}"],
+                            "path": ["v1", "formats"]
+                        },
+                        "description": "List all supported file formats grouped by category: documents, office, data, web, text, images, audio, archives."
+                    },
+                    "response": []
+                },
+                {
+                    "name": "spaCy Labels",
+                    "request": {
+                        "method": "GET",
+                        "header": [
+                            {
+                                "key": "Authorization",
+                                "value": "Bearer {{api_bearer_token}}",
+                                "type": "text"
+                            }
+                        ],
+                        "url": {
+                            "raw": "{{base_url}}/v1/spacy-labels",
+                            "host": ["{{base_url}}"],
+                            "path": ["v1", "spacy-labels"]
+                        },
+                        "description": "Available spaCy NER labels, source types (entity, regex, token_attr), and example field mappings for structured JSON extraction."
+                    },
+                    "response": []
+                }
+            ]
+        },
+        {
+            "name": "Convert",
+            "description": "Single-file and single-URL conversion to Markdown with optional structured JSON extraction.",
+            "item": [
+                {
+                    "name": "Convert File",
+                    "request": {
+                        "method": "POST",
+                        "header": [
+                            {
+                                "key": "Authorization",
+                                "value": "Bearer {{api_bearer_token}}",
+                                "type": "text"
+                            }
+                        ],
+                        "body": {
+                            "mode": "formdata",
+                            "formdata": [
+                                {
+                                    "key": "file",
+                                    "type": "file",
+                                    "src": "/path/to/your/document.pdf",
+                                    "description": "The file to convert (PDF, DOCX, XLSX, PPTX, etc.)"
+                                },
+                                {
+                                    "key": "plain_text",
+                                    "value": "false",
+                                    "type": "text",
+                                    "description": "Return raw Markdown text instead of JSON wrapper (true/false)"
+                                },
+                                {
+                                    "key": "return_json",
+                                    "value": "false",
+                                    "type": "text",
+                                    "description": "Also extract structured JSON using spaCy NER and regex (true/false)"
+                                },
+                                {
+                                    "key": "mappings",
+                                    "value": "{\"company\": {\"source_type\": \"entity\", \"label\": \"ORG\"}, \"person\": {\"source_type\": \"entity\", \"label\": \"PERSON\"}}",
+                                    "type": "text",
+                                    "description": "Optional JSON string defining field extraction mappings"
+                                }
+                            ]
+                        },
+                        "url": {
+                            "raw": "{{base_url}}/v1/convert/file",
+                            "host": ["{{base_url}}"],
+                            "path": ["v1", "convert", "file"]
+                        },
+                        "description": "Upload a single file and convert it to Markdown. Optionally enable structured JSON extraction with custom field mappings."
+                    },
+                    "response": []
+                },
+                {
+                    "name": "Convert URL",
+                    "request": {
+                        "method": "POST",
+                        "header": [
+                            {
+                                "key": "Authorization",
+                                "value": "Bearer {{api_bearer_token}}",
+                                "type": "text"
+                            },
+                            {
+                                "key": "Content-Type",
+                                "value": "application/json",
+                                "type": "text"
+                            }
+                        ],
+                        "body": {
+                            "mode": "raw",
+                            "raw": "{\n    \"url\": \"https://example.com/document.pdf\",\n    \"return_json\": false,\n    \"mappings\": {\n        \"company\": {\"source_type\": \"entity\", \"label\": \"ORG\"},\n        \"email\": {\"source_type\": \"regex\", \"pattern\": \"\\\\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\\\.[A-Z|a-z]{2,}\\\\b\"}\n    }\n}",
+                            "options": {
+                                "raw": {
+                                    "language": "json"
+                                }
+                            }
+                        },
+                        "url": {
+                            "raw": "{{base_url}}/v1/convert/url",
+                            "host": ["{{base_url}}"],
+                            "path": ["v1", "convert", "url"]
+                        },
+                        "description": "Convert a remote document URL to Markdown. Optionally enable structured JSON extraction with custom field mappings."
+                    },
+                    "response": []
+                }
+            ]
+        },
+        {
+            "name": "Batch",
+            "description": "Bulk conversion of multiple files and URLs.",
+            "item": [
+                {
+                    "name": "Batch Files",
+                    "request": {
+                        "method": "POST",
+                        "header": [
+                            {
+                                "key": "Authorization",
+                                "value": "Bearer {{api_bearer_token}}",
+                                "type": "text"
+                            }
+                        ],
+                        "body": {
+                            "mode": "formdata",
+                            "formdata": [
+                                {
+                                    "key": "files",
+                                    "type": "file",
+                                    "src": "/path/to/your/document1.pdf",
+                                    "description": "First file (up to 10 files max)"
+                                },
+                                {
+                                    "key": "files",
+                                    "type": "file",
+                                    "src": "/path/to/your/document2.docx",
+                                    "description": "Second file"
+                                }
+                            ]
+                        },
+                        "url": {
+                            "raw": "{{base_url}}/v1/batch/files",
+                            "host": ["{{base_url}}"],
+                            "path": ["v1", "batch", "files"]
+                        },
+                        "description": "Upload and convert up to 10 files in a single request. Returns aggregated results with per-file success/failure status."
+                    },
+                    "response": []
+                },
+                {
+                    "name": "Batch URLs",
+                    "request": {
+                        "method": "POST",
+                        "header": [
+                            {
+                                "key": "Authorization",
+                                "value": "Bearer {{api_bearer_token}}",
+                                "type": "text"
+                            },
+                            {
+                                "key": "Content-Type",
+                                "value": "application/json",
+                                "type": "text"
+                            }
+                        ],
+                        "body": {
+                            "mode": "raw",
+                            "raw": "{\n    \"urls\": [\n        \"https://example.com/report1.pdf\",\n        \"https://example.com/report2.docx\"\n    ]\n}",
+                            "options": {
+                                "raw": {
+                                    "language": "json"
+                                }
+                            }
+                        },
+                        "url": {
+                            "raw": "{{base_url}}/v1/batch/urls",
+                            "host": ["{{base_url}}"],
+                            "path": ["v1", "batch", "urls"]
+                        },
+                        "description": "Convert up to 20 URLs in a single request. Returns aggregated results with per-URL success/failure status."
+                    },
+                    "response": []
+                }
+            ]
+        }
+    ],
+    "event": [
+        {
+            "listen": "prerequest",
+            "script": {
+                "type": "text/javascript",
+                "exec": [
+                    "// Validate required environment variables before each request",
+                    "const baseUrl = pm.environment.get('base_url');",
+                    "const token = pm.environment.get('api_bearer_token');",
+                    "",
+                    "if (!baseUrl) {",
+                    "    pm.expect.fail('Missing environment variable: base_url');",
+                    "}",
+                    "",
+                    "if (!token && pm.request.url.path.join('/').startsWith('v1')) {",
+                    "    pm.expect.fail('Missing environment variable: api_bearer_token');",
+                    "}"
+                ]
+            }
+        }
+    ],
+    "variable": [
+        {
+            "key": "base_url",
+            "value": "http://localhost:7860",
+            "type": "string",
+            "description": "Base URL of the Reconciliation-not-ai-extractor-service (e.g., http://localhost:7860 or your deployed URL)"
+        },
+        {
+            "key": "api_bearer_token",
+            "value": "changeme",
+            "type": "string",
+            "description": "API bearer token for authenticated endpoints. Must match the service's API_BEARER_TOKEN environment variable."
+        }
+    ],
+    "auth": {
+        "type": "bearer",
+        "bearer": [
+            {
+                "key": "token",
+                "value": "{{api_bearer_token}}",
+                "type": "string"
+            }
+        ]
+    }
+}

pyproject.toml

@@ -0,0 +1,37 @@
+[build-system]
+requires = ["setuptools>=70", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "reconciliation-non-ai-extractor"
+version = "1.0.0"
+description = "Enterprise document extraction API powered by Microsoft MarkItDown, RapidOCR, and spaCy"
+readme = "README.md"
+requires-python = ">=3.10"
+license = { text = "MIT" }
+
+dependencies = [
+    "markitdown[all]>=0.1.5",
+    "fastapi>=0.111",
+    "uvicorn[standard]>=0.30",
+    "pydantic>=2.7",
+    "pydantic-settings>=2.0.0",
+    "python-multipart>=0.0.9",
+    "httpx>=0.27",
+    "pyfiglet>=1.0.0",
+    "rich>=13.7",
+    "numpy>=1.26.0",
+    "rapidocr-onnxruntime>=1.4.4",
+    "onnxruntime>=1.18.0",
+    "pillow>=10.0.0",
+    "pypdfium2>=4.30.0",
+    "pandas>=2.0.0",
+    "spacy>=3.7.0",
+]
+
+[project.optional-dependencies]
+dev = ["pytest>=8", "pytest-asyncio>=0.23"]
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["reconciliation_non_ai_extractor*"]

requirements.txt

@@ -0,0 +1,16 @@
+markitdown[all]>=0.1.5
+fastapi>=0.111.0
+uvicorn[standard]>=0.30.0
+pydantic>=2.7.0
+pydantic-settings>=2.0.0
+python-multipart>=0.0.9
+httpx>=0.27.0
+pyfiglet>=1.0.0
+rich>=13.7.0
+numpy>=1.26.0
+rapidocr-onnxruntime>=1.4.4
+onnxruntime>=1.18.0
+pillow>=10.0.0
+pypdfium2>=4.30.0
+pandas>=2.0.0
+spacy>=3.7.0

start.sh

@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+LOG_DIR="${LOG_DIR:-/app/logs}"
+mkdir -p "$LOG_DIR"
+LOG_FILE="$LOG_DIR/startup.log"
+
+log() {
+    local level="$1"
+    shift
+    local ts
+    ts="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
+    local msg="[$ts] [$level] $*"
+    echo "$msg"
+    echo "$msg" >> "$LOG_FILE"
+}
+
+info()  { log "INFO " "$@"; }
+ok()    { log "OK   " "$@"; }
+warn()  { log "WARN " "$@"; }
+err()   { log "ERROR" "$@"; }
+step()  { echo ""; log "STEP " "──── $* ────"; }
+
+info "========================================================"
+info "  Reconciliation Non-AI Extractor v1.0.0 — Production Startup"
+info "  $(date -u)"
+info "========================================================"
+
+step "1/2  Python dependencies"
+
+if python -c "import markitdown, fastapi, httpx" 2>/dev/null; then
+    ok "Core dependencies verified."
+else
+    err "Missing core dependencies. Check requirements.txt installation."
+    exit 1
+fi
+
+step "2/2  Core functionality check"
+
+python - << 'PYEOF' >> "$LOG_FILE" 2>&1
+import sys
+print("[functionality-test] Testing core imports...")
+try:
+    import markitdown
+    import fastapi
+    import httpx
+    import pandas
+    print("[functionality-test] All core dependencies imported successfully.")
+except ImportError as e:
+    print(f"[functionality-test] Import error: {e}")
+    sys.exit(1)
+print("[functionality-test] Core functionality check completed.")
+PYEOF
+
+if [ $? -eq 0 ]; then
+    ok "Core functionality check completed."
+else
+    warn "Core functionality check failed. Continuing startup..."
+fi
+
+step "Starting FastAPI server"
+
+HOST="${HOST:-0.0.0.0}"
+PORT="${PORT:-7860}"
+WORKERS="${WORKERS:-1}"
+LOG_LEVEL="${LOG_LEVEL:-info}"
+
+info "Starting uvicorn on $HOST:$PORT (workers=$WORKERS, log_level=$LOG_LEVEL) ..."
+info "Swagger UI : http://$HOST:$PORT/docs"
+info "ReDoc      : http://$HOST:$PORT/redoc"
+info "========================================================"
+
+exec python -m uvicorn app.api.server:app \
+    --host "$HOST" \
+    --port "$PORT" \
+    --workers "$WORKERS" \
+    --log-level "$LOG_LEVEL" \
+    --access-log