asdasd

Dockerfile

@@ -29,4 +29,4 @@ COPY --chown=user . /app
 
 RUN python3 -c "from paddleocr import PaddleOCR; PaddleOCR(use_angle_cls=True, lang='japan')"
 
-CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860", "--workers", "1"]
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]

api/endpoints.py

@@ -1,45 +1,83 @@
 import uuid
-from fastapi import APIRouter, UploadFile, File, HTTPException
+from fastapi import APIRouter, UploadFile, File, HTTPException, Header
 from schemas.responses import StandardResponse, ResponseStatus
 from services.worker import task_queue, results_store
+from core.config import settings
+import os
+import shutil
+import json
 
 router = APIRouter()
 
-@router.get("/health")
+@router.get("/", include_in_schema=False, response_model=StandardResponse[dict])
+async def root():
+    return StandardResponse(
+        status_code=ResponseStatus.SUCCESS,
+        data={"message": "Polyglot OCR Service Online"}
+    )
+
+@router.get("/health", response_model=StandardResponse[dict])
 async def health():
-    return {"status": "healthy", "queue_depth": task_queue.qsize()}
+    return StandardResponse(
+        status_code=ResponseStatus.SUCCESS,
+        data={
+            "worker_alive": True,
+            "queue_depth": task_queue.qsize()
+        }
+    )
 
-# 1. Add this to satisfy the internal health check
-@router.get("/")
-async def root():
-    return JSONResponse(content={"status": "alive"}, status_code=200)
 
 @router.post("/upload", response_model=StandardResponse[dict])
-async def upload(file: UploadFile = File(...)):
-    if not file.filename.lower().endswith(('.pdf', '.png', '.jpg', '.jpeg')):
-        raise HTTPException(status_code=400, detail="Unsupported file format.")
-    
+async def upload(file: UploadFile = File(...), content_length: int = Header(None)):
+    if task_queue.qsize() >= settings.MAX_QUEUE_SIZE:
+        return {"status_code": ResponseStatus.QUEUE_FULL, "data": {"message": "Server busy."}}
+
+    if (content_length or file.size) > settings.MAX_FILE_SIZE:
+        return {"status_code": ResponseStatus.FILE_SIZE_EXCEEDED, "data": {"limit": settings.MAX_FILE_SIZE}}
+
+    if not (ext := os.path.splitext(file.filename.lower())[1]) in {'.pdf', '.png', '.jpg', '.jpeg'}:
+        return {"status_code": ResponseStatus.INVALID_FILE_TYPE, "data": {"message": "Unsupported file."}}
+
     task_id = str(uuid.uuid4())
-    content = await file.read()
+    os.makedirs(settings.STORAGE_DIR, exist_ok=True)
+    file_path = os.path.join(settings.STORAGE_DIR, f"{task_id}{ext}")
     
-    results_store[task_id] = {"status": "pending"}
-    await task_queue.put((task_id, content, file.filename.lower().endswith('.pdf')))
-    
-    return StandardResponse(
-        status_code=ResponseStatus.SUCCESS,
-        data={"task_id": task_id, "queue_position": task_queue.qsize()}
-    )
+    try:
+        with open(file_path, "wb") as buffer:
+            shutil.copyfileobj(file.file, buffer)
+    finally:
+        file.file.close()
+
+    results_store[task_id] = {"file_path": file_path}
+    await task_queue.put((task_id, file_path))
+
+    return {
+        "status_code": ResponseStatus.SUCCESS,
+        "data": {"task_id": task_id, "queue_position": task_queue.qsize()}
+    }
+
 
 @router.get("/result/{task_id}", response_model=StandardResponse[dict])
 async def get_result(task_id: str):
-    result = results_store.get(task_id)
-    if not result:
-        raise HTTPException(status_code=404, detail="Task not found or expired.")
+    task_info = results_store.get(task_id)
     
-    # Map internal status to our Enum
-    status_map = {"completed": ResponseStatus.SUCCESS, "pending": ResponseStatus.PENDING, "failed": ResponseStatus.ERROR}
-    
-    return StandardResponse(
-        status_code=status_map.get(result["status"], ResponseStatus.PENDING),
-        data=result
-    )
+    if not task_info:
+        return {"status_code": ResponseStatus.TASK_NOT_FOUND, "data": {"message": "Expired or missing."}}
+
+    status_code = task_info.get("status_code")
+
+    # If successful, load from disk, return, and don't keep in RAM
+    if status_code == ResponseStatus.SUCCESS:
+        json_path = task_info.get("result_file")
+        if json_path and os.path.exists(json_path):
+            with open(json_path, 'r', encoding='utf-8') as f:
+                disk_data = json.load(f)
+            return {
+                "status_code": ResponseStatus.SUCCESS,
+                "data": disk_data
+            }
+
+    if "error" in task_info:
+        return {"status_code": status_code, "data": {"error": task_info["error"]}}
+
+    return {"status_code": ResponseStatus.PENDING, "data": {"message": "Processing..."}}

core/config.py

@@ -2,9 +2,12 @@ from pydantic_settings import BaseSettings
 
 class Settings(BaseSettings):
     PROJECT_NAME: str = "Polyglot OCR Japanese"
+    LANGUAGE: str = "japan"
     VERSION: str = "1.0.0"
     MAX_FILE_SIZE: int = 500 * 1024 * 1024
     STORAGE_QUOTA: int = 20 * 1024 * 1024 * 1024
     TASK_EXPIRY_SECONDS: int = 86400
+    MAX_QUEUE_SIZE: int = 100 
+    STORAGE_DIR: str = "/tmp/pglt"
 
 settings = Settings()

main.py

@@ -1,14 +1,16 @@
 import asyncio
 from fastapi import FastAPI
-from api.endpoints import router as api_v1_router
-from services.worker import ocr_worker, janitor_task
+from api.endpoints import router
+from services.worker import preprocessor_worker, ocr_worker, janitor_worker
 from core.config import settings
+import os
 
 app = FastAPI(title=settings.PROJECT_NAME)
-app.include_router(api.endpoints.router)
+app.include_router(router)
 
 @app.on_event("startup")
 async def startup_event():
+    os.makedirs(settings.STORAGE_DIR, exist_ok=True)
+    asyncio.create_task(preprocessor_worker())
     asyncio.create_task(ocr_worker())
-    asyncio.create_task(janitor_task())
-asdasd
+    asyncio.create_task(janitor_worker())

requirements.txt

@@ -1,5 +1,6 @@
 fastapi==0.128.8
 uvicorn==0.39.0
+pydantic_settings==2.11.0
 python-multipart==0.0.20
 paddlepaddle==3.3.0
 paddleocr==3.4.0

schemas/responses.py

@@ -1,21 +1,41 @@
 from enum import Enum
 from typing import Generic, TypeVar, Optional, Any
 from datetime import datetime, timezone
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, model_validator
 from core.config import settings
 
 T = TypeVar("T")
 
 class ResponseStatus(str, Enum):
+    # --- Standard States ---
     SUCCESS = "success"
-    ERROR = "error"
     PENDING = "pending"
+    
+    # --- Client-Side Errors (4xx range logic) ---
+    INVALID_FILE_TYPE = "invalid_file_type"      # Magic number mismatch
+    FILE_SIZE_EXCEEDED = "file_size_exceeded"    # Over MAX_FILE_SIZE
+    FILE_CORRUPTED = "file_corrupted"            # Fails integrity check (PIL/PDF)
+    TASK_NOT_FOUND = "task_not_found"            # Invalid or expired UUID
+    
+    # --- Server-Side Errors (5xx range logic) ---
+    INTERNAL_ERROR = "internal_server_error"      # Unhandled exceptions/OCR crashes
+    QUEUE_FULL = "queue_full"                    # task_queue.qsize() > limit
+    STORAGE_FULL = "storage_full"                # Total quota on disk exceeded
+    WORKER_UNAVAILABLE = "worker_unavailable"    # Background loop crashed
 
 class MetaData(BaseModel):
     version: str = settings.VERSION
     timestamp: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
 
 class StandardResponse(BaseModel, Generic[T]):
+    status: bool 
     status_code: ResponseStatus
     data: Optional[T] = None
-    metadata: MetaData = Field(default_factory=MetaData)
+    metadata: MetaData = Field(default_factory=MetaData)
+
+    @model_validator(mode='before')
+    @classmethod
+    def set_status(cls, data: Any) -> Any:
+        if isinstance(data, dict) and 'status_code' in data:
+            data['status'] = data['status_code'] == ResponseStatus.SUCCESS
+        return data

services/ocr_engine.py

@@ -1,8 +1,9 @@
 from paddleocr import PaddleOCR
+from core.config import settings
 
 # Initialized once upon import
 engine = PaddleOCR(
     use_angle_cls=True, 
-    lang='japan', 
+    lang=settings.LANGUAGE, 
     enable_mkldnn=False
 )

services/worker.py

@@ -6,50 +6,151 @@ import numpy as np
 from PIL import Image
 from pdf2image import convert_from_bytes
 from services.ocr_engine import engine
-from utils.helpers import json_ready
+from utils.helpers import json_ready, process_and_save_image
 from core.config import settings
+import os
 
 # Global State
 results_store = {} 
 task_queue = asyncio.Queue()
+ocr_queue = asyncio.Queue()
 
-async def ocr_worker():
+async def preprocessor_worker():
     while True:
-        task_id, content, is_pdf = await task_queue.get()
+        task_id, file_path = await task_queue.get()
+        try:
+            ext = os.path.splitext(file_path.lower())[1]
+            page_paths = []
+
+            if ext == '.pdf':
+                info = pdfinfo_from_path(file_path)
+                max_pages = info["Pages"]
+
+                for i in range(1, max_pages + 1):
+                    page = convert_from_path(
+                        file_path, 
+                        first_page=i, 
+                        last_page=i, 
+                        dpi=150
+                    )[0]
+                    
+                    chunk_path = process_and_save_image(page, file_path, i)
+                    page_paths.append(chunk_path)
+                    del page 
+            else:
+                with Image.open(file_path) as img:
+                    chunk_path = process_and_save_image(img, file_path, 0)
+                    page_paths.append(chunk_path)
+
+            await ocr_queue.put((task_id, page_paths))
+
+        except Exception as e:
+            results_store[task_id] = {
+                "status": "failed",
+                "status_code": ResponseStatus.FILE_CORRUPTED,
+                "error": f"Memory-safe conversion failed: {str(e)}"
+            }
+        finally:
+            if os.path.exists(file_path):
+                os.remove(file_path)
+            task_queue.task_done()
+
+async def ocr_worker():
+   while True:
+        task_id, page_paths = await ocr_queue.get()
         try:
             loop = asyncio.get_event_loop()
-            pages_images = [np.array(p.convert('RGB')) for p in convert_from_bytes(content)] if is_pdf \
-                           else [np.array(Image.open(io.BytesIO(content)).convert('RGB'))]
+            results_list = []
+            
+            is_pdf = len(page_paths) > 1
 
-            output = []
-            for i, img_arr in enumerate(pages_images):
+            for i, p_path in enumerate(page_paths):
+                img = Image.open(p_path)
+                img_arr = np.array(img.convert('RGB'))
+                
                 raw = await loop.run_in_executor(None, functools.partial(engine.ocr, img_arr))
                 
                 page_results = []
-                if isinstance(raw, list) and len(raw) > 0:
+                if isinstance(raw, list) and len(raw) > 0 and raw[0] is not None:
                     data = raw[0]
                     if isinstance(data, dict) and 'rec_texts' in data:
                         for t, s, b in zip(data['rec_texts'], data['rec_scores'], data['dt_polys']):
-                            page_results.append({"text": str(t), "confidence": float(s), "box": json_ready(b)})
+                            page_results.append({
+                                "text": str(t), 
+                                "confidence": round(float(s), 4), 
+                                "box": json_ready(b)
+                            })
                     else:
                         page_results = json_ready(data)
 
-                output.append({"page": i + 1, "results": page_results})
+                results_list.append({"page": i + 1, "results": page_results})
+                
+                if os.path.exists(p_path):
+                    os.remove(p_path)
+
+            inal_data = {"pages": results_list}
+
+            json_filename = f"{task_id}_results.json"
+            json_path = os.path.join(settings.STORAGE_DIR, json_filename)
             
+            with open(json_path, 'w', encoding='utf-8') as f:
+                json.dump(final_data, f, ensure_ascii=False, indent=2)
+
             results_store[task_id] = {
-                "status": "completed", 
-                "data": output, 
+                "status_code": ResponseStatus.SUCCESS,
+                "result_file": json_path,
                 "expires_at": time.time() + settings.TASK_EXPIRY_SECONDS
             }
+
+            del results_list
+            del final_data
+
         except Exception as e:
-            results_store[task_id] = {"status": "failed", "error": str(e), "expires_at": time.time() + settings.TASK_EXPIRY_SECONDS}
+            results_store[task_id] = {
+                "status_code": ResponseStatus.INTERNAL_ERROR,
+                "error": str(e),
+                "expires_at": time.time() + settings.TASK_EXPIRY_SECONDS
+            }
         finally:
-            task_queue.task_done()
+            ocr_queue.task_done()
+
+            
+async def janitor_worker():
+    """
+    Background task to purge files and memory records older than 3 hours.
+    Runs every 30 minutes to ensure the disk stays lean.
+    """
+    # 3 hours in seconds
+    EXPIRY_DURATION = 3 * 60 * 60 
 
-async def janitor_task():
     while True:
-        await asyncio.sleep(3600)
+        # Check every 30 minutes (1800 seconds)
+        await asyncio.sleep(1800) 
+        
         now = time.time()
-        to_delete = [tid for tid, res in results_store.items() if now > res.get("expires_at", 0)]
-        for tid in to_delete:
-            del results_store[tid]
+        expired_tasks = []
+
+        # Iterate safely over a copy of the keys
+        for task_id in list(results_store.keys()):
+            task_info = results_store.get(task_id)
+            if not task_info:
+                continue
+
+            # Calculate age based on when the task was completed/created
+            # If 'expires_at' was set at task completion, we check against that
+            if now > task_info.get("expires_at", 0):
+                
+                # 1. Physically remove the JSON results file
+                json_path = task_info.get("result_file")
+                if json_path and os.path.exists(json_path):
+                    try:
+                        os.remove(json_path)
+                    except Exception as e:
+                        print(f"Janitor failed to delete {json_path}: {e}")
+
+                # 2. Mark for memory cleanup
+                expired_tasks.append(task_id)
+
+        # 3. Remove from memory store
+        for task_id in expired_tasks:
+            del results_store[task_id]

utils/helpers.py

@@ -1,4 +1,5 @@
 import numpy as np
+from PIL import Image
 
 def json_ready(obj):
     """Recursively converts NumPy types to basic Python types for JSON compatibility."""
@@ -12,4 +13,17 @@ def json_ready(obj):
         return float(obj)
     if isinstance(obj, (np.int32, np.int64)):
         return int(obj)
-    return obj
+    return obj
+
+def process_and_save_image(pil_img, original_path, page_num):
+    img = pil_img.convert('RGB')
+    width, height = img.size
+    
+    scale = 960 / max(width, height)
+    if scale < 1.0:
+        new_size = (int(width * scale), int(height * scale))
+        img = img.resize(new_size, Image.Resampling.LANCZOS)
+    
+    chunk_path = f"{original_path}_p{page_num}.jpg"
+    img.save(chunk_path, "JPEG", quality=85, optimize=True)
+    return chunk_path