feat: cleanup junks

.gitignore

@@ -6,6 +6,7 @@ postman_collection.json
 # Runtime data (vector stores, SQLite DBs, model caches, etc.)
 data/
 *.so
+test_deploy_flow.py
 
 .Python
 build/
@@ -127,5 +128,5 @@ persistence/
 local_deploy.py
 test_vector_store_async.py
 deploy_sdk.py
-
+tests
 deploy_hf.py

app/api/server.py

@@ -13,7 +13,6 @@ from app.core.database import pool_manager
 from app.core.logger import get_logger
 from app.core.redis_client import create_redis_client, close_redis
 from app.core.scripts import load_scripts
-from app.core.vector_store.deps import init_vector_store_db
 from app.services.embeddings_service import EmbeddingService
 from app.services.vector_store_service import VectorStoreService
 from app.api.v1.router import api_v1_router
@@ -49,14 +48,12 @@ async def lifespan(app: FastAPI):
     _logger.info("Authentication database initialized")
 
     _logger.info("Initializing vector store database...")
-    await init_vector_store_db()
     await _vector_store_service.init_db()
     _logger.info("Vector store database initialized with %d stores", len(_vector_store_service.list_stores()))
 
     _logger.info("Initializing embedding service (loading 384-dim model)...")
     loop = asyncio.get_running_loop()
     await loop.run_in_executor(None, _embedding_service.load_model, 384)
-    # await loop.run_in_executor(None, _embedding_service.load_vision_model)  # DISABLED (OOM mitigation)
     _logger.info("Embedding service initialized with dims: %s", _embedding_service.loaded_dimensions)
     _logger.info("Vector store service initialized with %d existing stores", len(_vector_store_service.list_stores()))
 

app/api/v1/auth.py

@@ -7,7 +7,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.config import get_settings
 from app.core.auth.deps import get_current_user, get_db, get_temp_db_warning, require_application_id
-from app.core.auth.models import RefreshSession, User
+from app.core.auth.models import User
 from app.core.auth.schemas import (
     ChangePasswordSchema,
     ForgotPasswordSchema,
@@ -24,7 +24,6 @@ from app.core.auth.schemas import (
     UpdateProfileSchema,
     UserSchemaField,
     UserSchemaResponse,
-    DataResponse,
 )
 from app.services.auth_service import AuthService
 

app/api/v1/batch.py

@@ -1,16 +1,16 @@
 from __future__ import annotations
 
 import asyncio
-import concurrent.futures
 import json as json_mod
-import os
 import time
 from typing import Annotated, List, Optional
+from urllib.parse import urlparse
 
 import httpx
 from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, UploadFile
 
 from app.api.deps import get_converter_service, get_extraction_service, get_text_cleaner_service, require_auth
+from app.api.v1.convert import _build_metadata, _thread_pool
 from app.config import get_settings
 from app.core.logger import get_logger
 from app.models.domain import ConversionError
@@ -18,7 +18,6 @@ from app.models.schemas import (
     BatchFileResult,
     BatchResponse,
     BatchUrlRequest,
-    ConversionMetadata,
 )
 from app.services.converter_service import ConverterService
 from app.services.extraction_service import ExtractionService
@@ -29,21 +28,6 @@ _logger = get_logger(__name__)
 _settings = get_settings()
 _MAX_UPLOAD_BYTES = _settings.max_upload_bytes
 _MAX_BATCH_FILES = _settings.max_batch_files
-_MAX_WORKERS = min(32, (os.cpu_count() or 1) + 4)
-_thread_pool = concurrent.futures.ThreadPoolExecutor(max_workers=_MAX_WORKERS)
-
-
-def _build_metadata(result) -> ConversionMetadata:
-    return ConversionMetadata(
-        source=result.source,
-        char_count=result.char_count,
-        word_count=result.word_count,
-        line_count=result.line_count,
-        file_size_bytes=result.file_size_bytes,
-        mime_type=result.mime_type,
-        content_hash=result.content_hash,
-        token_estimate=result.token_estimate,
-    )
 
 
 def _batch_result_from_error(name: str, err: ConversionError) -> BatchFileResult:
@@ -165,7 +149,6 @@ async def batch_urls(
 
     async def process_single_url(url: str) -> BatchFileResult:
         _logger.info("Batch processing URL: %s", url)
-        from urllib.parse import urlparse
         parsed = urlparse(url)
         filename = parsed.path.split("/")[-1] or "url_content"
 

app/api/v1/convert.py

@@ -2,14 +2,15 @@ from __future__ import annotations
 
 import asyncio
 import concurrent.futures
+import hashlib
 import json as json_mod
 import os
 from typing import Annotated, Any, Dict, Optional
 from urllib.parse import urlparse
 
 import httpx
+from fastapi.responses import PlainTextResponse
 from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, UploadFile, status
-from pydantic import BaseModel
 
 from app.config import get_settings
 from app.api.deps import (
@@ -83,7 +84,6 @@ async def _build_response(
     if clean_content and content != result.markdown:
         lines = content.splitlines()
         words = content.split()
-        import hashlib
         metadata = ConversionMetadata(
             source=result.source,
             char_count=len(content),
@@ -171,7 +171,6 @@ async def convert_file(
         content = await loop.run_in_executor(_thread_pool, text_cleaner_service.clean, content)
 
     if plain_text:
-        from fastapi.responses import PlainTextResponse
         return PlainTextResponse(content)
 
     response = await _build_response(

app/api/v1/embeddings.py

@@ -2,19 +2,14 @@ from __future__ import annotations
 
 import asyncio
 import concurrent.futures
-# import io  # DISABLED (OOM mitigation) — only used by vision
 import os
 import time
-from typing import Annotated, List, Optional
-
-# import httpx  # DISABLED (OOM mitigation) — only used by vision
-from fastapi import APIRouter, Depends, File, Form, HTTPException, UploadFile, status
-# from PIL import Image  # DISABLED (OOM mitigation)
+from fastapi import APIRouter, Depends, HTTPException
 
 from app.api.deps import require_auth, get_embeddings_service
 from app.config import get_settings
 from app.core.logger import get_logger
-from app.models.schemas import EmbeddingItem, EmbeddingRequest, EmbeddingResponse  # , VisionUrlRequest  # DISABLED (OOM mitigation)
+from app.models.schemas import EmbeddingItem, EmbeddingRequest, EmbeddingResponse
 from app.services.embeddings_service import EmbeddingService
 
 router = APIRouter()
@@ -22,41 +17,6 @@ _logger = get_logger(__name__)
 _settings = get_settings()
 _MAX_WORKERS = min(32, (os.cpu_count() or 1) + 4)
 _thread_pool = concurrent.futures.ThreadPoolExecutor(max_workers=_MAX_WORKERS)
-# _MAX_VISION_ITEMS = 5  # DISABLED (OOM mitigation)
-# _MAX_IMAGE_BYTES = 15 * 1024 * 1024  # DISABLED (OOM mitigation)
-
-
-# def _validate_image(raw: bytes, source: str) -> Image.Image:  # DISABLED (OOM mitigation)
-#     # FIX: Check if the file is completely empty (0 bytes)
-#     if not raw:
-#         raise ValueError(f"File {source} is empty (0 bytes).")
-#         
-#     if len(raw) > _MAX_IMAGE_BYTES:
-#         raise ValueError(f"Image {source} exceeds 15 MB limit")
-#         
-#     try:
-#         img = Image.open(io.BytesIO(raw))
-#         img.load()
-#         if img.mode != "RGB":
-#             img = img.convert("RGB")
-#         return img
-#     except Exception as exc:
-#         raise ValueError(f"Invalid image {source}: {exc}")
-
-
-# async def _download_image(url: str) -> bytes:  # DISABLED (OOM mitigation)
-#     try:
-#         async with httpx.AsyncClient(timeout=30.0, follow_redirects=True) as client:
-#             resp = await client.get(url)
-#             resp.raise_for_status()
-#             ctype = resp.headers.get("content-type", "")
-#             if not ctype.startswith("image/"):
-#                 raise ValueError(f"URL {url} returned non-image Content-Type: {ctype}")
-#             return resp.content
-#     except httpx.HTTPError as exc:
-#         raise ValueError(f"Failed to download {url}: {exc}")
-
-
 @router.post(
     "/embeddings",
     response_model=EmbeddingResponse,
@@ -126,185 +86,3 @@ async def create_embeddings(
     )
 
 
-# @router.post(  # DISABLED (OOM mitigation)
-#     "/embeddings/vision/file",
-#     response_model=EmbeddingResponse,
-#     summary="Generate embeddings from uploaded images",
-# )
-# async def create_vision_embeddings_file(
-#     files: Annotated[List[UploadFile], File(description="Image files to embed (max 5)")],
-#     token: str = Depends(require_auth),
-#     embedding_service: EmbeddingService = Depends(get_embeddings_service),
-# ) -> EmbeddingResponse:
-#     if not files:
-#         raise HTTPException(status_code=400, detail={"success": False, "message": "No files provided."})
-#     if len(files) > _MAX_VISION_ITEMS:
-#         raise HTTPException(status_code=400, detail={"success": False, "message": f"Maximum {_MAX_VISION_ITEMS} images per request."})
-# 
-#     if not embedding_service._vision_loaded:
-#         raise HTTPException(status_code=503, detail={"success": False, "message": "Vision model not loaded."})
-# 
-#     _logger.info("Vision embedding file request: files=%s", len(files))
-# 
-#     dim = embedding_service.vision_dimension
-#     start = time.perf_counter()
-#     images: List[Image.Image] = []
-#     item_results: List[EmbeddingItem] = []
-# 
-#     for f in files:
-#         t0 = time.perf_counter()
-#         try:
-#             # FIX: Guarantee the file cursor is at the beginning before reading!
-#             await f.seek(0)
-#             raw = await f.read()
-#             
-#             img = await asyncio.get_running_loop().run_in_executor(_thread_pool, _validate_image, raw, f.filename or "unknown")
-#             images.append(img)
-#         except Exception as exc:
-#             elapsed = (time.perf_counter() - t0) * 1000
-#             item_results.append(EmbeddingItem(
-#                 success=False,
-#                 time_ms=round(elapsed, 3),
-#                 error_message=str(exc),
-#             ))
-# 
-#     if not images:
-#         total_ms = (time.perf_counter() - start) * 1000
-#         return EmbeddingResponse(
-#             success=False,
-#             time_ms=round(total_ms, 3),
-#             success_count=0,
-#             failed_count=len(item_results),
-#             error_message="No valid images could be processed.",
-#             results=item_results,
-#         )
-# 
-#     try:
-#         loop = asyncio.get_running_loop()
-#         vectors = await loop.run_in_executor(
-#             _thread_pool,
-#             embedding_service.generate_image_embedding,
-#             images,
-#         )
-#     except Exception as exc:
-#         elapsed = (time.perf_counter() - start) * 1000
-#         _logger.error("Vision embedding error: %s", exc)
-#         for _ in range(len(images) - len(item_results)):
-#             item_results.append(EmbeddingItem(success=False, time_ms=0, error_message=str(exc)))
-#         return EmbeddingResponse(
-#             success=False,
-#             time_ms=round(elapsed, 3),
-#             success_count=0,
-#             failed_count=len(item_results),
-#             error_message=str(exc),
-#             results=item_results,
-#         )
-# 
-#     total_ms = (time.perf_counter() - start) * 1000
-#     for i, vec in enumerate(vectors):
-#         item_results.append(EmbeddingItem(
-#             success=True,
-#             time_ms=round(total_ms / len(vectors), 3),
-#             embeddings=vec,
-#             dimension=dim,
-#         ))
-# 
-#     success_count = sum(1 for r in item_results if r.success)
-#     failed_count = len(item_results) - success_count
-#     _logger.info("Vision embedding success: items=%s, success=%s, failed=%s, total_ms=%s",
-#                  len(item_results), success_count, failed_count, round(total_ms, 3))
-#     return EmbeddingResponse(
-#         success=failed_count == 0,
-#         time_ms=round(total_ms, 3),
-#         success_count=success_count,
-#         failed_count=failed_count,
-#         results=item_results,
-#     )
-# 
-# 
-# @router.post(  # DISABLED (OOM mitigation)
-#     "/embeddings/vision/url",
-#     response_model=EmbeddingResponse,
-#     summary="Generate embeddings from image URLs",
-# )
-# async def create_vision_embeddings_url(
-#     body: VisionUrlRequest,
-#     token: str = Depends(require_auth),
-#     embedding_service: EmbeddingService = Depends(get_embeddings_service),
-# ) -> EmbeddingResponse:
-#     if not embedding_service._vision_loaded:
-#         raise HTTPException(status_code=503, detail={"success": False, "message": "Vision model not loaded."})
-# 
-#     _logger.info("Vision embedding URL request: urls=%s", len(body.urls))
-# 
-#     dim = embedding_service.vision_dimension
-#     start = time.perf_counter()
-#     images: List[Image.Image] = []
-#     item_results: List[EmbeddingItem] = []
-# 
-#     for url in body.urls:
-#         t0 = time.perf_counter()
-#         try:
-#             raw = await _download_image(url)
-#             img = await asyncio.get_running_loop().run_in_executor(_thread_pool, _validate_image, raw, url)
-#             images.append(img)
-#         except Exception as exc:
-#             elapsed = (time.perf_counter() - t0) * 1000
-#             item_results.append(EmbeddingItem(
-#                 success=False,
-#                 time_ms=round(elapsed, 3),
-#                 error_message=str(exc),
-#             ))
-# 
-#     if not images:
-#         total_ms = (time.perf_counter() - start) * 1000
-#         return EmbeddingResponse(
-#             success=False,
-#             time_ms=round(total_ms, 3),
-#             success_count=0,
-#             failed_count=len(item_results),
-#             error_message="No valid images could be downloaded.",
-#             results=item_results,
-#         )
-# 
-#     try:
-#         loop = asyncio.get_running_loop()
-#         vectors = await loop.run_in_executor(
-#             _thread_pool,
-#             embedding_service.generate_image_embedding,
-#             images,
-#         )
-#     except Exception as exc:
-#         elapsed = (time.perf_counter() - start) * 1000
-#         _logger.error("Vision embedding error: %s", exc)
-#         for _ in range(len(images) - len(item_results)):
-#             item_results.append(EmbeddingItem(success=False, time_ms=0, error_message=str(exc)))
-#         return EmbeddingResponse(
-#             success=False,
-#             time_ms=round(elapsed, 3),
-#             success_count=0,
-#             failed_count=len(item_results),
-#             error_message=str(exc),
-#             results=item_results,
-#         )
-# 
-#     total_ms = (time.perf_counter() - start) * 1000
-#     for i, vec in enumerate(vectors):
-#         item_results.append(EmbeddingItem(
-#             success=True,
-#             time_ms=round(total_ms / len(vectors), 3),
-#             embeddings=vec,
-#             dimension=dim,
-#         ))
-# 
-#     success_count = sum(1 for r in item_results if r.success)
-#     failed_count = len(item_results) - success_count
-#     _logger.info("Vision embedding success: items=%s, success=%s, failed=%s, total_ms=%s",
-#                  len(item_results), success_count, failed_count, round(total_ms, 3))
-#     return EmbeddingResponse(
-#         success=failed_count == 0,
-#         time_ms=round(total_ms, 3),
-#         success_count=success_count,
-#         failed_count=failed_count,
-#         results=item_results,
-#     )

app/api/v1/reconcile.py

@@ -2,7 +2,6 @@ from __future__ import annotations
 
 import asyncio
 import json
-import os
 import time
 import uuid
 from typing import Any, Dict, List, Optional
@@ -22,6 +21,7 @@ from app.services.reconciliation_service import (
     compare_rows,
     compare_schemas,
     download_file_with_retry,
+    extract_extension,
     normalize_dataframe,
     read_to_dataframe,
     reconcile_columns,
@@ -73,10 +73,6 @@ class ReconciliationResponse(BaseModel):
     results: List[ReconciliationPairResult]
 
 
-def _extract_extension(filename: str) -> str:
-    return os.path.splitext(filename)[1].lstrip(".").lower()
-
-
 def _failed_result(pair_index: int, error: str) -> Dict[str, Any]:
     return {
         "pair_index": pair_index,
@@ -246,8 +242,8 @@ async def reconcile_files(
             continue
         if src is None or dst is None:
             raise HTTPException(status_code=400, detail={"success": False, "message": f"Pair {idx}: Both source and destination files required."})
-        src_ext = _extract_extension(src.filename or "")
-        dst_ext = _extract_extension(dst.filename or "")
+        src_ext = extract_extension(src.filename or "")
+        dst_ext = extract_extension(dst.filename or "")
         _validate_extensions(src_ext, dst_ext, idx)
         pairs.append((src, dst, cm, idx, src_ext, dst_ext))
 
@@ -284,13 +280,13 @@ async def reconcile_urls(
         raise HTTPException(status_code=400, detail={"success": False, "message": "At least 1 pair is required."})
 
     for idx, pair in enumerate(body.pairs, 1):
-        _validate_extensions(_extract_extension(pair.source), _extract_extension(pair.destination), idx)
+        _validate_extensions(extract_extension(pair.source), extract_extension(pair.destination), idx)
 
     async with aiohttp.ClientSession() as session:
         async def process_url_pair(pair: ReconciliationPair, idx: int) -> Dict[str, Any]:
             async with semaphore_jobs:
-                src_ext = _extract_extension(pair.source)
-                dst_ext = _extract_extension(pair.destination)
+                src_ext = extract_extension(pair.source)
+                dst_ext = extract_extension(pair.destination)
 
                 try:
                     src_data, dst_data = await asyncio.gather(

app/api/v1/system.py

@@ -8,7 +8,7 @@ import time
 from datetime import datetime, timezone
 from pathlib import Path
 
-from fastapi import APIRouter, Depends, File, HTTPException, UploadFile, status
+from fastapi import APIRouter, Depends, File, HTTPException, UploadFile
 from fastapi.responses import StreamingResponse
 
 from app.api.deps import get_extraction_service, require_auth
@@ -40,8 +40,17 @@ _logger = get_logger(__name__)
 _MAINTENANCE_MODE: bool = False
 
 
+def _maintenance_lock_path() -> Path:
+    return _get_data_dir() / ".maintenance.lock"
+
+
 def is_maintenance() -> bool:
-    return _MAINTENANCE_MODE or os.environ.get("APP_UNDER_MAINTENANCE", "").lower() in ("1", "true", "yes")
+    if _MAINTENANCE_MODE:
+        return True
+    if os.environ.get("APP_UNDER_MAINTENANCE", "").lower() in ("1", "true", "yes"):
+        return True
+    lock_path = _maintenance_lock_path()
+    return lock_path.is_file()
 
 
 @router.get("/health", response_model=HealthResponse, summary="Health check")
@@ -198,6 +207,12 @@ async def upload_and_restore(
 async def enable_maintenance(token: str = Depends(require_auth)):
     global _MAINTENANCE_MODE
     _MAINTENANCE_MODE = True
+    lock_path = _maintenance_lock_path()
+    try:
+        lock_path.parent.mkdir(parents=True, exist_ok=True)
+        lock_path.write_text(str(time.time()), encoding="utf-8")
+    except Exception as exc:
+        _logger.warning("Could not write maintenance lock file: %s", exc)
     _logger.warning("Maintenance mode ENABLED — all write operations blocked")
     return {"success": True, "message": "Maintenance mode enabled", "maintenance": True}
 
@@ -206,14 +221,27 @@ async def enable_maintenance(token: str = Depends(require_auth)):
 async def disable_maintenance(token: str = Depends(require_auth)):
     global _MAINTENANCE_MODE
     _MAINTENANCE_MODE = False
+    lock_path = _maintenance_lock_path()
+    try:
+        lock_path.unlink(missing_ok=True)
+    except Exception as exc:
+        _logger.warning("Could not remove maintenance lock file: %s", exc)
     _logger.warning("Maintenance mode DISABLED — write operations resumed")
     return {"success": True, "message": "Maintenance mode disabled", "maintenance": False}
 
 
 @router.get("/maintenance", summary="Check maintenance mode status")
 async def maintenance_status(token: str = Depends(require_auth)):
+    lock_path = _maintenance_lock_path()
+    source = "off"
+    if _MAINTENANCE_MODE:
+        source = "api_flag"
+    elif os.environ.get("APP_UNDER_MAINTENANCE"):
+        source = "env_var"
+    elif lock_path.is_file():
+        source = "lock_file"
     return {
         "success": True,
         "maintenance": is_maintenance(),
-        "source": "api_flag" if _MAINTENANCE_MODE else ("env_var" if os.environ.get("APP_UNDER_MAINTENANCE") else "off"),
+        "source": source,
     }

app/api/v1/vector_stores.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import asyncio
 import time
 
+import httpx
 from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, UploadFile, status
 
 from app.api.deps import get_vector_store_service, require_auth
@@ -128,7 +129,7 @@ async def get_vector_store(
         embedding_dimension=stats["embedding_dimension"],
         document_count=stats["document_count"],
         created_at=stats["created_at"],
-        metadata=stats["metadata_json"],
+        metadata=stats["metadata"],
     )
 
 
@@ -287,8 +288,6 @@ async def ingest_pdf_url(
     if record is None:
         raise HTTPException(status_code=404, detail=f"Vector store {store_id} not found")
 
-    import httpx
-
     try:
         async with httpx.AsyncClient(timeout=30.0, follow_redirects=True) as client:
             resp = await client.get(body.url)

app/api/v1/webhook_socket.py

@@ -1,9 +1,10 @@
 from __future__ import annotations
 
 import asyncio
+import hmac as hmac_mod
 import json
 
-from fastapi import APIRouter, Depends, Request, WebSocket, WebSocketDisconnect
+from fastapi import APIRouter, Depends, HTTPException, Request, WebSocket, WebSocketDisconnect
 
 from app.api.deps import require_auth
 from app.models.schemas import (
@@ -29,7 +30,6 @@ async def create_channel(
     token: str = Depends(require_auth),
 ):
     if body.channel_id and manager.get_channel(body.channel_id):
-        from fastapi import HTTPException
         raise HTTPException(status_code=409, detail=f"Channel '{body.channel_id}' already exists")
 
     ch = manager.create_channel(
@@ -73,7 +73,6 @@ async def channel_info(
     channel_id: str,
     token: str = Depends(require_auth),
 ):
-    from fastapi import HTTPException
     ch = manager.get_channel(channel_id)
     if not ch:
         raise HTTPException(status_code=404, detail="Channel not found")
@@ -94,7 +93,6 @@ async def delete_channel(
     channel_id: str,
     token: str = Depends(require_auth),
 ):
-    from fastapi import HTTPException
     if manager.delete_channel(channel_id):
         return ChannelDeleteResponse(deleted=channel_id)
     raise HTTPException(status_code=404, detail="Channel not found")
@@ -105,7 +103,6 @@ async def handle_webhook(
     channel_id: str,
     request: Request,
 ):
-    from fastapi import HTTPException
     ch = manager.get_channel(channel_id)
     if not ch:
         raise HTTPException(status_code=404, detail="Channel not found")
@@ -174,7 +171,6 @@ async def websocket_endpoint(
         return
 
     if ch.secret:
-        import hmac as hmac_mod
         if not hmac_mod.compare_digest(secret, ch.secret):
             await websocket.accept()
             await websocket.send_json({"event": "error", "message": "unauthorized"})

app/config.py

@@ -115,5 +115,3 @@ DEFAULT_STREAM = False
 @lru_cache(maxsize=1)
 def get_settings() -> Settings:
     return Settings()
-
-# Persistence test marker - v2

app/core/auth/deps.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import logging
-from datetime import datetime, timedelta, timezone
 from typing import Annotated, Any, AsyncGenerator, Optional
 
 import jwt
@@ -38,10 +37,6 @@ AsyncSessionLocal = async_sessionmaker(engine, class_=AsyncSession, expire_on_co
 ph = PasswordHasher()
 
 
-def _now() -> datetime:
-    return datetime.now(timezone.utc)
-
-
 class TempDatabaseWarning:
     code: str = "TEMP_DATABASE"
     message: str = (

app/core/banner.py

@@ -1 +0,0 @@
-

app/core/database/mongodb.py

@@ -99,7 +99,4 @@ class MongoDBExecutor(BaseExecutor):
     async def _close_pool(self, pool: AsyncIOMotorClient) -> None:
         pool.close()
 
-    async def _get_or_create_pool(self) -> AsyncIOMotorClient:
-        if self._pool is not None:
-            return self._pool
-        return await super()._get_or_create_pool()
+

app/core/database/mysql.py

@@ -1,7 +1,5 @@
 from __future__ import annotations
 
-import asyncio
-import sys
 from typing import Any
 
 import aiomysql
@@ -27,32 +25,6 @@ class MySQLExecutor(BaseExecutor):
             pool_recycle=3600,
         )
 
-    async def _get_or_create_pool(self) -> aiomysql.Pool:
-        async with self._lock:
-            if self._pool is not None:
-                return self._pool
-
-            last_error: Exception | None = None
-            for attempt in range(self._max_connection_retries):
-                try:
-                    self._pool = await asyncio.wait_for(
-                        self._create_pool(),
-                        timeout=self._config.connection_timeout_seconds,
-                    )
-                    return self._pool
-                except Exception as exc:
-                    last_error = exc
-                    _logger.warning(
-                        "MySQL connection failed (attempt %d): %s",
-                        attempt + 1, exc,
-                    )
-                    if attempt < self._max_connection_retries - 1:
-                        await asyncio.sleep(0.1 * (2**attempt))
-
-            raise RuntimeError(
-                f"Failed to connect to MySQL after {self._max_connection_retries} attempts"
-            ) from last_error
-
     async def _execute_queries(
         self, pool: aiomysql.Pool, queries: list[str], use_transaction: bool
     ) -> list[StatementResult]:

app/models/__init__.py

@@ -15,7 +15,6 @@ from app.models.schemas import (
     SpacyLabelsResponse,
     SupportedFormatsResponse,
     UrlRequest,
-    # VisionUrlRequest,  # DISABLED (OOM mitigation)
 )
 
 __all__ = [
@@ -34,5 +33,4 @@ __all__ = [
     "InfoResponse",
     "SupportedFormatsResponse",
     "SpacyLabelsResponse",
-    # "VisionUrlRequest",  # DISABLED (OOM mitigation)
 ]

app/models/schemas.py

@@ -334,20 +334,8 @@ class EmbeddingResponse(BaseModel):
     results: List[EmbeddingItem]
 
 
-# class VisionUrlRequest(BaseModel):  # DISABLED (OOM mitigation)
-#     urls: List[str] = Field(..., min_length=1, max_length=5, description="Array of image URLs to embed (max 5)")
-# 
-#     @field_validator("urls")
-#     @classmethod
-#     def validate_urls(cls, v: List[str]) -> List[str]:
-#         for url in v:
-#             if not url.startswith(("http://", "https://")):
-#                 raise ValueError(f"Invalid URL scheme: {url}")
-#         return v
-
-
 class CodeItem(BaseModel):
-    language: Literal["python", "javascript"]  # "java" DISABLED (OOM mitigation)
+    language: Literal["python", "javascript"]
     code: str = Field(..., min_length=1, max_length=65536, description="Source code to execute")
 
 

app/services/auth_service.py

@@ -14,6 +14,7 @@ from sqlalchemy import select, update
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.config import get_settings
+from app.core.auth.models import _utcnow as _now
 from app.core.auth.models import RefreshSession, Role, User
 from app.core.auth.schemas import (
     ChangePasswordSchema,
@@ -31,10 +32,6 @@ _settings = get_settings()
 ph = PasswordHasher()
 
 
-def _now() -> datetime:
-    return datetime.now(timezone.utc)
-
-
 def _token_key(raw: str) -> str:
     return hashlib.sha256(raw.encode("utf-8")).hexdigest()
 

app/services/code_executor_service.py

@@ -70,27 +70,6 @@ class CodeSanitizer:
             (r"worker_threads", "worker_threads not allowed"),
             (r"Buffer\s*\.", "Buffer not allowed"),
         ],
-        # "java": [  # DISABLED (OOM mitigation)
-        #     (r"ProcessBuilder", "ProcessBuilder not allowed"),
-        #     (r"Runtime\.exec", "Runtime.exec() not allowed"),
-        #     (r"Runtime\.getRuntime\s*\(\s*\)", "Runtime.getRuntime() not allowed"),
-        #     (r"System\.exit\s*\(", "System.exit() not allowed"),
-        #     (r"System\.gc\s*\(", "System.gc() not allowed"),
-        #     (r"File\s*\(", "File operations not allowed"),
-        #     (r"FileInputStream", "FileInputStream not allowed"),
-        #     (r"FileOutputStream", "FileOutputStream not allowed"),
-        #     (r"FileReader", "FileReader not allowed"),
-        #     (r"FileWriter", "FileWriter not allowed"),
-        #     (r"Socket\s*\(", "Socket not allowed"),
-        #     (r"ServerSocket\s*\(", "ServerSocket not allowed"),
-        #     (r"URL\s*\(", "URL not allowed"),
-        #     (r"Class\.forName", "Class.forName() not allowed"),
-        #     (r"Thread\s*\(", "Thread not allowed"),
-        #     (r"ThreadPoolExecutor", "ThreadPoolExecutor not allowed"),
-        #     (r"Runtime\.", "Runtime not allowed"),
-        #     (r"System\.getProperty", "System.getProperty not allowed"),
-        #     (r"System\.getenv", "System.getenv not allowed"),
-        # ],
     }
 
     @classmethod
@@ -101,11 +80,6 @@ class CodeSanitizer:
                 return False, f"Forbidden: {message}"
         return True, None
 
-    # @classmethod  # DISABLED (OOM mitigation)
-    # def validate_java_class(cls, code: str) -> tuple[bool, Optional[str]]:
-    #     if not re.search(r"public\s+class\s+Main\s*\{", code):
-    #         return False, "Java code must have 'public class Main' with 'public static void main(String[] args)'"
-    #     return True, None
 
 
 class CodeExecutorService:
@@ -131,15 +105,6 @@ class CodeExecutorService:
                 "language": language, "timed_out": False,
             }
 
-        # if language == "java":  # DISABLED (OOM mitigation)
-        #     valid, err = CodeSanitizer.validate_java_class(code)
-        #     if not valid:
-        #         return {
-        #             "success": False, "output": "", "error": err,
-        #             "exit_code": None, "execution_time_ms": None,
-        #             "language": language, "timed_out": False,
-        #         }
-
         exec_timeout = min(timeout or self._max_execution_time, self._max_execution_time)
 
         async with self._semaphore:
@@ -233,14 +198,14 @@ class CodeExecutorService:
 
     async def check_runtimes(self) -> dict[str, str]:
         status = {}
-        for lang, runtime in [("python", "python3"), ("javascript", "node")]:  # "java" DISABLED (OOM mitigation)
+        for lang, runtime in [("python", "python3"), ("javascript", "node")]:
             path = shutil.which(runtime)
             status[lang] = f"found at {path}" if path else "missing"
         return status
 
     @staticmethod
     def _filename_for(language: str) -> str:
-        return {"python": "code.py", "javascript": "code.js"  # "java": "Main.java" DISABLED (OOM mitigation)
+        return {"python": "code.py", "javascript": "code.js"
                }[language]
 
     @staticmethod
@@ -251,7 +216,4 @@ class CodeExecutorService:
             return ["python3", str(sandbox_py), str(run_dir / filename)]
         elif language == "javascript":
             return ["node", str(sandbox_js), str(run_dir / filename)]
-        # elif language == "java":  # DISABLED (OOM mitigation)
-        #     return ["sh", "-c",
-        #             f"cd {run_dir} && javac Main.java 2>&1 && java -XX:CompressedClassSpaceSize=64m -Xmx96m Main 2>&1"]
         return []

app/services/converter_service.py

@@ -8,7 +8,6 @@ import time
 from pathlib import Path
 from urllib.parse import urlparse
 
-# import httpx  # DISABLED (OOM mitigation) — only used by YouTube
 from markitdown import MarkItDown
 
 from app.core.constants import IMAGE_EXTENSIONS, IMAGE_MIME_PREFIXES
@@ -19,101 +18,6 @@ from app.services.ocr_service import ocr_image, ocr_pdf
 _logger = get_logger(__name__)
 
 
-# def _extract_youtube_video_id(url_or_id: str) -> str:  # DISABLED (OOM mitigation)
-#     if len(url_or_id) == 11 and not url_or_id.startswith("http"):
-#         return url_or_id
-#     pattern = r"(?:v=|\/)([0-9A-Za-z_-]{11}).*"
-#     match = re.search(pattern, url_or_id)
-#     if match:
-#         return match.group(1)
-#     raise ValueError(f"Could not extract a valid YouTube ID from: {url_or_id}")
-# 
-# 
-# def _fetch_youtube_oembed(video_id: str) -> dict | None:  # DISABLED (OOM mitigation)
-#     oembed_url = f"https://www.youtube.com/oembed?url=https://www.youtube.com/watch?v={video_id}&format=json"
-#     try:
-#         resp = httpx.get(oembed_url, timeout=10.0)
-#         resp.raise_for_status()
-#         return resp.json()
-#     except Exception as exc:
-#         _logger.warning("YouTube oEmbed failed for %s: %s", video_id, exc)
-#         return None
-# 
-# 
-# def _fetch_youtube_transcript(video_id: str) -> str | None:  # DISABLED (OOM mitigation)
-#     try:
-#         import json as _json
-#         import os as _os
-# 
-#         from youtube_transcript_api import YouTubeTranscriptApi
-#         from youtube_transcript_api.formatters import TextFormatter
-#         from youtube_transcript_api._errors import (
-#             TranscriptsDisabled,
-#             NoTranscriptFound,
-#             VideoUnavailable,
-#         )
-# 
-#         kwargs: dict = {}
-#         cookies_raw = _os.environ.get("YOUTUBE_COOKIES", "").strip()
-#         if cookies_raw:
-#             try:
-#                 import requests as _requests
-#                 session = _requests.Session()
-#                 session.cookies.update(_json.loads(cookies_raw))
-#                 kwargs["http_client"] = session
-#             except Exception as exc:
-#                 _logger.warning("Failed to apply YOUTUBE_COOKIES: %s", exc)
-# 
-#         ytt_api = YouTubeTranscriptApi(**kwargs)
-#         transcript_data = ytt_api.fetch(video_id, languages=["en"])
-#         formatter = TextFormatter()
-#         return formatter.format_transcript(transcript_data)
-#     except TranscriptsDisabled:
-#         _logger.warning("Transcripts disabled for video %s", video_id)
-#     except NoTranscriptFound:
-#         _logger.warning("No transcript found for video %s in language 'en'", video_id)
-#     except VideoUnavailable:
-#         _logger.warning("Video %s is unavailable, deleted, or private", video_id)
-#     except ValueError as ve:
-#         _logger.warning("Invalid video ID %s: %s", video_id, ve)
-#     except Exception as exc:
-#         _logger.warning("YouTube transcript failed for %s: %s", video_id, exc)
-#     return None
-# 
-# 
-# def _convert_youtube(url: str) -> ConversionResult | None:  # DISABLED (OOM mitigation)
-#     try:
-#         video_id = _extract_youtube_video_id(url)
-#     except ValueError:
-#         return None
-# 
-#     oembed = _fetch_youtube_oembed(video_id)
-#     transcript = _fetch_youtube_transcript(video_id)
-# 
-#     lines = ["# YouTube\n"]
-#     title = (oembed or {}).get("title", "")
-#     if title:
-#         lines.append(f"\n## {title}\n")
-# 
-#     author = (oembed or {}).get("author_name", "")
-#     if author:
-#         lines.append(f"\n- **Channel:** {author}\n")
-# 
-#     desc = (oembed or {}).get("description", "")
-#     if desc:
-#         lines.append(f"\n### Description\n{desc}\n")
-# 
-#     if transcript:
-#         lines.append(f"\n### Transcript\n{transcript}\n")
-#     else:
-#         if not title and not desc:
-#             return None
-#         lines.append("\n> No transcript available.\n")
-# 
-#     markdown = "".join(lines)
-#     return _build_result(url, markdown, 0, "text/html", 0.0)
-
-
 def _is_image(ext: str, mime: str) -> bool:
     return ext.lower() in IMAGE_EXTENSIONS or any(mime.startswith(p) for p in IMAGE_MIME_PREFIXES)
 
@@ -193,27 +97,6 @@ class ConverterService:
 
         start = time.perf_counter()
 
-        # try:  # DISABLED (OOM mitigation) — YouTube transcription
-        #     _extract_youtube_video_id(url)
-        # except ValueError:
-        #     pass
-        # else:
-        #     result = _convert_youtube(url)
-        #     if result is not None:
-        #         result = ConversionResult(
-        #             source=result.source,
-        #             markdown=result.markdown,
-        #             char_count=result.char_count,
-        #             word_count=result.word_count,
-        #             line_count=result.line_count,
-        #             duration_ms=(time.perf_counter() - start) * 1000,
-        #             file_size_bytes=result.file_size_bytes,
-        #             mime_type=result.mime_type,
-        #             content_hash=result.content_hash,
-        #         )
-        #         return result
-        #     _logger.warning("YouTube conversion returned None, falling back to markitdown: %s", url)
-
         try:
             url_ext = Path(urlparse(url).path).suffix.lower()
             if url_ext in IMAGE_EXTENSIONS:

app/services/dataset_metadata_service.py

@@ -614,23 +614,4 @@ async def extract_metadata(
     return result
 
 
-# ---------------------------------------------------------------------------
-# CLI convenience
-# ---------------------------------------------------------------------------
-
-# if __name__ == "__main__":
-#     import json
-#     import sys
-
-#     async def _main() -> None:
-#         if len(sys.argv) < 2:
-#             print("Usage: python solution.py <file_path_or_url>")
-#             sys.exit(1)
-
-#         arg = sys.argv[1]
-#         src: FileSource = arg if _is_url(arg) else Path(arg)  # type: ignore[assignment]
-
-#         result = await extract_metadata(src)
-#         print(json.dumps(result, indent=2, default=str))
 
-#     asyncio.run(_main())

app/services/embeddings_service.py

@@ -5,25 +5,15 @@ import os
 from typing import Dict, List, Optional
 
 import numpy as np
-# import torch  # DISABLED (OOM mitigation) — only used by vision
-# import torch.nn.functional as F  # DISABLED (OOM mitigation)
-# from PIL import Image  # DISABLED (OOM mitigation)
 from sentence_transformers import SentenceTransformer
-# from transformers import AutoImageProcessor, AutoModel  # DISABLED (OOM mitigation)
 
 _logger = logging.getLogger(__name__)
 
 # Only 384-dim embedding is enabled. 768 and 1024 are disabled to reduce memory usage.
 _MODEL_MAP: Dict[int, str] = {
     384: "ibm-granite/granite-embedding-small-english-r2",
-    # 768: "nomic-ai/nomic-embed-text-v1.5",   # DISABLED (OOM mitigation)
-    # 1024: "lightonai/modernbert-embed-large", # DISABLED (OOM mitigation)
 }
 
-# _VISION_MODEL_NAME = "nomic-ai/nomic-embed-vision-v1.5"  # DISABLED (OOM mitigation)
-# _VISION_DIMENSION = 768
-
-
 class EmbeddingService:
     def __init__(self, models_dir: Optional[str] = None) -> None:
         self._models: Dict[int, SentenceTransformer] = {}
@@ -37,9 +27,6 @@ class EmbeddingService:
             self._device = "cpu"
 
         self._loaded_dimensions: List[int] = []
-        # self._vision_processor: Optional[AutoImageProcessor] = None  # DISABLED (OOM mitigation)
-        # self._vision_model: Optional[AutoModel] = None              # DISABLED (OOM mitigation)
-        # self._vision_loaded = False
 
     def load_model(self, dimension: int) -> None:
         if dimension in self._models:
@@ -65,35 +52,6 @@ class EmbeddingService:
         for dim in _MODEL_MAP:
             self.load_model(dim)
 
-    # def load_vision_model(self) -> None:  # DISABLED (OOM mitigation)
-    #     if self._vision_loaded:
-    #         return
-    #     local_path = os.path.join(self._models_dir, "vision")
-    #     source = local_path if os.path.isdir(local_path) else _VISION_MODEL_NAME
-    # 
-    #     cfg_path = os.path.join(local_path, "config.json")
-    #     if os.path.exists(cfg_path):
-    #         import json
-    #         with open(cfg_path) as f:
-    #             d = json.load(f)
-    #         if isinstance(d.get("n_inner"), float):
-    #             d["n_inner"] = int(d["n_inner"])
-    #             with open(cfg_path, "w") as f:
-    #                 json.dump(d, f, indent=2)
-    #             _logger.info("Patched vision model config: n_inner float -> int")
-    # 
-    #     _logger.info("Loading vision embedding model from %s", source)
-    #     self._vision_processor = AutoImageProcessor.from_pretrained(source)
-    #     self._vision_model = AutoModel.from_pretrained(
-    #         source,
-    #         trust_remote_code=True,
-    #         _fast_init=False,
-    #     )
-    #     self._vision_model.eval()
-    #     self._vision_model.to(self._device)
-    #     self._vision_loaded = True
-    #     _logger.info("Loaded vision embedding model (device=%s)", self._device)
-
     def generate_embedding(self, text: List[str], dimension: int) -> List[List[float]]:
         if dimension not in self._models:
             raise ValueError(f"Model for dimension {dimension} not loaded")
@@ -109,20 +67,6 @@ class EmbeddingService:
         )
         return result.tolist()
 
-    # def generate_image_embedding(self, images: List[Image.Image]) -> List[List[float]]:  # DISABLED (OOM mitigation)
-    #     if not self._vision_loaded or self._vision_model is None or self._vision_processor is None:
-    #         raise ValueError("Vision model not loaded")
-    #     all_embeddings: List[List[float]] = []
-    #     with torch.no_grad():
-    #         for image in images:
-    #             inputs = self._vision_processor(image, return_tensors="pt")
-    #             inputs = {k: v.to(self._device) for k, v in inputs.items()}
-    #             outputs = self._vision_model(**inputs)
-    #             emb = outputs.last_hidden_state[:, 0]
-    #             emb = F.normalize(emb, p=2, dim=1)
-    #             all_embeddings.append(emb.cpu().numpy().flatten().tolist())
-    #     return all_embeddings
-
     @property
     def loaded_dimensions(self) -> List[int]:
         return list(self._loaded_dimensions)
@@ -130,6 +74,3 @@ class EmbeddingService:
     def is_loaded(self, dimension: int) -> bool:
         return dimension in self._models
 
-    # @property  # DISABLED (OOM mitigation)
-    # def vision_dimension(self) -> int:
-    #     return _VISION_DIMENSION

app/services/vector_store_service.py

@@ -522,3 +522,6 @@ class VectorStoreService:
         self._collections.clear()
         self._stores.clear()
         self._thread_pool.shutdown(wait=True)
+        self._thread_pool = concurrent.futures.ThreadPoolExecutor(
+            max_workers=_MAX_WORKERS, thread_name_prefix="zvec"
+        )

tests/test_webhook_socket.py

@@ -20,10 +20,6 @@ AUTH_HEADER = {"Authorization": f"Bearer {API_KEY}"}
 BASE = "http://localhost:7860/api/v1"
 
 
-# ---------------------------------------------------------------------------
-#  Unit tests for ChannelManager
-# ---------------------------------------------------------------------------
-
 class TestChannelManager:
     def test_create_channel(self):
         mgr = ChannelManager()
@@ -61,7 +57,6 @@ class TestChannelManager:
         mgr = ChannelManager()
         mgr.create_channel(channel_id="dup")
         ch2 = mgr.create_channel(channel_id="dup")
-        # Should overwrite
         assert mgr.get_channel("dup") is ch2
 
     def test_default_buffer_from_manager(self):
@@ -113,10 +108,43 @@ class TestChannelManager:
         sig2 = sign_payload(secret, body)
         assert sig1 == sig2
 
+    def test_channel_info_fields(self):
+        mgr = ChannelManager()
+        mgr.create_channel(channel_id="info-test", buffer_size=5)
+        ch = mgr.get_channel("info-test")
+        assert ch.channel_id == "info-test"
+        assert ch.buffer_size == 5
+        assert ch.message_count == 0
+        assert ch.created_at > 0
+        assert ch.last_activity > 0
+
+    def test_publish_with_subscriber(self):
+        mgr = ChannelManager()
+        mgr.create_channel(channel_id="sub-test")
+        ch = mgr.get_channel("sub-test")
+
+        async def dummy():
+            return ch.channel_id
+        q = asyncio.Queue()
+        ch.subscribers[dummy] = q
+
+        result = asyncio.run(mgr.publish("sub-test", {"msg": "hello"}))
+        assert result == 1
+        assert ch.message_count == 1
+
+    def test_multiple_publishes(self):
+        mgr = ChannelManager()
+        ch = mgr.create_channel(channel_id="multi-pub", buffer_size=10)
+        for i in range(5):
+            asyncio.run(mgr.publish("multi-pub", {"n": i}))
+        assert ch.message_count == 5
+        assert len(ch.history) == 5
+
+    def test_channel_manager_defaults(self):
+        mgr = ChannelManager()
+        assert mgr.default_buffer == 0
+        assert mgr.channels == {}
 
-# ---------------------------------------------------------------------------
-#  Integration tests (requires running server @ localhost:7860)
-# ---------------------------------------------------------------------------
 
 pytestmark_integration = pytest.mark.skipif(
     not os.environ.get("RUN_INTEGRATION_TESTS"),
@@ -169,7 +197,7 @@ class TestWebhookSocketIntegration:
 
     async def test_create_channel_no_auth(self):
         resp = await self.client.post("/channels", json={})
-        assert resp.status_code == 403
+        assert resp.status_code in (401, 403)
 
     async def test_create_duplicate_channel(self):
         await self._create_channel(channel_id="dup-test")
@@ -405,10 +433,6 @@ class TestWebhookSocketIntegration:
         assert not_found.status_code == 404
 
 
-# ---------------------------------------------------------------------------
-#  Runner (standalone)
-# ---------------------------------------------------------------------------
-
 if __name__ == "__main__":
     import subprocess
     import sys as _sys