ok

app/api/deps.py

@@ -9,9 +9,14 @@ from app.services.database_service import DatabaseService
 from app.services.embeddings_service import EmbeddingService
 from app.services.extraction_service import ExtractionService
 from app.services.ocr_service import OCRService
+from app.services.text_cleaner_service import TextCleanerService
 from app.services.web_search_service import WebSearchService
 
 
+def get_text_cleaner_service() -> TextCleanerService:
+    return TextCleanerService()
+
+
 def get_auth_service() -> AuthService:
     return AuthService()
 

app/api/v1/convert.py

@@ -15,13 +15,15 @@ from app.config import get_settings
 from app.api.deps import (
     get_converter_service,
     get_extraction_service,
+    get_text_cleaner_service,
     require_auth,
 )
 from app.core.logger import get_logger
-from app.models.domain import ConversionError
+from app.models.domain import ConversionError, count_tokens
 from app.models.schemas import ConversionMetadata, ConversionResponse, UrlRequest
 from app.services.converter_service import ConverterService
 from app.services.extraction_service import ExtractionService
+from app.services.text_cleaner_service import TextCleanerService
 
 router = APIRouter()
 _logger = get_logger(__name__)
@@ -52,17 +54,24 @@ async def _build_response(
     raw_data: Optional[bytes] = None,
     mappings: Optional[Dict[str, Dict[str, Any]]] = None,
     extraction_service: ExtractionService = None,
+    clean_content: bool = False,
+    text_cleaner_service: TextCleanerService = None,
 ) -> ConversionResponse:
     json_content: Optional[Any] = None
     error_message: Optional[str] = None
 
+    content = result.markdown
+    if clean_content and text_cleaner_service:
+        loop = asyncio.get_running_loop()
+        content = await loop.run_in_executor(_thread_pool, text_cleaner_service.clean, content)
+
     if return_json and filename and extraction_service:
         loop = asyncio.get_running_loop()
         json_result = await loop.run_in_executor(
             _thread_pool,
             extraction_service.extract_structured,
             filename,
-            result.markdown,
+            content,
             mappings,
             raw_data,
         )
@@ -71,13 +80,30 @@ async def _build_response(
         else:
             json_content = json_result
 
+    if clean_content and content != result.markdown:
+        lines = content.splitlines()
+        words = content.split()
+        import hashlib
+        metadata = ConversionMetadata(
+            source=result.source,
+            char_count=len(content),
+            word_count=len(words),
+            line_count=len(lines),
+            file_size_bytes=result.file_size_bytes,
+            mime_type=result.mime_type,
+            content_hash=hashlib.sha256(content.encode()).hexdigest(),
+            token_estimate=max(1, count_tokens(content)),
+        )
+    else:
+        metadata = _build_metadata(result)
+
     return ConversionResponse(
         success=True,
         time_ms=round(result.duration_ms, 3),
-        content=result.markdown,
+        content=content,
         return_json=return_json,
         json_content=json_content,
-        metadata=_build_metadata(result),
+        metadata=metadata,
         error_message=error_message,
     )
 
@@ -109,10 +135,12 @@ async def convert_file(
     file: Annotated[UploadFile, File(description="File to convert")],
     plain_text: bool = Form(False),
     return_json: bool = Form(False),
+    clean_content: bool = Form(False),
     mappings: Optional[str] = Form(None, description="JSON string with field mappings"),
     token: str = Depends(require_auth),
     converter_service: ConverterService = Depends(get_converter_service),
     extraction_service: ExtractionService = Depends(get_extraction_service),
+    text_cleaner_service: TextCleanerService = Depends(get_text_cleaner_service),
 ):
     if file is None:
         raise HTTPException(status_code=400, detail={"success": False, "message": "No file provided."})
@@ -137,9 +165,14 @@ async def convert_file(
         _raise_for_error(outcome)
 
     _logger.info("Conversion successful for %s", file.filename)
+
+    content = outcome.markdown
+    if clean_content:
+        content = await loop.run_in_executor(_thread_pool, text_cleaner_service.clean, content)
+
     if plain_text:
         from fastapi.responses import PlainTextResponse
-        return PlainTextResponse(outcome.markdown)
+        return PlainTextResponse(content)
 
     response = await _build_response(
         outcome,
@@ -148,6 +181,8 @@ async def convert_file(
         raw_data=raw,
         mappings=parsed_mappings,
         extraction_service=extraction_service,
+        clean_content=clean_content,
+        text_cleaner_service=text_cleaner_service,
     )
     _logger.info("Request completed for %s", file.filename)
     return response

app/models/domain.py

@@ -1,8 +1,23 @@
 from __future__ import annotations
 
-import hashlib
 from dataclasses import dataclass, field
-from typing import Any, Dict, Optional
+
+import tiktoken
+
+
+_ENCODING_CACHE: dict[str, tiktoken.Encoding] = {}
+
+
+def _get_encoding(name: str = "o200k_base") -> tiktoken.Encoding:
+    if name not in _ENCODING_CACHE:
+        _ENCODING_CACHE[name] = tiktoken.get_encoding(name)
+    return _ENCODING_CACHE[name]
+
+
+def count_tokens(text: str, encoding_name: str = "o200k_base") -> int:
+    if not text:
+        return 0
+    return len(_get_encoding(encoding_name).encode(text, disallowed_special=()))
 
 
 @dataclass(frozen=True)
@@ -20,7 +35,7 @@ class ConversionResult:
 
     @property
     def token_estimate(self) -> int:
-        return max(1, self.word_count * 4 // 3)
+        return max(1, count_tokens(self.markdown))
 
 
 @dataclass(frozen=True)

app/services/text_cleaner_service.py

@@ -0,0 +1,154 @@
+import re
+import html
+from cleantext import clean
+from text_unidecode import unidecode
+
+
+def clean_text(
+    text: str,
+    *,
+    # Basic cleanup
+    normalize_whitespace: bool = True,
+    remove_newlines: bool = True,
+    strip: bool = True,
+    # Character handling
+    to_lowercase: bool = False,
+    remove_punctuation: bool = False,
+    # Content removal
+    remove_urls: bool = False,
+    remove_emails: bool = False,
+    remove_phone_numbers: bool = False,
+    remove_numbers: bool = False,
+    remove_digits: bool = False,
+    # Escape sequences
+    fix_escape_sequences: bool = False,
+    remove_html_entities: bool = True,
+    # Special characters
+    remove_currency_symbols: bool = True,
+    remove_emoji: bool = True,
+    normalize_unicode: bool = True,
+    # Custom replacements
+    custom_replacements: dict | None = None,
+) -> str:
+    """
+    Robust generic string cleaner using validated cleantext API parameters 
+    and text_unidecode for superior ASCII transliteration.
+
+    Parameters
+    ----------
+    text                  : Input string to clean.
+    normalize_whitespace  : Normalize all whitespace variants to single space.
+    remove_newlines       : Remove line breaks (\\n, \\r, etc.).
+    strip                 : Strip leading/trailing whitespace.
+    to_lowercase          : Convert text to lowercase.
+    remove_punctuation    : Remove punctuation characters.
+    remove_urls           : Remove URLs (http/https/www).
+    remove_emails         : Remove email addresses.
+    remove_phone_numbers  : Remove phone numbers.
+    remove_numbers        : Remove standalone numbers.
+    remove_digits         : Remove all digit characters.
+    fix_escape_sequences  : Fix literal escape sequences (\\\\n, \\\\t, etc.).
+    remove_html_entities  : Decode HTML entities (& → &).
+    remove_currency_symbols: Remove $, €, £, ¥, etc.
+    remove_emoji          : Remove emoji characters.
+    normalize_unicode     : Apply unicode fix + transliterate to ASCII via unidecode.
+    custom_replacements   : Dict of {exact_string: replacement} applied first.
+
+    Returns
+    -------
+    str : Cleaned string.
+    """
+
+    # ── Guard: handle None / non-string input safely ─────────────────────────
+    if text is None:
+        return ""
+    if not isinstance(text, str):
+        text = str(text)
+    if not text.strip():
+        return ""
+
+    # ────────────────────────────────────────────────────────────────────────
+    # STEP 1 ── Custom replacements (exact string match, applied first)
+    # ────────────────────────────────────────────────────────────────────────
+    if custom_replacements:
+        for target, replacement in custom_replacements.items():
+            text = text.replace(target, replacement)
+
+    # ────────────────────────────────────────────────────────────────────────
+    # STEP 2 ── Fix literal escape sequences BEFORE any other processing
+    # ────────────────────────────────────────────────────────────────────────
+    if fix_escape_sequences:
+        LITERAL_ESCAPE_MAP = [
+            ("\\n",  " "),   
+            ("\\t",  " "),   
+            ("\\r",  " "),   
+            ("\\v",  " "),   
+            ("\\f",  " "),   
+            ("\\a",  ""),    
+            ("\\b",  ""),    
+            ("\\\\", " "),   
+            ("\\/",  "/"),   
+            ("\\'",  "'"),   
+            ('\\"',  '"'),   
+        ]
+        for literal, replacement in LITERAL_ESCAPE_MAP:
+            text = text.replace(literal, replacement)
+
+    # ────────────────────────────────────────────────────────────────────────
+    # STEP 3 ── Decode HTML entities  (&amp; → &,  &lt; → <,  &#39; → ')
+    # ────────────────────────────────────────────────────────────────────────
+    if remove_html_entities:
+        text = html.unescape(text)
+
+    # ────────────────────────────────────────────────────────────────────────
+    # STEP 4 ── Core cleaning via cleantext (validated API parameters only)
+    #           We disable cleantext's to_ascii because text_unidecode 
+    #           handles transliteration far more robustly in STEP 5.
+    # ────────────────────────────────────────────────────────────────────────
+    text = clean(
+        text,
+        fix_unicode=True,                 # Fix mojibake/encoding errors
+        to_ascii=False,                   # Defer to text_unidecode
+        lower=to_lowercase,
+        normalize_whitespace=normalize_whitespace,
+        no_line_breaks=remove_newlines,
+        strip_lines=strip,
+        no_urls=remove_urls,
+        no_emails=remove_emails,
+        no_phone_numbers=remove_phone_numbers,
+        no_numbers=remove_numbers,
+        no_digits=remove_digits,
+        no_currency_symbols=remove_currency_symbols,
+        no_punct=remove_punctuation,
+        no_emoji=remove_emoji,
+        replace_with_url="",
+        replace_with_email="",
+        replace_with_phone_number="",
+        replace_with_number="",
+        replace_with_digit="",
+        replace_with_currency_symbol="",
+        replace_with_punct="",
+        lang="en",
+    )
+
+    # ────────────────────────────────────────────────────────────────────────
+    # STEP 5 ── Robust ASCII transliteration via text_unidecode
+    #           Converts remaining non-ASCII (accents, cyrillic, greek, etc.)
+    # ────────────────────────────────────────────────────────────────────────
+    if normalize_unicode:
+        text = unidecode(text)
+
+    # ────────────────────────────────────────────────────────────────────────
+    # STEP 6 ── Post-clean whitespace tidy-up
+    #           Removal + transliteration may leave stray multi-spaces
+    # ────────────────────────────────────────────────────────────────────────
+    text = re.sub(r" {2,}", " ", text)   
+    if strip:
+        text = text.strip()
+
+    return text
+
+
+class TextCleanerService:
+    def clean(self, text: str, **kwargs) -> str:
+        return clean_text(text, **kwargs)

requirements.txt

@@ -23,6 +23,7 @@ torch==2.12.1
 einops
 spacy>=3.7.0
 phonenumbers>=8.13.0
+tiktoken>=0.9.0
 
 # Async database drivers
 aiomysql>=0.3.2
@@ -35,4 +36,8 @@ scrapling[all]>=0.4.0
 # Chat / AI
 redis>=5.0.0
 defusedxml>=0.7.1
-chardet>=5.2.0
+chardet>=5.2.0
+
+# Text processing
+clean-text>=0.6.0
+Unidecode>=1.3.8