refactor(core): overhaul architecture for better performance, efficiency, and maintainability

.dockerignore

@@ -0,0 +1,52 @@
+# VCS / IDE
+.git
+.gitignore
+.github
+.gitattributes
+.idea
+.vscode
+.editorconfig
+.windsurf
+.cursor
+
+# Python
+__pycache__
+*.py[cod]
+*.so
+*.egg
+*.egg-info
+.Python
+.venv
+venv
+env
+.tox
+.nox
+.pytest_cache
+.mypy_cache
+.ruff_cache
+.coverage
+.coverage.*
+htmlcov
+dist
+build
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Project
+tests
+docs
+.env
+.env.*
+!.env.example
+*.log
+*.local
+*.bak
+.DS_Store
+
+# Tooling / docs that don't belong in the runtime image
+Makefile
+.pre-commit-config.yaml
+ARCHITECTURE.md
+CONTRIBUTING.md
+LICENSE
+poetry.lock

.env.example

@@ -0,0 +1,62 @@
+# =============================================================================
+# Docsifer environment configuration
+# Copy to ".env" and adjust for your environment.
+# =============================================================================
+
+# General
+DOCSIFER_ENVIRONMENT=production
+DOCSIFER_LOG_LEVEL=INFO
+DOCSIFER_LOG_JSON=true
+
+# HTTP / API
+DOCSIFER_CORS_ORIGINS=*
+DOCSIFER_CORS_ALLOW_CREDENTIALS=false
+DOCSIFER_REQUEST_TIMEOUT_SEC=55
+DOCSIFER_MAX_UPLOAD_BYTES=10485760
+DOCSIFER_GZIP_MIN_SIZE=1024
+DOCSIFER_ENABLE_SECURITY_HEADERS=true
+
+# Concurrency / resources (tune to your hardware)
+DOCSIFER_MAX_CONCURRENT_CONVERSIONS=2
+DOCSIFER_MAX_QUEUE_DEPTH=10
+DOCSIFER_MAX_PER_IP_CONCURRENT=1
+DOCSIFER_WORKER_POOL_SIZE=4
+DOCSIFER_MIN_FREE_MEMORY_MB=512
+DOCSIFER_MIN_FREE_DISK_MB=256
+DOCSIFER_MEMORY_WATCHDOG_PCT=90
+DOCSIFER_ENABLE_MEMORY_WATCHDOG=false
+DOCSIFER_DISK_CLEANUP_INTERVAL_SEC=600
+DOCSIFER_DISK_CLEANUP_TTL_SEC=3600
+
+# Conversion
+DOCSIFER_TOKEN_MODEL=gpt-4o
+DOCSIFER_DEFAULT_OPENAI_BASE_URL=https://api.openai.com/v1
+DOCSIFER_DEFAULT_OPENAI_MODEL=gpt-4o-mini
+DOCSIFER_OPENAI_REQUEST_TIMEOUT_SEC=60
+DOCSIFER_OPENAI_CONNECT_TIMEOUT_SEC=10
+DOCSIFER_OPENAI_MAX_RETRIES=2
+DOCSIFER_LLM_CACHE_MAX_SIZE=16
+DOCSIFER_LLM_CACHE_TTL_SEC=600
+
+# SSRF guard
+DOCSIFER_URL_ALLOW_PRIVATE_NETWORKS=false
+DOCSIFER_URL_ALLOWED_SCHEMES=http,https
+
+# Analytics / Redis (Upstash recommended; leave empty for in-memory mode)
+DOCSIFER_REDIS_URL=
+DOCSIFER_REDIS_TOKEN=
+DOCSIFER_ANALYTICS_ENABLED=true
+DOCSIFER_ANALYTICS_SYNC_INTERVAL_SEC=1800
+DOCSIFER_ANALYTICS_LABEL=docsifer
+
+# Quotas
+DOCSIFER_QUOTA_ENABLED=false
+DOCSIFER_QUOTA_ANON_RPH=10
+DOCSIFER_QUOTA_ANON_RPD=50
+DOCSIFER_QUOTA_BYOK_RPH=60
+DOCSIFER_QUOTA_BYOK_RPD=500
+DOCSIFER_QUOTA_AUTH_KEYS=
+
+# Observability
+DOCSIFER_ENABLE_PROMETHEUS=false
+DOCSIFER_SENTRY_DSN=

.github/workflows/ci.yml

@@ -0,0 +1,63 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+concurrency:
+  group: ci-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  lint-test:
+    name: Lint, type-check, test
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+
+      - name: Install system deps
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libmagic1 ffmpeg
+
+      - name: Install Python deps
+        run: |
+          python -m pip install --upgrade pip wheel
+          pip install -r requirements.txt
+          pip install pytest pytest-asyncio ruff mypy fakeredis
+
+      - name: Ruff lint
+        run: ruff check .
+
+      - name: Ruff format check
+        run: ruff format --check .
+
+      - name: Mypy
+        run: mypy
+        continue-on-error: true  # tighten over time
+
+      - name: Pytest
+        env:
+          DOCSIFER_LOG_JSON: "false"
+        run: pytest -q
+
+  docker-build:
+    name: Docker build
+    runs-on: ubuntu-latest
+    needs: lint-test
+    steps:
+      - uses: actions/checkout@v4
+      - name: Build image
+        run: docker build -t docsifer:ci .

.pre-commit-config.yaml

@@ -0,0 +1,28 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.6.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-added-large-files
+        args: [--maxkb=1024]
+      - id: check-merge-conflict
+      - id: detect-private-key
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.6.9
+    hooks:
+      - id: ruff
+        args: [--fix]
+      - id: ruff-format
+
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.11.2
+    hooks:
+      - id: mypy
+        additional_dependencies:
+          - pydantic>=2.6
+          - pydantic-settings>=2.2
+        args: [--config-file=pyproject.toml]
+        exclude: ^tests/

.pylintrc

@@ -1,579 +0,0 @@
-[MAIN]
-
-# Specify a configuration file.
-#rcfile=
-
-# Python code to execute, usually for sys.path manipulation such as
-# pygtk.require().
-#init-hook=
-
-# Files or directories to be skipped. They should be base names, not
-# paths.
-ignore=CVS
-
-# Add files or directories matching the regex patterns to the ignore-list. The
-# regex matches against paths and can be in Posix or Windows format.
-ignore-paths=
-
-# Files or directories matching the regex patterns are skipped. The regex
-# matches against base names, not paths.
-ignore-patterns=^\.#
-
-# Pickle collected data for later comparisons.
-persistent=yes
-
-# List of plugins (as comma separated values of python modules names) to load,
-# usually to register additional checkers.
-load-plugins=
-    pylint.extensions.check_elif,
-    pylint.extensions.bad_builtin,
-    pylint.extensions.docparams,
-    pylint.extensions.for_any_all,
-    pylint.extensions.set_membership,
-    pylint.extensions.code_style,
-    pylint.extensions.overlapping_exceptions,
-    pylint.extensions.typing,
-    pylint.extensions.redefined_variable_type,
-    pylint.extensions.comparison_placement,
-    pylint.extensions.mccabe,
-
-# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
-# number of processors available to use.
-jobs=0
-
-# When enabled, pylint would attempt to guess common misconfiguration and emit
-# user-friendly hints instead of false-positive error messages.
-suggestion-mode=yes
-
-# Allow loading of arbitrary C extensions. Extensions are imported into the
-# active Python interpreter and may run arbitrary code.
-unsafe-load-any-extension=no
-
-# A comma-separated list of package or module names from where C extensions may
-# be loaded. Extensions are loading into the active Python interpreter and may
-# run arbitrary code
-extension-pkg-allow-list=
-
-# Minimum supported python version
-py-version = 3.7.2
-
-# Control the amount of potential inferred values when inferring a single
-# object. This can help the performance when dealing with large functions or
-# complex, nested conditions.
-limit-inference-results=100
-
-# Specify a score threshold to be exceeded before program exits with error.
-fail-under=10.0
-
-# Return non-zero exit code if any of these messages/categories are detected,
-# even if score is above --fail-under value. Syntax same as enable. Messages
-# specified are enabled, while categories only check already-enabled messages.
-fail-on=
-
-
-[MESSAGES CONTROL]
-
-# Only show warnings with the listed confidence levels. Leave empty to show
-# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED
-# confidence=
-
-# Enable the message, report, category or checker with the given id(s). You can
-# either give multiple identifier separated by comma (,) or put this option
-# multiple time (only on the command line, not in the configuration file where
-# it should appear only once). See also the "--disable" option for examples.
-enable=
-    use-symbolic-message-instead,
-    useless-suppression,
-
-# Disable the message, report, category or checker with the given id(s). You
-# can either give multiple identifiers separated by comma (,) or put this
-# option multiple times (only on the command line, not in the configuration
-# file where it should appear only once).You can also use "--disable=all" to
-# disable everything first and then re-enable specific checks. For example, if
-# you want to run only the similarities checker, you can use "--disable=all
-# --enable=similarities". If you want to run only the classes checker, but have
-# no Warning level messages displayed, use"--disable=all --enable=classes
-# --disable=W"
-
-disable=
-    attribute-defined-outside-init,
-    invalid-name,
-    missing-docstring,
-    protected-access,
-    too-few-public-methods,
-    # handled by black
-    format,
-    # We anticipate #3512 where it will become optional
-    fixme,
-    cyclic-import,
-    import-error,
-    # 
-    unnecessary-pass,
-    unrecognized-option,
-    cell-var-from-loop,
-    no-member,
-    wrong-import-order,
-    raise-missing-from,
-    consider-using-f-string
-
-
-[REPORTS]
-
-# Set the output format. Available formats are text, parseable, colorized, msvs
-# (visual studio) and html. You can also give a reporter class, eg
-# mypackage.mymodule.MyReporterClass.
-output-format=text
-
-# Tells whether to display a full report or only the messages
-reports=no
-
-# Python expression which should return a note less than 10 (10 is the highest
-# note). You have access to the variables 'fatal', 'error', 'warning', 'refactor', 'convention'
-# and 'info', which contain the number of messages in each category, as
-# well as 'statement', which is the total number of statements analyzed. This
-# score is used by the global evaluation report (RP0004).
-evaluation=max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10))
-
-# Template used to display messages. This is a python new-style format string
-# used to format the message information. See doc for all details
-#msg-template=
-
-# Activate the evaluation score.
-score=yes
-
-
-[LOGGING]
-
-# Logging modules to check that the string format arguments are in logging
-# function parameter format
-logging-modules=logging
-
-# The type of string formatting that logging methods do. `old` means using %
-# formatting, `new` is for `{}` formatting.
-logging-format-style=old
-
-
-[MISCELLANEOUS]
-
-# List of note tags to take in consideration, separated by a comma.
-notes=FIXME,XXX,TODO
-
-# Regular expression of note tags to take in consideration.
-#notes-rgx=
-
-
-[SIMILARITIES]
-
-# Minimum lines number of a similarity.
-min-similarity-lines=6
-
-# Ignore comments when computing similarities.
-ignore-comments=yes
-
-# Ignore docstrings when computing similarities.
-ignore-docstrings=yes
-
-# Ignore imports when computing similarities.
-ignore-imports=yes
-
-# Signatures are removed from the similarity computation
-ignore-signatures=yes
-
-
-[VARIABLES]
-
-# Tells whether we should check for unused import in __init__ files.
-init-import=no
-
-# A regular expression matching the name of dummy variables (i.e. expectedly
-# not used).
-dummy-variables-rgx=_$|dummy
-
-# List of additional names supposed to be defined in builtins. Remember that
-# you should avoid defining new builtins when possible.
-additional-builtins=
-
-# List of strings which can identify a callback function by name. A callback
-# name must start or end with one of those strings.
-callbacks=cb_,_cb
-
-# Tells whether unused global variables should be treated as a violation.
-allow-global-unused-variables=yes
-
-# List of names allowed to shadow builtins
-allowed-redefined-builtins=
-
-# Argument names that match this expression will be ignored. Default to name
-# with leading underscore.
-ignored-argument-names=_.*
-
-# List of qualified module names which can have objects that can redefine
-# builtins.
-redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io
-
-
-[FORMAT]
-
-# Maximum number of characters on a single line.
-max-line-length=120
-
-# Regexp for a line that is allowed to be longer than the limit.
-ignore-long-lines=^\s*(# )?<?https?://\S+>?$
-
-# Allow the body of an if to be on the same line as the test if there is no
-# else.
-single-line-if-stmt=no
-
-# Allow the body of a class to be on the same line as the declaration if body
-# contains single statement.
-single-line-class-stmt=no
-
-# Maximum number of lines in a module
-max-module-lines=1000
-
-# String used as indentation unit. This is usually "    " (4 spaces) or "\t" (1
-# tab).
-indent-string='    '
-
-# Number of spaces of indent required inside a hanging or continued line.
-indent-after-paren=4
-
-# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
-expected-line-ending-format=
-
-
-[BASIC]
-
-# Good variable names which should always be accepted, separated by a comma
-good-names=i,j,k,ex,Run,_
-
-# Good variable names regexes, separated by a comma. If names match any regex,
-# they will always be accepted
-good-names-rgxs=
-
-# Bad variable names which should always be refused, separated by a comma
-bad-names=foo,bar,baz,toto,tutu,tata
-
-# Bad variable names regexes, separated by a comma. If names match any regex,
-# they will always be refused
-bad-names-rgxs=
-
-# Colon-delimited sets of names that determine each other's naming style when
-# the name regexes allow several styles.
-name-group=
-
-# Include a hint for the correct naming format with invalid-name
-include-naming-hint=no
-
-# Naming style matching correct function names.
-function-naming-style=snake_case
-
-# Regular expression matching correct function names
-function-rgx=[a-z_][a-z0-9_]{2,30}$
-
-# Naming style matching correct variable names.
-variable-naming-style=snake_case
-
-# Regular expression matching correct variable names
-variable-rgx=[a-z_][a-z0-9_]{2,30}$
-
-# Naming style matching correct constant names.
-const-naming-style=UPPER_CASE
-
-# Regular expression matching correct constant names
-const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$
-
-# Naming style matching correct attribute names.
-attr-naming-style=snake_case
-
-# Regular expression matching correct attribute names
-attr-rgx=[a-z_][a-z0-9_]{2,}$
-
-# Naming style matching correct argument names.
-argument-naming-style=snake_case
-
-# Regular expression matching correct argument names
-argument-rgx=[a-z_][a-z0-9_]{2,30}$
-
-# Naming style matching correct class attribute names.
-class-attribute-naming-style=any
-
-# Regular expression matching correct class attribute names
-class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
-
-# Naming style matching correct class constant names.
-class-const-naming-style=UPPER_CASE
-
-# Regular expression matching correct class constant names. Overrides class-
-# const-naming-style.
-#class-const-rgx=
-
-# Naming style matching correct inline iteration names.
-inlinevar-naming-style=any
-
-# Regular expression matching correct inline iteration names
-inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$
-
-# Naming style matching correct class names.
-class-naming-style=PascalCase
-
-# Regular expression matching correct class names
-class-rgx=[A-Z_][a-zA-Z0-9]+$
-
-
-# Naming style matching correct module names.
-module-naming-style=snake_case
-
-# Regular expression matching correct module names
-module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
-
-
-# Naming style matching correct method names.
-method-naming-style=snake_case
-
-# Regular expression matching correct method names
-method-rgx=[a-z_][a-z0-9_]{2,}$
-
-# Regular expression which can overwrite the naming style set by typevar-naming-style.
-#typevar-rgx=
-
-# Regular expression which should only match function or class names that do
-# not require a docstring. Use ^(?!__init__$)_ to also check __init__.
-no-docstring-rgx=__.*__
-
-# Minimum line length for functions/classes that require docstrings, shorter
-# ones are exempt.
-docstring-min-length=-1
-
-# List of decorators that define properties, such as abc.abstractproperty.
-property-classes=abc.abstractproperty
-
-
-[TYPECHECK]
-
-# Regex pattern to define which classes are considered mixins if ignore-mixin-
-# members is set to 'yes'
-mixin-class-rgx=.*MixIn
-
-# List of module names for which member attributes should not be checked
-# (useful for modules/projects where namespaces are manipulated during runtime
-# and thus existing member attributes cannot be deduced by static analysis). It
-# supports qualified module names, as well as Unix pattern matching.
-ignored-modules=
-
-# List of class names for which member attributes should not be checked (useful
-# for classes with dynamically set attributes). This supports the use of
-# qualified names.
-ignored-classes=SQLObject, optparse.Values, thread._local, _thread._local
-
-# List of members which are set dynamically and missed by pylint inference
-# system, and so shouldn't trigger E1101 when accessed. Python regular
-# expressions are accepted.
-generated-members=REQUEST,acl_users,aq_parent,argparse.Namespace
-
-# List of decorators that create context managers from functions, such as
-# contextlib.contextmanager.
-contextmanager-decorators=contextlib.contextmanager
-
-# Tells whether to warn about missing members when the owner of the attribute
-# is inferred to be None.
-ignore-none=yes
-
-# This flag controls whether pylint should warn about no-member and similar
-# checks whenever an opaque object is returned when inferring. The inference
-# can return multiple potential results while evaluating a Python object, but
-# some branches might not be evaluated, which results in partial inference. In
-# that case, it might be useful to still emit no-member and other checks for
-# the rest of the inferred objects.
-ignore-on-opaque-inference=yes
-
-# Show a hint with possible names when a member name was not found. The aspect
-# of finding the hint is based on edit distance.
-missing-member-hint=yes
-
-# The minimum edit distance a name should have in order to be considered a
-# similar match for a missing member name.
-missing-member-hint-distance=1
-
-# The total number of similar names that should be taken in consideration when
-# showing a hint for a missing member.
-missing-member-max-choices=1
-
-[SPELLING]
-
-# Spelling dictionary name. Available dictionaries: none. To make it working
-# install python-enchant package.
-spelling-dict=
-
-# List of comma separated words that should not be checked.
-spelling-ignore-words=
-
-# List of comma separated words that should be considered directives if they
-# appear and the beginning of a comment and should not be checked.
-spelling-ignore-comment-directives=fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy:,pragma:,# noinspection
-
-# A path to a file that contains private dictionary; one word per line.
-spelling-private-dict-file=.pyenchant_pylint_custom_dict.txt
-
-# Tells whether to store unknown words to indicated private dictionary in
-# --spelling-private-dict-file option instead of raising a message.
-spelling-store-unknown-words=no
-
-# Limits count of emitted suggestions for spelling mistakes.
-max-spelling-suggestions=2
-
-
-[DESIGN]
-
-# Maximum number of arguments for function / method
-max-args=10
-
-# Maximum number of locals for function / method body
-max-locals=25
-
-# Maximum number of return / yield for function / method body
-max-returns=11
-
-# Maximum number of branch for function / method body
-max-branches=27
-
-# Maximum number of statements in function / method body
-max-statements=100
-
-# Maximum number of parents for a class (see R0901).
-max-parents=7
-
-# List of qualified class names to ignore when counting class parents (see R0901).
-ignored-parents=
-
-# Maximum number of attributes for a class (see R0902).
-max-attributes=11
-
-# Minimum number of public methods for a class (see R0903).
-min-public-methods=2
-
-# Maximum number of public methods for a class (see R0904).
-max-public-methods=25
-
-# Maximum number of boolean expressions in an if statement (see R0916).
-max-bool-expr=5
-
-# List of regular expressions of class ancestor names to
-# ignore when counting public methods (see R0903).
-exclude-too-few-public-methods=
-
-max-complexity=10
-
-[CLASSES]
-
-# List of method names used to declare (i.e. assign) instance attributes.
-defining-attr-methods=__init__,__new__,setUp,__post_init__
-
-# List of valid names for the first argument in a class method.
-valid-classmethod-first-arg=cls
-
-# List of valid names for the first argument in a metaclass class method.
-valid-metaclass-classmethod-first-arg=mcs
-
-# List of member names, which should be excluded from the protected access
-# warning.
-exclude-protected=_asdict,_fields,_replace,_source,_make
-
-# Warn about protected attribute access inside special methods
-check-protected-access-in-special-methods=no
-
-[IMPORTS]
-
-# List of modules that can be imported at any level, not just the top level
-# one.
-allow-any-import-level=
-
-# Allow wildcard imports from modules that define __all__.
-allow-wildcard-with-all=no
-
-# Analyse import fallback blocks. This can be used to support both Python 2 and
-# 3 compatible code, which means that the block might have code that exists
-# only in one or another interpreter, leading to false positives when analysed.
-analyse-fallback-blocks=no
-
-# Deprecated modules which should not be used, separated by a comma
-deprecated-modules=regsub,TERMIOS,Bastion,rexec
-
-# Create a graph of every (i.e. internal and external) dependencies in the
-# given file (report RP0402 must not be disabled)
-import-graph=
-
-# Create a graph of external dependencies in the given file (report RP0402 must
-# not be disabled)
-ext-import-graph=
-
-# Create a graph of internal dependencies in the given file (report RP0402 must
-# not be disabled)
-int-import-graph=
-
-# Force import order to recognize a module as part of the standard
-# compatibility libraries.
-known-standard-library=
-
-# Force import order to recognize a module as part of a third party library.
-known-third-party=enchant
-
-# Couples of modules and preferred modules, separated by a comma.
-preferred-modules=
-
-
-[EXCEPTIONS]
-
-# Exceptions that will emit a warning when being caught. Defaults to
-# "Exception"
-overgeneral-exceptions=Exception
-
-
-[TYPING]
-
-# Set to ``no`` if the app / library does **NOT** need to support runtime
-# introspection of type annotations. If you use type annotations
-# **exclusively** for type checking of an application, you're probably fine.
-# For libraries, evaluate if some users what to access the type hints at
-# runtime first, e.g., through ``typing.get_type_hints``. Applies to Python
-# versions 3.7 - 3.9
-runtime-typing = no
-
-
-[DEPRECATED_BUILTINS]
-
-# List of builtins function names that should not be used, separated by a comma
-bad-functions=map,input
-
-
-[REFACTORING]
-
-# Maximum number of nested blocks for function / method body
-max-nested-blocks=5
-
-# Complete name of functions that never returns. When checking for
-# inconsistent-return-statements if a never returning function is called then
-# it will be considered as an explicit return statement and no message will be
-# printed.
-never-returning-functions=sys.exit,argparse.parse_error
-
-
-[STRING]
-
-# This flag controls whether inconsistent-quotes generates a warning when the
-# character used as a quote delimiter is used inconsistently within a module.
-check-quote-consistency=no
-
-# This flag controls whether the implicit-str-concat should generate a warning
-# on implicit string concatenation in sequences defined over several lines.
-check-str-concat-over-line-jumps=no
-
-
-[CODE_STYLE]
-
-# Max line length for which to sill emit suggestions. Used to prevent optional
-# suggestions which would get split by a code formatter (e.g., black). Will
-# default to the setting for ``max-line-length``.
-#max-line-length-suggestions=
-
-W0107:unnecessary-pass

ARCHITECTURE.md

@@ -0,0 +1,140 @@
+# Architecture
+
+This document explains how the Docsifer codebase is structured and why.
+
+## Goals
+
+1. **Correctness** — eliminate the bugs identified in the audit (SSRF,
+   path-traversal, broken cookies, race conditions, deprecated FastAPI APIs).
+2. **Production-grade safety** — never crash a free-tier container under load.
+3. **Operability** — config driven, observable, well-tested.
+4. **Performance** — minimize redundant I/O, reuse expensive clients.
+
+## Module map
+
+```
+docsifer/
+├── api/              # HTTP layer
+│   ├── deps.py
+│   ├── error_handlers.py
+│   ├── middleware.py        # request-id, body limit, security headers
+│   ├── schemas.py           # Pydantic request/response models
+│   └── v1/
+│       ├── convert.py       # POST /v1/convert
+│       ├── stats.py         # GET  /v1/stats
+│       └── health.py        # GET  /v1/healthz, /v1/readyz
+├── core/             # Pure business logic
+│   ├── service.py           # DocsiferService
+│   ├── llm_registry.py      # TTL-cached MarkItDown(LLM) instances
+│   ├── html_cleaner.py      # selectolax-based HTML scrub
+│   ├── tokenizer.py         # tiktoken wrapper with fallbacks
+│   ├── mime.py              # safe MIME detection
+│   └── url_guard.py         # SSRF protection
+├── analytics/        # Lifespan-managed analytics
+│   ├── service.py
+│   ├── periods.py           # ISO 8601 week numbering
+│   └── store.py             # AnalyticsStore protocol + Upstash & in-memory
+├── safety/           # Anti-crash primitives (Section N of the audit)
+│   ├── conversion_gate.py
+│   ├── per_ip_limiter.py
+│   ├── resource_guard.py
+│   ├── circuit_breaker.py
+│   ├── memory_watchdog.py
+│   └── disk_cleanup.py
+├── ui/               # Gradio UI (optional)
+│   └── gradio_app.py
+├── config.py         # pydantic-settings
+├── exceptions.py
+├── logging_config.py
+└── main.py           # FastAPI app factory + lifespan
+```
+
+## Request flow (POST /v1/convert)
+
+```
+Client
+  │  multipart/form-data
+  ▼
+[ middleware ]
+  ├─ request_id          → X-Request-ID header + ContextVar
+  ├─ body_limit          → 413 if Content-Length > MAX
+  └─ security_headers    → X-Content-Type-Options, X-Frame-Options, …
+  │
+  ▼
+[ route handler convert.py ]
+  ├─ parse JSON forms via Pydantic (422 on errors)
+  ├─ acquire PerIPLimiter slot (429 on overflow)
+  ├─ acquire ConversionGate slot (503 when full)
+  ├─ stream upload to disk in worker thread
+  ├─ ResourceGuard checks RAM/disk
+  ├─ asyncio.wait_for() guards against runaway conversions
+  └─ DocsiferService.convert_file(...)
+       │
+       ▼
+[ DocsiferService (core) ]
+  ├─ choose converter:
+  │     ├─ no api_key → cached `MarkItDown()`
+  │     └─ otherwise  → LLMRegistry.get(LLMConfig)
+  ├─ HTML path  → clean in-memory → MarkItDown.convert_stream()
+  └─ everything else → normalize_extension() → MarkItDown.convert(path)
+```
+
+The route returns an `ORJSONResponse`; large markdown payloads are gzipped by
+the `GZipMiddleware`.
+
+## Lifespan
+
+`docsifer.main._lifespan` owns the lifecycle of every singleton:
+
+- `DocsiferService`            — built once, has a bounded `ThreadPoolExecutor`.
+- `AnalyticsService`           — loads totals from Upstash, kicks off the
+  background sync loop, **flushes pending counters on shutdown**.
+- `ConversionGate`, `PerIPLimiter`, `ResourceGuard` — no I/O, just config.
+- `disk_cleanup_loop`          — periodic temp-file sweeper.
+- `memory_watchdog_loop`       — optional, sends SIGTERM on RSS overrun.
+
+All background tasks share an `asyncio.Event` so shutdown is deterministic.
+
+## Concurrency model
+
+- One global `asyncio.Semaphore` (in `ConversionGate`) bounds in-flight
+  conversions. Anything beyond `max_concurrent + max_queue` is rejected with
+  `503 + Retry-After`.
+- `PerIPLimiter` enforces fairness so a single IP cannot monopolize the gate.
+- `ResourceGuard` runs ahead of every conversion to short-circuit OOM.
+- All synchronous work happens in a dedicated `ThreadPoolExecutor` to keep
+  the event loop responsive.
+
+## Analytics
+
+- Increments are stored in an in-process `pending` counter and applied to a
+  `totals` counter atomically. The lock-free `_snapshot` dict is what
+  `/v1/stats` returns — readers are never blocked by writers.
+- The background sync loop pipelines `HINCRBY` operations into Upstash so a
+  full flush is a single HTTP round-trip when the server supports pipelines.
+- Failures keep `pending` intact for the next attempt; on shutdown the
+  service performs one final flush.
+
+## Security posture
+
+- **SSRF** — every URL goes through `validate_url()` which resolves the host
+  and rejects private/loopback/link-local/multicast addresses by default.
+- **Path traversal** — `Path(filename).name` strips any directory component.
+- **Body limit** — enforced by middleware before the body is consumed.
+- **Extension allowlist** — server-side allowlist independent of the UI.
+- **CORS** — `allow_credentials` is automatically disabled when origins
+  contain `*` (which would otherwise be invalid per the spec).
+- **Error responses** — never echo internal exception messages; only the
+  `public_message` of the typed exception is returned.
+
+## Free-tier defaults
+
+The defaults in `config.py` target a 2 vCPU / 16 GB RAM container (Hugging
+Face Spaces basic):
+
+- `max_upload_bytes = 10 MB`
+- `max_concurrent_conversions = 2`
+- `max_queue_depth = 10`
+- `max_per_ip_concurrent = 1`
+- `request_timeout_sec = 55` (just under HF's 60 s gateway)
+- `analytics_sync_interval_sec = 1800`

CONTRIBUTING.md

@@ -0,0 +1,42 @@
+# Contributing
+
+Thanks for your interest in Docsifer.
+
+## Development setup
+
+```bash
+git clone https://github.com/lh0x00/docsifer.git
+cd docsifer
+make install
+cp .env.example .env
+make run
+```
+
+Hop into <http://localhost:7860/docs> for the OpenAPI docs.
+
+## Style
+
+We use:
+
+- **ruff** for linting and formatting (`make lint` / `make format`).
+- **mypy** for type checking (`make type`).
+- **pytest** + **pytest-asyncio** for tests (`make test`).
+
+Pre-commit hooks are configured in `.pre-commit-config.yaml`. Install them
+with `pre-commit install` to catch issues before pushing.
+
+## Pull requests
+
+1. Branch from `main`.
+2. Add tests for any new behavior — see `tests/unit` and `tests/integration`.
+3. Make sure `make lint test` passes locally.
+4. Update `README.md` / `ARCHITECTURE.md` when you change behavior.
+
+## Bug reports
+
+Please include:
+
+- Docsifer version (`docsifer/__init__.py:__version__`).
+- Python and OS versions.
+- A minimal reproduction (curl command or test snippet).
+- Logs (with `DOCSIFER_LOG_JSON=false` for readability).

Dockerfile

@@ -1,34 +1,96 @@
-# Use Python 3.10.9 as the base image for consistent runtime environment
-FROM python:3.10.9
-
-# Add metadata labels
-LABEL maintainer="lamhieu.vk@gmail.com"
-LABEL description="Docsifer: Efficient Data Conversion to Markdown using FastAPI and Hugging Face Transformers."
-LABEL version="1.0"
-
-# Setup non-root user for security
-RUN useradd -m -u 1000 user
-USER user
-ENV HOME=/home/user \
-  PATH=/home/user/.local/bin:$PATH
-
-# Set working directory for all subsequent commands
-WORKDIR $HOME/app
-
-# Copy application files
-# Copy requirements first to leverage Docker cache
-COPY --chown=user requirements.txt .
-COPY --chown=user . .
-
-# Install Python dependencies
-# --no-cache-dir reduces image size
-# --upgrade ensures latest compatible versions
-RUN pip install --no-cache-dir --upgrade -r requirements.txt
-
-# Expose service port
+# syntax=docker/dockerfile:1.6
+# =============================================================================
+# Docsifer — production image (multi-stage, slim, HF Spaces compatible)
+# Build:   docker build -t docsifer .
+# Run:     docker run --rm -p 7860:7860 --env-file .env docsifer
+# =============================================================================
+
+ARG PYTHON_VERSION=3.11
+
+# -----------------------------------------------------------------------------
+# Stage 1 — builder: compile wheels into /install
+# -----------------------------------------------------------------------------
+FROM python:${PYTHON_VERSION}-slim AS builder
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1 \
+    PIP_NO_CACHE_DIR=0
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+        build-essential \
+        libmagic1 \
+        ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /build
+COPY requirements.txt .
+RUN --mount=type=cache,target=/root/.cache/pip \
+    pip install --upgrade pip wheel \
+    && pip install --prefix=/install -r requirements.txt
+
+# -----------------------------------------------------------------------------
+# Stage 2 — runtime: minimal final image
+# -----------------------------------------------------------------------------
+FROM python:${PYTHON_VERSION}-slim AS runtime
+
+LABEL org.opencontainers.image.title="Docsifer" \
+      org.opencontainers.image.description="Document → Markdown service powered by MarkItDown" \
+      org.opencontainers.image.source="https://github.com/lh0x00/docsifer" \
+      org.opencontainers.image.licenses="MIT"
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PIP_NO_CACHE_DIR=1 \
+    HF_HOME=/home/app/.cache/huggingface \
+    XDG_CACHE_HOME=/home/app/.cache \
+    TMPDIR=/tmp \
+    DOCSIFER_TMP_DIR=/tmp \
+    DOCSIFER_LOG_JSON=true \
+    DOCSIFER_ENVIRONMENT=production \
+    PORT=7860 \
+    WEB_CONCURRENCY=2
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+        ca-certificates \
+        libmagic1 \
+        ffmpeg \
+        curl \
+    && rm -rf /var/lib/apt/lists/*
+
+# Hugging Face Spaces requires uid 1000 with write access to /home/user.
+# We honor that convention so the image is portable.
+RUN useradd --create-home --uid 1000 --shell /bin/bash app
+
+# Bring in the pre-built site-packages from the builder
+COPY --from=builder /install /usr/local
+
+WORKDIR /home/app/app
+COPY --chown=app:app . .
+
+# Pre-create the cache dirs so HF Spaces / users with restricted FS still work
+RUN mkdir -p "$HF_HOME" "$XDG_CACHE_HOME" \
+    && chown -R app:app /home/app
+
+USER app
+
 EXPOSE 7860
 
-# Launch FastAPI application using uvicorn server
-# --host 0.0.0.0: Listen on all network interfaces
-# --port 7860: Run on port 7860
-CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]
+HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 \
+    CMD curl -fsS "http://127.0.0.1:${PORT}/v1/healthz" || exit 1
+
+# Gunicorn + Uvicorn workers with deliberate request recycling so any slow
+# library-level memory leak is bounded.  ``WEB_CONCURRENCY`` and ``PORT`` can
+# be tuned via environment variables.
+CMD ["sh", "-c", "exec gunicorn docsifer.main:app \
+      --bind 0.0.0.0:${PORT} \
+      --worker-class uvicorn.workers.UvicornWorker \
+      --workers ${WEB_CONCURRENCY} \
+      --timeout 120 \
+      --graceful-timeout 30 \
+      --max-requests 500 \
+      --max-requests-jitter 50 \
+      --access-logfile - \
+      --error-logfile -"]

Makefile

@@ -0,0 +1,45 @@
+.PHONY: help install dev lint format type test cov run docker-build docker-run clean
+
+PY ?= python3
+APP := docsifer.main:app
+PORT ?= 7860
+HOST ?= 0.0.0.0
+
+help:  ## Show this help
+	@awk 'BEGIN {FS=":.*##"; printf "Targets:\n"} /^[a-zA-Z0-9_.-]+:.*##/ {printf "  %-16s %s\n", $$1, $$2}' $(MAKEFILE_LIST)
+
+install:  ## Install runtime + dev dependencies (using pip + requirements.txt)
+	$(PY) -m pip install --upgrade pip wheel
+	$(PY) -m pip install -r requirements.txt
+	$(PY) -m pip install pytest pytest-asyncio ruff mypy fakeredis
+
+dev: install  ## Install in editable / dev mode
+	$(PY) -m pip install -e .
+
+lint:  ## Lint with ruff
+	ruff check .
+
+format:  ## Format with ruff
+	ruff format .
+
+type:  ## Type-check with mypy
+	mypy
+
+test:  ## Run pytest
+	pytest -q
+
+cov:  ## Run pytest with coverage
+	pytest --cov=docsifer --cov-report=term-missing
+
+run:  ## Run the dev server (uvicorn, autoreload)
+	uvicorn $(APP) --host $(HOST) --port $(PORT) --reload
+
+docker-build:  ## Build the production Docker image
+	docker build -t docsifer:local .
+
+docker-run:  ## Run the Docker image on $(PORT)
+	docker run --rm -p $(PORT):7860 --env-file .env docsifer:local
+
+clean:  ## Remove caches and build artefacts
+	rm -rf .pytest_cache .ruff_cache .mypy_cache .coverage htmlcov dist build *.egg-info
+	find . -name __pycache__ -type d -prune -exec rm -rf {} +

README.md

@@ -1,107 +1,133 @@
 ---
 title: Docsifer
-emoji: 👻 / 📚
-colorFrom: green
-colorTo: indigo
+emoji: 📚
+colorFrom: indigo
+colorTo: violet
 sdk: docker
-app_file: app.py
+app_port: 7860
 pinned: false
+license: mit
+short_description: Convert documents into clean, LLM-ready Markdown.
 ---
 
-# 📄 Docsifer: Efficient Data Conversion to Markdown
+<div align="center">
 
-**Docsifer** is a powerful FastAPI + Gradio service for converting various data formats (PDF, PowerPoint, Word, Excel, Images, Audio, HTML, etc.) to Markdown. It leverages the [MarkItDown](https://github.com/microsoft/markitdown) library and can optionally use LLMs (via OpenAI) for richer extraction (OCR, speech-to-text, etc.).
+# 📚 Docsifer
 
-## ✨ Key Features
+**Convert documents into clean, LLM-ready Markdown.**
 
-- **Comprehensive Format Support**: 
-  - **PDF**: Extracts text and structure effectively.
-  - **PowerPoint**: Converts slides into Markdown-friendly content.
-  - **Word**: Processes `.docx` files with precision.
-  - **Excel**: Extracts tabular data as Markdown tables.
-  - **Images**: Reads **EXIF metadata** and applies **OCR** for text extraction.
-  - **Audio**: Retrieves **EXIF metadata** and performs **speech transcription**.
-  - **HTML**: Transforms web pages into Markdown.
-  - **Text-Based Formats**: Handles CSV, JSON, XML with ease.
-  - **ZIP Files**: Iterates over contents for batch processing.
-- **LLM Integration**: Leverages OpenAI's GPT-4 for enhanced extraction quality and contextual understanding.
-- **Efficient and Fast**: Optimized for speed while maintaining high accuracy.
-- **Easy Deployment**: Dockerized for hassle-free setup and scalability.
-- **Interactive Playground**: Test conversion processes interactively using a **Gradio-powered interface**.
-- **Usage Analytics**: Tracks token usage and access statistics via Upstash Redis.
+PDF · Word · PowerPoint · Excel · HTML · Audio · Image · CSV · JSON · ZIP
 
-## 🚀 Use Cases
+[![Python](https://img.shields.io/badge/python-3.10%2B-blue?logo=python&logoColor=white)](#)
+[![FastAPI](https://img.shields.io/badge/FastAPI-0.110%2B-009688?logo=fastapi&logoColor=white)](#)
+[![Docker](https://img.shields.io/badge/docker-ready-2496ED?logo=docker&logoColor=white)](#)
+[![License](https://img.shields.io/badge/license-MIT-green)](LICENSE)
 
-- **Knowledge Indexing**: Convert various document formats into Markdown for indexing and search.
-- **Text Analysis**: Prepare data for semantic analysis and NLP tasks.
-- **Content Transformation**: Simplify content preparation for blogs, documentation, or databases.
-- **Metadata Extraction**: Extract meaningful metadata from images and audio for categorization and tagging.
+</div>
 
-## 🛠️ Getting Started
+---
 
-### 1. Clone the Repository
+## Features
 
-```bash
-git clone https://github.com/lh0x00/docsifer.git
-cd docsifer
-```
+- **Multi-format** — PDF, Office, audio (Whisper), images (vision), HTML, CSV, JSON, ZIP, and more — powered by [MarkItDown](https://github.com/microsoft/markitdown).
+- **Optional LLM** — bring your own OpenAI-compatible key for OCR, transcription and structured layout extraction.
+- **Production-grade** — bounded concurrency, per-IP fairness, memory watchdog, disk cleanup, circuit breaker.
+- **Hardened** — SSRF guard, path-traversal sanitation, body-size limits, security headers.
+- **Observable** — JSON logs with request id, `/v1/healthz`, `/v1/readyz`, `/v1/stats`.
+- **Privacy-first** — files are processed in-memory and discarded immediately.
+
+## Quickstart
 
-### 2. Build and Run with Docker
-Make sure Docker is installed and running on your machine.
 ```bash
-docker build -t lightweight-embeddings .
-docker run -p 7860:7860 lightweight-embeddings
+# Local
+make install
+cp .env.example .env
+make run
+
+# Docker
+docker build -t docsifer .
+docker run --rm -p 7860:7860 --env-file .env docsifer
 ```
 
-The API will now be accessible at `http://localhost:7860`.
+Open <http://localhost:7860> for the UI or <http://localhost:7860/docs> for the API.
 
-## 📖 API Overview
+## API
 
-### Endpoints
+| Method | Path           | Description                       |
+| ------ | -------------- | --------------------------------- |
+| POST   | `/v1/convert`  | Convert a file or URL to Markdown |
+| GET    | `/v1/stats`    | Usage analytics snapshot          |
+| GET    | `/v1/healthz`  | Liveness probe                    |
+| GET    | `/v1/readyz`   | Readiness probe                   |
 
-- **`/v1/convert`**: Convert a file to Markdown. Supports both file uploads and file path inputs. Accepts optional OpenAI parameters to enable LLM-based enhancements.
-- **`/v1/stats`**: Retrieve usage statistics, including access counts and token usage.
+### Examples
 
-### Interactive Docs
+Basic conversion:
 
-- Visit the [Swagger UI](http://localhost:7860/docs) for detailed, interactive documentation.
-- Explore additional resources with [ReDoc](http://localhost:7860/redoc).
-
-## 🔬 Playground
+```bash
+curl -X POST http://localhost:7860/v1/convert \
+     -F "file=@document.pdf"
+```
 
-### Interactive Conversion
+With LLM enhancement:
 
-- Test file conversion directly in the browser using the **Gradio interface**.
-- Simply visit `http://localhost:7860` after starting the server to access the playground.
+```bash
+curl -X POST http://localhost:7860/v1/convert \
+     -F "file=@page.html" \
+     -F 'openai={"api_key":"sk-...","model":"gpt-4o-mini"}'
+```
 
-### Features
+Convert a URL:
 
-- **File Upload**: Upload a file directly or provide a local file path.
-- **OpenAI Integration**: Optionally provide OpenAI API details to enhance conversion with LLM capabilities.
-- **Conversion Result**: View the resulting Markdown output instantly.
-- **Usage Statistics**: Monitor access and token usage through the Gradio interface.
+```bash
+curl -X POST http://localhost:7860/v1/convert \
+     -F "url=https://example.com/article"
+```
 
-## 🌐 Resources
+## Configuration
 
-- **Documentation**: [Explore full documentation](https://lamhieu-docsifer.hf.space/docs)
-- **Hugging Face Space**: [Try the live demo](https://huggingface.co/spaces/lh0x00/docsifer)
-- **GitHub Repository**: [View source code](https://github.com/lh0x00/docsifer)
+All settings are environment-driven (prefix `DOCSIFER_`). See [`.env.example`](.env.example) for the full list. Common knobs:
 
-## 💡 Why Docsifer?
+| Variable                              | Default | Purpose                              |
+| ------------------------------------- | ------- | ------------------------------------ |
+| `DOCSIFER_MAX_UPLOAD_BYTES`           | `10MB`  | Hard upload limit                    |
+| `DOCSIFER_MAX_CONCURRENT_CONVERSIONS` | `2`     | Global parallelism                   |
+| `DOCSIFER_MAX_QUEUE_DEPTH`            | `10`    | Reject 503 when exceeded             |
+| `DOCSIFER_MAX_PER_IP_CONCURRENT`      | `1`     | Per-IP fairness                      |
+| `DOCSIFER_REQUEST_TIMEOUT_SEC`        | `55`    | Conversion timeout                   |
+| `DOCSIFER_REDIS_URL` / `_TOKEN`       | local   | Upstash Redis for analytics          |
+| `DOCSIFER_URL_ALLOW_PRIVATE_NETWORKS` | `false` | Disable to block SSRF                |
+| `WEB_CONCURRENCY` *(Docker)*          | `2`     | Number of Gunicorn workers           |
 
-1. **Versatile and Comprehensive**: Handles a wide range of formats, making it a one-stop solution for content conversion.
-2. **AI-Powered**: Uses OpenAI's GPT-4 to enhance extraction accuracy and adapt to complex data structures.
-3. **User-Friendly**: Offers intuitive APIs and a built-in interactive interface for experimentation.
-4. **Scalable and Efficient**: Optimized for performance with Docker support and asynchronous processing.
-5. **Transparent Analytics**: Tracks usage metrics to help monitor and manage service consumption.
+## Architecture
 
-## 👥 Contributors
+```
+docsifer/
+├── api/         FastAPI layer — routes, schemas, middleware
+├── core/        Pure logic — converter, MIME, tokenizer, LLM cache
+├── analytics/   Lifespan-managed analytics (Upstash + in-memory)
+├── safety/      Anti-crash primitives (gate, limiter, watchdog, breaker)
+├── ui/          Optional Gradio playground
+├── config.py    Pydantic settings
+├── exceptions.py
+├── logging_config.py
+└── main.py      App factory + lifespan
+```
 
-- **lamhieu / lh0x00** – Creator and Maintainer ([GitHub](https://github.com/lh0x00), [HuggingFace](https://huggingface.co/lamhieu))
+See [`ARCHITECTURE.md`](ARCHITECTURE.md) for the full design notes.
 
-Contributions are welcome! Check out the [contribution guidelines](https://github.com/lh0x00/docsifer/blob/main/CONTRIBUTING.md).
+## Development
 
-## 📜 License
+```bash
+make install   # runtime + dev deps
+make lint      # ruff
+make format    # ruff format
+make type      # mypy
+make test      # pytest
+make cov       # pytest with coverage
+```
 
-This project is licensed under the **MIT License**. See the [LICENSE](https://github.com/lh0x00/docsifer/blob/main/LICENSE) file for details.
+## License
 
+[MIT](LICENSE) © Lam Hieu — built on top of the wonderful
+[MarkItDown](https://github.com/microsoft/markitdown).

app.py

@@ -1 +1,5 @@
-from docsifer import app
+"""ASGI entry point for Hugging Face Spaces / uvicorn / gunicorn."""
+
+from docsifer.main import app
+
+__all__ = ["app"]

docsifer/__init__.py

@@ -1,501 +1,6 @@
-# filename: __init__.py
+"""Docsifer: efficient data conversion to Markdown."""
 
-import json
-import logging
-import tempfile
-from typing import Tuple, Optional
+from .config import get_settings
 
-import gradio as gr
-import pandas as pd
-import requests
-from fastapi import FastAPI
-from fastapi.middleware.cors import CORSMiddleware
-from gradio.routes import mount_gradio_app
-from pathlib import Path
-
-# If you want to generate unique filenames, e.g. scuid:
-from scuid import scuid
-
-
-class LogFilter(logging.Filter):
-    """
-    A custom logging filter that only keeps log records containing '/v1'
-    in the request path. This helps to filter out other logs and reduce noise.
-    """
-
-    def filter(self, record):
-        # Only keep log records that contain "/v1" in the request path
-        if record.args and len(record.args) >= 3:
-            if "/v1" in str(record.args[2]):
-                return True
-        return False
-
-
-logger = logging.getLogger("uvicorn.access")
-logger.addFilter(LogFilter())
-
-__version__ = "1.0.0"
-__author__ = "lamhieu"
-__description__ = "Docsifer: Efficient Data Conversion to Markdown."
-__metadata__ = {
-    "project": "Docsifer",
-    "version": __version__,
-    "description": (
-        "Effortlessly convert various files to Markdown, including PDF, PowerPoint, Word, Excel, "
-        "images, audio, HTML, JSON, CSV, XML, ZIP, and more."
-    ),
-    "docs": "https://lamhieu-docsifer.hf.space/docs",
-    "github": "https://github.com/lh0x00/docsifer",
-    "spaces": "https://huggingface.co/spaces/lh0x00/docsifer",
-}
-
-# Docsifer API Endpoints (can be replaced with your live URLs if desired)
-DOCSIFER_API_URL = "http://localhost:7860/v1/convert"
-DOCSIFER_STATS_URL = "http://localhost:7860/v1/stats"
-
-APP_DESCRIPTION = f"""
-# 📝 **Docsifer: Convert Your Documents to Markdown**
-
-Welcome to **Docsifer**, a specialized service that converts your files—like PDF, PPT, Word, Excel, images, audio, HTML, JSON, CSV, XML, ZIP, etc.—into **Markdown** using **MarkItDown** at the core. Optionally, you can leverage **LLMs** (OpenAI) for advanced text extraction.
-
-### Features & Privacy
-
-- **Open Source**: The entire Docsifer codebase is publicly available for review and contribution.
-- **Efficient & Flexible**: Supports multiple file formats, ensuring quick and accurate Markdown conversion.
-- **Privacy-Focused**: We never store user data; all processing is temporary, with only minimal anonymous stats collected for call and token counts.
-- **Production-Ready**: Easy Docker deployment, interactive Gradio playground, and comprehensive REST API documentation.
-- **Community & Collaboration**: Contribute on [GitHub]({__metadata__["github"]}) or try it out on [Hugging Face Spaces]({__metadata__["spaces"]}).
-
-### 🔗 Resources
-- [Documentation]({__metadata__["docs"]}) | [GitHub]({__metadata__["github"]}) | [Live Demo]({__metadata__["spaces"]})
-"""
-
-app = FastAPI(
-    title="Docsifer Service API",
-    description=__description__,
-    version=__version__,
-    docs_url="/docs",
-    redoc_url="/redoc",
-)
-
-# Configure CORS (Cross-Origin Resource Sharing)
-app.add_middleware(
-    CORSMiddleware,
-    allow_origins=["*"],  # Adjust if needed for specific domains
-    allow_credentials=True,
-    allow_methods=["*"],
-    allow_headers=["*"],
-)
-
-# Import and include your existing router (with /v1 endpoints)
-from .router import router
-
-app.include_router(router, prefix="/v1")
-
-
-def call_convert_api(
-    file_obj: Optional[bytes],
-    filename: str = "",
-    url: Optional[str] = None,
-    cleanup: bool = True,
-    openai_base_url: Optional[str] = None,
-    openai_api_key: Optional[str] = None,
-    openai_model: Optional[str] = None,
-    http_cookies: Optional[str] = None,
-) -> Tuple[str, str]:
-    """
-    Call the /v1/convert endpoint, returning (markdown_content, md_file_path).
-    - If there's an error, the first return value is an error message (str),
-      the second is an empty string.
-
-    Args:
-        file_obj (Optional[bytes]): The raw file bytes to be sent. If None, 'url' is used.
-        filename (str): Name of the file (will be posted to the endpoint).
-        url (str, optional): URL to be converted (used only if file_obj is None).
-        cleanup (bool): Whether to enable cleanup mode for HTML files.
-        openai_base_url (str, optional): Base URL for OpenAI or compatible LLM.
-        openai_api_key (str, optional): API key for the LLM.
-        openai_model (str, optional): Model name to use for LLM-based extraction.
-        http_cookies (str, optional): JSON-formatted string representing cookies for HTTP requests.
-
-    Returns:
-        (str, str):
-            - markdown_content (str): The conversion result in Markdown form or an error message.
-            - tmp_md_path (str): The path to the temporary .md file for download.
-    """
-    # Build the "openai" object
-    openai_dict = {}
-    if openai_api_key and openai_api_key.strip():
-        openai_dict["api_key"] = openai_api_key
-    if openai_base_url and openai_base_url.strip():
-        openai_dict["base_url"] = openai_base_url
-    if openai_model and openai_model.strip():
-        openai_dict["model"] = openai_model
-
-    # Build the "settings" object
-    settings_dict = {"cleanup": cleanup}
-
-    data = {
-        # Must match the `Form(...)` fields named "openai" and "settings"
-        "openai": json.dumps(openai_dict),
-        "settings": json.dumps(settings_dict),
-    }
-
-    # If the user left the OpenAI fields blank, remove the `openai` key from data
-    if len(openai_dict) <= 3:
-        data.pop("openai")
-
-    # Build the HTTP configuration object
-    if http_cookies and http_cookies.strip():
-        try:
-            cookies_obj = json.loads(http_cookies)
-        except Exception as e:
-            return (f"❌ Invalid JSON for HTTP Cookies: {str(e)}", "")
-        data["http"] = json.dumps({"cookies": cookies_obj})
-
-    # Decide if we're sending a file or a URL
-    files = {}
-    if file_obj:
-        # If file is provided, it takes priority
-        files = {"file": (filename, file_obj)}
-        data["url"] = ""  # ensure 'url' is empty on the form
-    elif url and url.strip():
-        data["url"] = url.strip()
-    else:
-        return ("❌ Please upload a file or provide a URL.", "")
-
-    # Perform the POST request
-    try:
-        response = requests.post(DOCSIFER_API_URL, files=files, data=data, timeout=30)
-    except requests.exceptions.RequestException as e:
-        return (f"❌ Network Error: {str(e)}", "")
-
-    if response.status_code != 200:
-        return (f"❌ API Error {response.status_code}: {response.text}", "")
-
-    # Parse the API response
-    try:
-        converted = response.json()
-        # Expected structure: { "filename": "...", "markdown": "..." }
-        markdown_content = converted["markdown"]
-    except Exception as e:
-        return (f"❌ Error parsing JSON: {str(e)}", "")
-
-    # Write the returned Markdown to a temp .md file
-    with tempfile.NamedTemporaryFile(
-        mode="w+", suffix=".md", dir="/tmp", delete=False
-    ) as tmp_file:
-        tmp_file.write(markdown_content)
-        tmp_md_path = tmp_file.name
-
-    return (markdown_content, tmp_md_path)
-
-
-def call_stats_api_df() -> Tuple[pd.DataFrame, pd.DataFrame]:
-    """
-    Call /v1/stats endpoint to retrieve analytics data and return two DataFrames:
-    - access_df: Access statistics
-    - tokens_df: Token usage statistics
-
-    Raises:
-        ValueError: If the stats endpoint fails or returns invalid data.
-
-    Returns:
-        Tuple[pd.DataFrame, pd.DataFrame]:
-            (access_df, tokens_df) with columns ["Model", "Total", "Daily",
-            "Weekly", "Monthly", "Yearly"].
-    """
-    try:
-        response = requests.get(DOCSIFER_STATS_URL, timeout=10)
-    except requests.exceptions.RequestException as e:
-        raise ValueError(f"Failed to fetch stats: {str(e)}")
-
-    if response.status_code != 200:
-        raise ValueError(f"Failed to fetch stats: {response.text}")
-
-    data = response.json()
-    # Expected structure:
-    # {
-    #   "access": { <period>: {"docsifer": count, ...}, ... },
-    #   "tokens": { <period>: {"docsifer": count, ...}, ... }
-    # }
-    access_data = data.get("access", {})
-    tokens_data = data.get("tokens", {})
-
-    def build_stats_df(bucket: dict) -> pd.DataFrame:
-        """
-        Helper function to transform a nested dictionary (by period, by model)
-        into a tabular pandas DataFrame.
-        """
-        all_models = set()
-        for period_key in ["total", "daily", "weekly", "monthly", "yearly"]:
-            period_dict = bucket.get(period_key, {})
-            all_models.update(period_dict.keys())
-
-        result_dict = {
-            "Model": [],
-            "Total": [],
-            "Daily": [],
-            "Weekly": [],
-            "Monthly": [],
-            "Yearly": [],
-        }
-
-        for model in sorted(all_models):
-            result_dict["Model"].append(model)
-            result_dict["Total"].append(bucket.get("total", {}).get(model, 0))
-            result_dict["Daily"].append(bucket.get("daily", {}).get(model, 0))
-            result_dict["Weekly"].append(bucket.get("weekly", {}).get(model, 0))
-            result_dict["Monthly"].append(bucket.get("monthly", {}).get(model, 0))
-            result_dict["Yearly"].append(bucket.get("yearly", {}).get(model, 0))
-
-        return pd.DataFrame(result_dict)
-
-    access_df = build_stats_df(access_data)
-    tokens_df = build_stats_df(tokens_data)
-    return access_df, tokens_df
-
-
-def create_main_interface():
-    """
-    Create a Gradio Blocks interface that includes:
-      1) 'Conversion Playground' Tab:
-         - File upload OR URL-based conversion
-         - Optional OpenAI configuration and HTTP configuration
-         - Convert button
-         - Display of conversion result as Markdown
-         - Downloadable .md file
-      2) 'Analytics Stats' Tab:
-         - Button to fetch usage statistics
-         - DataFrames for Access Stats and Token Stats
-
-    Returns:
-        Gradio Blocks instance that can be mounted into the FastAPI app.
-    """
-    with gr.Blocks(title="Docsifer: Convert to Markdown", theme="default") as demo:
-        gr.Markdown(APP_DESCRIPTION)
-
-        with gr.Tab("Conversion Playground"):
-            gr.Markdown("### Convert your files or a URL to Markdown with Docsifer.")
-
-            with gr.Row():
-                # Left Column: File Upload, URL Input, Settings, Button
-                with gr.Column():
-                    file_input = gr.File(
-                        label="Upload File (optional)",
-                        file_types=[
-                            ".html",
-                            ".htm",
-                            ".zip",
-                            ".jpg",
-                            ".jpeg",
-                            ".png",
-                            ".csv",
-                            ".ipynb",
-                            ".msg",
-                            ".xml",
-                            ".docx",
-                            ".json",
-                            ".pptx",
-                            ".xls",
-                            ".xlsx",
-                            ".pdf",
-                            ".mp3",
-                            ".wav",
-                        ],
-                        type="binary",
-                    )
-
-                    url_input = gr.Textbox(
-                        label="URL (optional)",
-                        placeholder="Enter a URL if no file is uploaded",
-                    )
-
-                    with gr.Accordion("OpenAI Configuration (Optional)", open=False):
-                        gr.Markdown(
-                            "Provide these if you'd like **LLM-assisted** extraction. "
-                            "Supports both OpenAI and OpenAI-compatible APIs. "
-                            "If left blank, basic conversion (no LLM) will be used."
-                        )
-                        openai_base_url = gr.Textbox(
-                            label="Base URL",
-                            placeholder="https://api.openai.com/v1",
-                            value="https://api.openai.com/v1",
-                        )
-                        openai_api_key = gr.Textbox(
-                            label="API Key",
-                            placeholder="sk-...",
-                            type="password",
-                        )
-                        openai_model = gr.Textbox(
-                            label="Model ID",
-                            placeholder="e.g. gpt-4o-mini",
-                            value="gpt-4o-mini",
-                        )
-
-                    with gr.Accordion("HTTP Configuration (Optional)", open=False):
-                        gr.Markdown(
-                            "Provide additional HTTP configuration. "
-                            "In particular, you can specify cookies as a JSON object to be included in the request."
-                        )
-                        http_cookies = gr.Textbox(
-                            label="Cookies",
-                            placeholder='e.g. {"session": "abcd1234"}',
-                            lines=3,
-                        )
-
-                    with gr.Accordion("Conversion Settings", open=True):
-                        gr.Markdown(
-                            "Enable to remove <style> tags or hidden elements "
-                            "from `.html` files before conversion."
-                        )
-                        cleanup_toggle = gr.Checkbox(
-                            label="Enable Cleanup",
-                            value=True,
-                        )
-
-                    convert_btn = gr.Button("Convert")
-
-                    gr.Markdown(
-                        """
-                        ### cURL Examples
-
-                        **Convert via File Upload (multipart/form-data)**:
-                        ```bash
-                        curl -X POST \\
-                            "https://lamhieu-docsifer.hf.space/v1/convert" \\
-                            -F "file=@/path/to/local/document.pdf" \\
-                            -F "openai={\\"api_key\\":\\"sk-xxxxx\\",\\"model\\":\\"gpt-4o-mini\\",\\"base_url\\":\\"https://api.openai.com/v1\\"}" \\
-                            -F "settings={\\"cleanup\\":true}"
-                        ```
-
-                        **Convert from a URL (no file)**:
-                        ```bash
-                        curl -X POST \\
-                            "https://lamhieu-docsifer.hf.space/v1/convert" \\
-                            -F "url=https://example.com/page.html" \\
-                            -F "openai={\\"api_key\\":\\"sk-xxxxx\\",\\"model\\":\\"gpt-4o-mini\\",\\"base_url\\":\\"https://api.openai.com/v1\\"}" \\
-                            -F "settings={\\"cleanup\\":true}"
-                        ```
-                        """
-                    )
-
-                # Right Column: Conversion Result Display & Download
-                with gr.Column():
-                    # Display the result as Markdown
-                    output_md = gr.Textbox(
-                        label="Markdown Preview",
-                        lines=40,
-                        interactive=True,
-                        show_copy_button=True,
-                    )
-
-                    # The user can still download the .md file
-                    download_file = gr.File(
-                        label="Download",
-                        interactive=False,
-                        visible=True,
-                    )
-
-            # Callback function triggered by convert_btn.click
-            def on_convert(
-                file_bytes, url_str, base_url, api_key, model_id, cleanup, http_cookies
-            ):
-                """
-                Converts the uploaded file or a URL to Markdown by calling the Docsifer API.
-                Returns the resulting Markdown content and path to the temporary .md file for download.
-
-                Args:
-                    file_bytes (bytes): The raw file content (None if not uploaded).
-                    url_str (str): The URL to convert (only used if file_bytes is None).
-                    base_url (str): The base URL for OpenAI or compatible LLM.
-                    api_key (str): The API key for the LLM.
-                    model_id (str): The model to use for the LLM.
-                    cleanup (bool): Whether to enable cleanup on HTML files.
-                    http_cookies (str): JSON-formatted string for HTTP cookies.
-
-                Returns:
-                    (str, str):
-                        - The Markdown content or error message.
-                        - The path to the temp .md file for download.
-                """
-                if not file_bytes and not url_str:
-                    return "❌ Please upload a file or provide a URL.", None
-
-                # Create a unique temporary filename if file is present
-                unique_name = f"{scuid()}.tmp" if file_bytes else ""
-
-                # Call the convert API with HTTP configuration
-                markdown, temp_md_path = call_convert_api(
-                    file_obj=file_bytes,
-                    filename=unique_name,
-                    url=url_str,
-                    openai_base_url=base_url,
-                    openai_api_key=api_key,
-                    openai_model=model_id,
-                    cleanup=cleanup,
-                    http_cookies=http_cookies,
-                )
-
-                return markdown, temp_md_path
-
-            # Link the on_convert function to the convert_btn
-            convert_btn.click(
-                fn=on_convert,
-                inputs=[
-                    file_input,
-                    url_input,
-                    openai_base_url,
-                    openai_api_key,
-                    openai_model,
-                    cleanup_toggle,
-                    http_cookies,
-                ],
-                outputs=[output_md, download_file],
-            )
-
-        with gr.Tab("Analytics Stats"):
-            gr.Markdown(
-                "View Docsifer usage statistics (access count, token usage, etc.)"
-            )
-            stats_btn = gr.Button("Get Stats")
-
-            access_df = gr.DataFrame(
-                label="Access Stats",
-                headers=["Model", "Total", "Daily", "Weekly", "Monthly", "Yearly"],
-                interactive=False,
-            )
-            tokens_df = gr.DataFrame(
-                label="Token Stats",
-                headers=["Model", "Total", "Daily", "Weekly", "Monthly", "Yearly"],
-                interactive=False,
-            )
-
-            # When the button is clicked, call_stats_api_df returns two dataframes
-            stats_btn.click(
-                fn=call_stats_api_df,
-                inputs=[],
-                outputs=[access_df, tokens_df],
-            )
-
-    return demo
-
-
-main_interface = create_main_interface()
-mount_gradio_app(app, main_interface, path="/")
-
-
-@app.on_event("startup")
-async def startup_event():
-    """
-    Logs a startup message when the Docsifer Service is starting.
-    """
-    logger.info("Docsifer Service is starting up...")
-
-
-@app.on_event("shutdown")
-async def shutdown_event():
-    """
-    Logs a shutdown message when the Docsifer Service is shutting down.
-    """
-    logger.info("Docsifer Service is shutting down.")
+__version__ = get_settings().app_version
+__all__ = ["__version__"]

docsifer/analytics.py

@@ -1,290 +0,0 @@
-# filename: analytics.py
-
-import logging
-import asyncio
-from upstash_redis import Redis as UpstashRedis
-from datetime import datetime
-from collections import defaultdict
-from typing import Dict
-from functools import partial
-
-logger = logging.getLogger(__name__)
-
-
-class Analytics:
-    def __init__(
-        self, url: str, token: str, sync_interval: int = 60, max_retries: int = 5
-    ):
-        """
-        Initializes the Analytics class with an Upstash Redis client (HTTP-based),
-        wrapped in async methods by using run_in_executor.
-
-        We maintain two dictionaries:
-         - current_totals: absolute counters (loaded from Redis at startup).
-         - new_increments: only the new usage since last sync.
-
-        Both structures only track a single label "docsifer" for access/tokens.
-        """
-        self.url = url
-        self.token = token
-        self.sync_interval = sync_interval
-        self.max_retries = max_retries
-
-        # Create the synchronous Upstash Redis client over HTTP
-        self.redis_client = self._create_redis_client()
-
-        # current_totals: absolute counters from Redis
-        self.current_totals = {
-            "access": defaultdict(lambda: defaultdict(int)),
-            "tokens": defaultdict(lambda: defaultdict(int)),
-        }
-        # new_increments: only new usage since the last successful sync
-        self.new_increments = {
-            "access": defaultdict(lambda: defaultdict(int)),
-            "tokens": defaultdict(lambda: defaultdict(int)),
-        }
-
-        # Async lock to protect shared data
-        self.lock = asyncio.Lock()
-
-        # Start initial sync from Redis, then a periodic sync task
-        asyncio.create_task(self._initialize())
-
-        logger.info("Initialized Analytics with Upstash Redis: %s", url)
-
-    def _create_redis_client(self) -> UpstashRedis:
-        """Creates and returns a new Upstash Redis (synchronous) client."""
-        return UpstashRedis(url=self.url, token=self.token)
-
-    async def _initialize(self):
-        """
-        Fetch existing data from Redis into current_totals,
-        then start the periodic sync task.
-        """
-        try:
-            await self._sync_from_redis()
-            logger.info("Initial sync from Upstash Redis completed successfully.")
-        except Exception as e:
-            logger.error("Error during initial Redis sync: %s", e)
-
-        asyncio.create_task(self._start_sync_task())
-
-    def _get_period_keys(self):
-        """
-        Returns day, week, month, year keys based on current UTC date.
-        Also consider "total" if you want an all-time key.
-
-        Example: ("2025-01-14", "2025-W02", "2025-01", "2025", "total")
-        """
-        now = datetime.utcnow()
-        day_key = now.strftime("%Y-%m-%d")
-        week_key = f"{now.year}-W{now.strftime('%U')}"
-        month_key = now.strftime("%Y-%m")
-        year_key = now.strftime("%Y")
-        # For convenience, also track everything in "total".
-        return day_key, week_key, month_key, year_key, "total"
-
-    async def access(self, tokens: int):
-        """
-        Records an access and token usage for the "docsifer" label.
-        This function updates both current_totals and new_increments.
-        """
-        day_key, week_key, month_key, year_key, total_key = self._get_period_keys()
-
-        async with self.lock:
-            # For each time period, increment "docsifer" usage
-            for period in [day_key, week_key, month_key, year_key, total_key]:
-                # Increase new usage
-                self.new_increments["access"][period]["docsifer"] += 1
-                self.new_increments["tokens"][period]["docsifer"] += tokens
-
-                # Also update the absolute totals for immediate stats
-                self.current_totals["access"][period]["docsifer"] += 1
-                self.current_totals["tokens"][period]["docsifer"] += tokens
-
-    async def stats(self) -> Dict[str, Dict[str, Dict[str, int]]]:
-        """
-        Returns a snapshot of current stats (absolute totals).
-        We use current_totals, which is always up to date.
-        """
-        async with self.lock:
-            return {
-                "access": {
-                    period: dict(models)
-                    for period, models in self.current_totals["access"].items()
-                },
-                "tokens": {
-                    period: dict(models)
-                    for period, models in self.current_totals["tokens"].items()
-                },
-            }
-
-    async def _sync_from_redis(self):
-        """
-        Pull existing data from Redis into current_totals and reset new_increments.
-        We read "analytics:access:*" and "analytics:tokens:*" keys via SCAN.
-        """
-        loop = asyncio.get_running_loop()
-
-        async with self.lock:
-            # Reset both structures
-            self.current_totals = {
-                "access": defaultdict(lambda: defaultdict(int)),
-                "tokens": defaultdict(lambda: defaultdict(int)),
-            }
-            self.new_increments = {
-                "access": defaultdict(lambda: defaultdict(int)),
-                "tokens": defaultdict(lambda: defaultdict(int)),
-            }
-
-            # ---------------------------
-            # Load "access" data
-            # ---------------------------
-            cursor = 0
-            while True:
-                scan_result = await loop.run_in_executor(
-                    None,
-                    partial(
-                        self.redis_client.scan,
-                        cursor=cursor,
-                        match="analytics:access:*",
-                        count=1000,
-                    ),
-                )
-                cursor, keys = scan_result[0], scan_result[1]
-
-                for key in keys:
-                    # key => "analytics:access:<period>"
-                    period = key.replace("analytics:access:", "")
-                    data = await loop.run_in_executor(
-                        None,
-                        partial(self.redis_client.hgetall, key),
-                    )
-                    for name_key, count_str in data.items():
-                        self.current_totals["access"][period][name_key] = int(count_str)
-
-                if cursor == 0:
-                    break
-
-            # ---------------------------
-            # Load "tokens" data
-            # ---------------------------
-            cursor = 0
-            while True:
-                scan_result = await loop.run_in_executor(
-                    None,
-                    partial(
-                        self.redis_client.scan,
-                        cursor=cursor,
-                        match="analytics:tokens:*",
-                        count=1000,
-                    ),
-                )
-                cursor, keys = scan_result[0], scan_result[1]
-
-                for key in keys:
-                    # key => "analytics:tokens:<period>"
-                    period = key.replace("analytics:tokens:", "")
-                    data = await loop.run_in_executor(
-                        None,
-                        partial(self.redis_client.hgetall, key),
-                    )
-                    for name_key, count_str in data.items():
-                        self.current_totals["tokens"][period][name_key] = int(count_str)
-
-                if cursor == 0:
-                    break
-
-    async def _sync_to_redis(self):
-        """
-        Push the new_increments to Redis with HINCRBY,
-        then reset new_increments to zero if successful.
-        """
-        loop = asyncio.get_running_loop()
-
-        async with self.lock:
-            try:
-                # Sync "access" increments
-                for period, models in self.new_increments["access"].items():
-                    redis_key = f"analytics:access:{period}"
-                    for name_key, count_val in models.items():
-                        if count_val != 0:
-                            await loop.run_in_executor(
-                                None,
-                                partial(
-                                    self.redis_client.hincrby,
-                                    redis_key,
-                                    name_key,
-                                    count_val,
-                                ),
-                            )
-
-                # Sync "tokens" increments
-                for period, models in self.new_increments["tokens"].items():
-                    redis_key = f"analytics:tokens:{period}"
-                    for name_key, count_val in models.items():
-                        if count_val != 0:
-                            await loop.run_in_executor(
-                                None,
-                                partial(
-                                    self.redis_client.hincrby,
-                                    redis_key,
-                                    name_key,
-                                    count_val,
-                                ),
-                            )
-
-                logger.info("Analytics data synced to Upstash Redis.")
-
-                # Reset new_increments only
-                self.new_increments = {
-                    "access": defaultdict(lambda: defaultdict(int)),
-                    "tokens": defaultdict(lambda: defaultdict(int)),
-                }
-
-            except Exception as e:
-                logger.error("Error syncing to Redis: %s", e)
-                raise e
-
-    async def _start_sync_task(self):
-        """Periodically sync local increments to Redis."""
-        while True:
-            await asyncio.sleep(self.sync_interval)
-            try:
-                await self._sync_to_redis()
-            except Exception as e:
-                logger.error("Error during scheduled sync: %s", e)
-                await self._handle_redis_reconnection()
-
-    async def _handle_redis_reconnection(self):
-        """
-        Attempts to reconnect to Redis if connection fails (HTTP-based, stateless).
-        """
-        loop = asyncio.get_running_loop()
-        retry_count = 0
-        delay = 1
-
-        while retry_count < self.max_retries:
-            try:
-                logger.info(
-                    "Attempting Redis reconnection (attempt %d)...", retry_count + 1
-                )
-                await loop.run_in_executor(None, self.redis_client.close)
-                self.redis_client = self._create_redis_client()
-                logger.info("Reconnected to Redis successfully.")
-                return
-            except Exception as e:
-                logger.error("Reconnection attempt %d failed: %s", retry_count + 1, e)
-                retry_count += 1
-                await asyncio.sleep(delay)
-                delay *= 2
-
-        logger.critical("Max reconnection attempts reached. Redis is unavailable.")
-
-    async def close(self):
-        """
-        Close the Upstash Redis client (though it's stateless over HTTP).
-        """
-        loop = asyncio.get_running_loop()
-        await loop.run_in_executor(None, self.redis_client.close)
-        logger.info("Redis client closed.")

docsifer/analytics/__init__.py

@@ -0,0 +1,13 @@
+"""Analytics package."""
+
+from .periods import period_keys
+from .service import AnalyticsService
+from .store import AnalyticsStore, InMemoryStore, UpstashStore
+
+__all__ = [
+    "AnalyticsService",
+    "AnalyticsStore",
+    "InMemoryStore",
+    "UpstashStore",
+    "period_keys",
+]

docsifer/analytics/periods.py

@@ -0,0 +1,23 @@
+"""Period-key helpers for analytics buckets.
+
+Uses ISO 8601 week numbering (``%G-W%V``) so values at year boundaries are
+consistent across years (fixes the legacy ``%U`` bug).
+"""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+
+def period_keys(now: datetime | None = None) -> tuple[str, str, str, str, str]:
+    """Return ``(day, week, month, year, "total")`` for ``now`` (UTC by default)."""
+    if now is None:
+        now = datetime.now(tz=timezone.utc)
+    elif now.tzinfo is None:
+        now = now.replace(tzinfo=timezone.utc)
+
+    day_key = now.strftime("%Y-%m-%d")
+    week_key = now.strftime("%G-W%V")  # ISO week
+    month_key = now.strftime("%Y-%m")
+    year_key = now.strftime("%Y")
+    return day_key, week_key, month_key, year_key, "total"

docsifer/analytics/service.py

@@ -0,0 +1,193 @@
+"""Lifespan-managed analytics service.
+
+Improvements over the legacy implementation:
+- Lifespan-managed (no ``asyncio.create_task`` at import time → fixes Bug A5).
+- ``new_increments`` accumulator is **never reset on read paths** so concurrent
+  ``access()`` calls during the initial load are no longer lost (Bug A5/A6).
+- Failed Redis syncs keep the increments queued for the next attempt; an
+  optional reconnection backoff retries the sync immediately when possible.
+- ``stats()`` returns an immutable snapshot updated atomically after every
+  ``access()`` so reads are lock-free (Section C.4).
+- Pluggable :class:`AnalyticsStore` enables in-memory mode when Redis is
+  unavailable (Section N.10).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import copy
+import logging
+from collections import defaultdict
+from typing import Any
+
+from .periods import period_keys
+from .store import AnalyticsStore, NestedCounter
+
+logger = logging.getLogger(__name__)
+
+
+def _empty_counter() -> NestedCounter:
+    return {
+        "access": defaultdict(lambda: defaultdict(int)),
+        "tokens": defaultdict(lambda: defaultdict(int)),
+    }
+
+
+class AnalyticsService:
+    """Async-friendly analytics aggregator backed by a pluggable store."""
+
+    def __init__(
+        self,
+        *,
+        store: AnalyticsStore,
+        sync_interval_sec: int = 1800,
+        max_retries: int = 5,
+        label: str = "docsifer",
+    ) -> None:
+        self._store = store
+        self._sync_interval = max(1, sync_interval_sec)
+        self._max_retries = max(0, max_retries)
+        self._label = label
+
+        self._totals: NestedCounter = _empty_counter()
+        self._pending: NestedCounter = _empty_counter()
+        self._snapshot: dict[str, dict[str, dict[str, int]]] = {
+            "access": {},
+            "tokens": {},
+        }
+        self._lock = asyncio.Lock()
+        self._sync_task: asyncio.Task[None] | None = None
+        self._stopped = asyncio.Event()
+        self._healthy = True
+
+    # ------------------------------------------------------------------
+    # Lifecycle
+    # ------------------------------------------------------------------
+    async def start(self) -> None:
+        """Load existing data from the store and launch the sync loop."""
+        try:
+            initial = await self._store.load_all()
+            async with self._lock:
+                # Merge instead of replacing so any access() events that arrived
+                # while loading are preserved.
+                for metric, periods in initial.items():
+                    for period, labels in periods.items():
+                        for label, count in labels.items():
+                            self._totals[metric][period][label] += int(count)
+                self._refresh_snapshot()
+            logger.info("Analytics initial load complete")
+        except Exception as exc:
+            logger.warning("Analytics initial load failed: %s", exc)
+            self._healthy = False
+
+        self._sync_task = asyncio.create_task(self._sync_loop(), name="analytics-sync")
+
+    async def stop(self) -> None:
+        """Flush pending increments and stop the background loop."""
+        self._stopped.set()
+        if self._sync_task is not None:
+            self._sync_task.cancel()
+            with contextlib.suppress(asyncio.CancelledError):
+                await self._sync_task
+        try:
+            await self._flush_once()
+        except Exception as exc:
+            logger.warning("Final analytics flush failed: %s", exc)
+        with contextlib.suppress(Exception):
+            await self._store.close()
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+    async def access(self, tokens: int, *, label: str | None = None) -> None:
+        """Record one access plus ``tokens`` token usage for ``label``."""
+        label = label or self._label
+        keys = period_keys()
+        async with self._lock:
+            for period in keys:
+                self._pending["access"][period][label] += 1
+                self._pending["tokens"][period][label] += int(tokens)
+                self._totals["access"][period][label] += 1
+                self._totals["tokens"][period][label] += int(tokens)
+            self._refresh_snapshot()
+
+    async def stats(self) -> dict[str, Any]:
+        """Return an immutable snapshot of the current totals."""
+        snap = self._snapshot
+        return {
+            "access": snap["access"],
+            "tokens": snap["tokens"],
+            "healthy": self._healthy,
+        }
+
+    async def ping(self) -> bool:
+        return await self._store.ping()
+
+    @property
+    def healthy(self) -> bool:
+        return self._healthy
+
+    # ------------------------------------------------------------------
+    # Internal
+    # ------------------------------------------------------------------
+    def _refresh_snapshot(self) -> None:
+        """Build a new immutable snapshot dict from the live counters."""
+        self._snapshot = {
+            "access": {p: dict(m) for p, m in self._totals["access"].items()},
+            "tokens": {p: dict(m) for p, m in self._totals["tokens"].items()},
+        }
+
+    async def _flush_once(self) -> None:
+        async with self._lock:
+            if not self._has_pending(self._pending):
+                return
+            payload = copy.deepcopy(self._pending)
+            # Optimistically clear; restore on failure
+            cleared = self._pending
+            self._pending = _empty_counter()
+        try:
+            await self._store.apply_increments(payload)
+            self._healthy = True
+        except Exception:
+            # Restore the pending payload so we retry next tick
+            async with self._lock:
+                for metric, periods in payload.items():
+                    for period, labels in periods.items():
+                        for label, count in labels.items():
+                            self._pending[metric][period][label] += count
+                _ = cleared  # silence "unused"
+            self._healthy = False
+            raise
+
+    async def _sync_loop(self) -> None:
+        retries = 0
+        backoff = 1.0
+        while not self._stopped.is_set():
+            try:
+                await asyncio.wait_for(
+                    self._stopped.wait(), timeout=self._sync_interval
+                )
+                return  # stopped
+            except asyncio.TimeoutError:
+                pass
+
+            try:
+                await self._flush_once()
+                retries = 0
+                backoff = 1.0
+            except Exception as exc:
+                logger.warning("Analytics sync failed: %s", exc)
+                if retries < self._max_retries:
+                    retries += 1
+                    await asyncio.sleep(min(backoff, 60.0))
+                    backoff *= 2
+
+    @staticmethod
+    def _has_pending(counter: NestedCounter) -> bool:
+        for periods in counter.values():
+            for labels in periods.values():
+                for count in labels.values():
+                    if count:
+                        return True
+        return False

docsifer/analytics/store.py

@@ -0,0 +1,183 @@
+"""Storage backends for analytics counters.
+
+The :class:`AnalyticsStore` protocol abstracts the persistence layer so the
+:class:`docsifer.analytics.service.AnalyticsService` can switch between
+Upstash Redis (production) and an :class:`InMemoryStore` (tests / local dev /
+graceful-degraded mode when Redis is unavailable).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from collections import defaultdict
+from functools import partial
+from typing import Protocol, runtime_checkable
+
+logger = logging.getLogger(__name__)
+
+# Top-level metric → period → label → count
+NestedCounter = dict[str, dict[str, dict[str, int]]]
+
+
+def _empty_counter() -> NestedCounter:
+    return {
+        "access": defaultdict(lambda: defaultdict(int)),
+        "tokens": defaultdict(lambda: defaultdict(int)),
+    }
+
+
+@runtime_checkable
+class AnalyticsStore(Protocol):
+    async def load_all(self) -> NestedCounter: ...
+
+    async def apply_increments(self, increments: NestedCounter) -> None: ...
+
+    async def ping(self) -> bool: ...
+
+    async def close(self) -> None: ...
+
+
+# ---------------------------------------------------------------------------
+# In-memory store (fallback / tests)
+# ---------------------------------------------------------------------------
+class InMemoryStore:
+    def __init__(self) -> None:
+        self._data: NestedCounter = _empty_counter()
+
+    async def load_all(self) -> NestedCounter:
+        return _empty_counter()
+
+    async def apply_increments(self, increments: NestedCounter) -> None:
+        for metric, periods in increments.items():
+            for period, models in periods.items():
+                for label, count in models.items():
+                    if count:
+                        self._data[metric][period][label] += count
+
+    async def ping(self) -> bool:
+        return True
+
+    async def close(self) -> None:  # pragma: no cover - nothing to do
+        return None
+
+
+# ---------------------------------------------------------------------------
+# Upstash Redis (HTTP) store
+# ---------------------------------------------------------------------------
+class UpstashStore:
+    """Upstash Redis-backed store using batched pipeline writes (Section N.9)."""
+
+    KEY_PREFIX = "analytics"
+
+    def __init__(self, url: str, token: str | None) -> None:
+        from upstash_redis import Redis as UpstashRedis  # lazy import
+
+        self._UpstashRedis = UpstashRedis
+        self._url = url
+        self._token = token
+        self._client = self._make_client()
+
+    def _make_client(self):  # type: ignore[no-untyped-def]
+        return self._UpstashRedis(url=self._url, token=self._token)
+
+    async def load_all(self) -> NestedCounter:
+        loop = asyncio.get_running_loop()
+        result: NestedCounter = _empty_counter()
+        for metric in ("access", "tokens"):
+            cursor = 0
+            pattern = f"{self.KEY_PREFIX}:{metric}:*"
+            while True:
+                scan = await loop.run_in_executor(
+                    None,
+                    partial(self._client.scan, cursor=cursor, match=pattern, count=1000),
+                )
+                cursor = scan[0]
+                keys = scan[1] or []
+                for key in keys:
+                    period = key.split(":", 2)[-1]
+                    data = await loop.run_in_executor(
+                        None, partial(self._client.hgetall, key)
+                    )
+                    for label, count in (data or {}).items():
+                        try:
+                            result[metric][period][label] = int(count)
+                        except (TypeError, ValueError):
+                            continue
+                if int(cursor) == 0:
+                    break
+        return result
+
+    async def apply_increments(self, increments: NestedCounter) -> None:
+        loop = asyncio.get_running_loop()
+        ops: list = []
+        for metric, periods in increments.items():
+            for period, models in periods.items():
+                key = f"{self.KEY_PREFIX}:{metric}:{period}"
+                for label, count in models.items():
+                    if count:
+                        ops.append((key, label, int(count)))
+        if not ops:
+            return
+
+        # Prefer pipeline when available (single HTTP round trip)
+        pipeline = getattr(self._client, "pipeline", None)
+        if callable(pipeline):
+            def _run() -> None:
+                pipe = pipeline()
+                for key, label, count in ops:
+                    pipe.hincrby(key, label, count)
+                pipe.execute()
+
+            await loop.run_in_executor(None, _run)
+            return
+
+        # Fallback: parallel single-op calls
+        await asyncio.gather(
+            *(
+                loop.run_in_executor(
+                    None, partial(self._client.hincrby, key, label, count)
+                )
+                for key, label, count in ops
+            )
+        )
+
+    async def ping(self) -> bool:
+        loop = asyncio.get_running_loop()
+        try:
+            await loop.run_in_executor(None, self._client.ping)
+            return True
+        except Exception as exc:
+            logger.warning("Upstash ping failed: %s", exc)
+            return False
+
+    async def close(self) -> None:
+        loop = asyncio.get_running_loop()
+        with logger_suppress("Upstash close failed"):
+            await loop.run_in_executor(None, self._client.close)
+
+
+# ---------------------------------------------------------------------------
+# Helper
+# ---------------------------------------------------------------------------
+class logger_suppress:  # noqa: N801 - tiny utility, lower-case name for context-manager idiom
+    """Context manager that logs (rather than raises) any exception."""
+
+    def __init__(self, message: str) -> None:
+        self._message = message
+
+    def __enter__(self) -> "logger_suppress":
+        return self
+
+    def __exit__(self, exc_type, exc, tb) -> bool:  # type: ignore[no-untyped-def]
+        if exc_type is not None:
+            logger.warning("%s: %s", self._message, exc)
+        return True
+
+    async def __aenter__(self) -> "logger_suppress":
+        return self
+
+    async def __aexit__(self, exc_type, exc, tb) -> bool:  # type: ignore[no-untyped-def]
+        if exc_type is not None:
+            logger.warning("%s: %s", self._message, exc)
+        return True

docsifer/api/__init__.py

@@ -0,0 +1 @@
+"""HTTP/API layer."""

docsifer/api/deps.py

@@ -0,0 +1,56 @@
+"""FastAPI dependency providers wired against the app state."""
+
+from __future__ import annotations
+
+from fastapi import Depends, Request
+
+from ..analytics import AnalyticsService
+from ..config import Settings, get_settings
+from ..core.service import DocsiferService
+from ..safety import ConversionGate, PerIPLimiter, ResourceGuard
+
+
+def settings_dep() -> Settings:
+    return get_settings()
+
+
+def converter_dep(request: Request) -> DocsiferService:
+    return request.app.state.converter  # type: ignore[no-any-return]
+
+
+def analytics_dep(request: Request) -> AnalyticsService:
+    return request.app.state.analytics  # type: ignore[no-any-return]
+
+
+def conversion_gate_dep(request: Request) -> ConversionGate:
+    return request.app.state.conversion_gate  # type: ignore[no-any-return]
+
+
+def per_ip_limiter_dep(request: Request) -> PerIPLimiter:
+    return request.app.state.per_ip_limiter  # type: ignore[no-any-return]
+
+
+def resource_guard_dep(request: Request) -> ResourceGuard:
+    return request.app.state.resource_guard  # type: ignore[no-any-return]
+
+
+def client_ip_dep(request: Request) -> str:
+    forwarded = request.headers.get("x-forwarded-for")
+    if forwarded:
+        return forwarded.split(",")[0].strip()
+    real = request.headers.get("x-real-ip")
+    if real:
+        return real.strip()
+    return request.client.host if request.client else "unknown"
+
+
+__all__ = [
+    "Depends",
+    "analytics_dep",
+    "client_ip_dep",
+    "conversion_gate_dep",
+    "converter_dep",
+    "per_ip_limiter_dep",
+    "resource_guard_dep",
+    "settings_dep",
+]

docsifer/api/error_handlers.py

@@ -0,0 +1,90 @@
+"""Translate domain and unexpected exceptions into safe JSON responses."""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from fastapi import FastAPI, HTTPException, Request
+from fastapi.exceptions import RequestValidationError
+from fastapi.responses import JSONResponse
+
+from ..exceptions import DocsiferError
+from ..logging_config import request_id_var
+
+logger = logging.getLogger(__name__)
+
+
+def _payload(
+    *,
+    error: str,
+    message: str,
+    details: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    body: dict[str, Any] = {
+        "error": error,
+        "message": message,
+        "request_id": request_id_var.get(),
+    }
+    if details:
+        body["details"] = details
+    return body
+
+
+async def _docsifer_handler(request: Request, exc: DocsiferError) -> JSONResponse:
+    log = logger.warning if exc.status_code < 500 else logger.exception
+    log(
+        "%s on %s: %s",
+        exc.__class__.__name__,
+        request.url.path,
+        exc,
+    )
+    return JSONResponse(
+        status_code=exc.status_code,
+        content=_payload(
+            error=exc.__class__.__name__,
+            message=exc.public_message,
+            details=exc.details or None,
+        ),
+    )
+
+
+async def _http_exception_handler(request: Request, exc: HTTPException) -> JSONResponse:
+    return JSONResponse(
+        status_code=exc.status_code,
+        content=_payload(
+            error="HTTPException",
+            message=str(exc.detail) if exc.detail else "HTTP error",
+        ),
+    )
+
+
+async def _validation_handler(
+    request: Request, exc: RequestValidationError
+) -> JSONResponse:
+    return JSONResponse(
+        status_code=422,
+        content=_payload(
+            error="ValidationError",
+            message="Request validation failed",
+            details={"errors": exc.errors()},
+        ),
+    )
+
+
+async def _fallback_handler(request: Request, exc: Exception) -> JSONResponse:
+    logger.exception("Unhandled exception on %s", request.url.path)
+    return JSONResponse(
+        status_code=500,
+        content=_payload(
+            error="InternalServerError",
+            message="An unexpected error occurred",
+        ),
+    )
+
+
+def register_exception_handlers(app: FastAPI) -> None:
+    app.add_exception_handler(DocsiferError, _docsifer_handler)  # type: ignore[arg-type]
+    app.add_exception_handler(HTTPException, _http_exception_handler)
+    app.add_exception_handler(RequestValidationError, _validation_handler)  # type: ignore[arg-type]
+    app.add_exception_handler(Exception, _fallback_handler)

docsifer/api/middleware.py

@@ -0,0 +1,78 @@
+"""Custom middleware: request id, body size limit, security headers."""
+
+from __future__ import annotations
+
+import secrets
+from typing import Awaitable, Callable
+
+from fastapi import FastAPI, Request, Response
+from fastapi.responses import JSONResponse
+
+from ..config import Settings
+from ..logging_config import request_id_var
+
+ASGICall = Callable[[Request], Awaitable[Response]]
+
+REQUEST_ID_HEADER = "X-Request-ID"
+_SECURITY_HEADERS = {
+    "X-Content-Type-Options": "nosniff",
+    "X-Frame-Options": "DENY",
+    "Referrer-Policy": "no-referrer",
+    "X-XSS-Protection": "0",
+}
+
+
+async def _request_id_middleware(request: Request, call_next: ASGICall) -> Response:
+    rid = request.headers.get(REQUEST_ID_HEADER) or secrets.token_hex(8)
+    token = request_id_var.set(rid)
+    try:
+        response = await call_next(request)
+    finally:
+        request_id_var.reset(token)
+    response.headers[REQUEST_ID_HEADER] = rid
+    return response
+
+
+def _make_body_limit_middleware(
+    max_bytes: int,
+) -> Callable[[Request, ASGICall], Awaitable[Response]]:
+    async def _body_limit(request: Request, call_next: ASGICall) -> Response:
+        cl = request.headers.get("content-length")
+        if cl is not None:
+            try:
+                content_length = int(cl)
+            except ValueError:
+                content_length = None
+            if content_length is not None and content_length > max_bytes:
+                return JSONResponse(
+                    status_code=413,
+                    content={
+                        "error": "PayloadTooLargeError",
+                        "message": "Payload too large",
+                        "details": {"max_bytes": max_bytes},
+                        "request_id": request_id_var.get(),
+                    },
+                )
+        return await call_next(request)
+
+    return _body_limit
+
+
+async def _security_headers_middleware(request: Request, call_next: ASGICall) -> Response:
+    response = await call_next(request)
+    for header, value in _SECURITY_HEADERS.items():
+        response.headers.setdefault(header, value)
+    return response
+
+
+def register_middleware(app: FastAPI, settings: Settings) -> None:
+    """Wire middleware in the correct order.
+
+    Order (outermost first): request-id → body-limit → security-headers.
+    Starlette executes them in *registration* order on the way in and reverse
+    order on the way out, so the first one we add runs outermost.
+    """
+    app.middleware("http")(_request_id_middleware)
+    app.middleware("http")(_make_body_limit_middleware(settings.max_upload_bytes))
+    if settings.enable_security_headers:
+        app.middleware("http")(_security_headers_middleware)

docsifer/api/schemas.py

@@ -0,0 +1,76 @@
+"""Pydantic request/response models for the public API."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class OpenAIConfig(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    api_key: str | None = None
+    base_url: str | None = None
+    model: str | None = None
+
+    def is_enabled(self) -> bool:
+        return bool(self.api_key and self.api_key.strip())
+
+    def to_dict(self) -> dict[str, Any] | None:
+        if not self.is_enabled():
+            return None
+        out: dict[str, Any] = {"api_key": self.api_key}
+        if self.base_url:
+            out["base_url"] = self.base_url
+        if self.model:
+            out["model"] = self.model
+        return out
+
+
+class HTTPConfig(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    cookies: dict[str, str] | None = None
+    headers: dict[str, str] | None = None
+
+    def to_dict(self) -> dict[str, Any] | None:
+        out: dict[str, Any] = {}
+        if self.cookies:
+            out["cookies"] = self.cookies
+        if self.headers:
+            out["headers"] = self.headers
+        return out or None
+
+
+class ConvertSettings(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    cleanup: bool = Field(
+        default=True,
+        description="Strip <style>, <script> and hidden elements from HTML before conversion.",
+    )
+
+
+class ConvertResponse(BaseModel):
+    filename: str
+    markdown: str
+
+
+class StatsResponse(BaseModel):
+    access: dict[str, dict[str, int]] = Field(default_factory=dict)
+    tokens: dict[str, dict[str, int]] = Field(default_factory=dict)
+    healthy: bool = True
+
+
+class HealthResponse(BaseModel):
+    status: str
+    version: str | None = None
+    details: dict[str, Any] | None = None
+
+
+class ErrorResponse(BaseModel):
+    error: str
+    message: str
+    request_id: str | None = None
+    details: dict[str, Any] | None = None

docsifer/api/v1/__init__.py

@@ -0,0 +1,14 @@
+"""v1 API routers."""
+
+from fastapi import APIRouter
+
+from .convert import router as convert_router
+from .health import router as health_router
+from .stats import router as stats_router
+
+router = APIRouter()
+router.include_router(convert_router)
+router.include_router(stats_router)
+router.include_router(health_router)
+
+__all__ = ["router"]

docsifer/api/v1/convert.py

@@ -0,0 +1,213 @@
+"""``POST /v1/convert`` endpoint.
+
+Implements all the safety controls from Section N (admission, per-IP
+fairness, resource guard) plus the bug fixes from Section A:
+
+- A3 path-traversal: filename is sanitized via ``Path(name).name``.
+- A4 SSRF: URLs are validated by :func:`validate_url`.
+- F4 body size: enforced by middleware.
+- F5 MIME allowlist: enforced here against ``settings.allowed_extensions``.
+- F6 leak: errors mapped to :class:`DocsiferError` subclasses.
+- C6 blocking I/O: file is streamed to disk in a worker thread.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import secrets
+import shutil
+import tempfile
+from pathlib import Path
+from typing import Any
+
+from fastapi import APIRouter, BackgroundTasks, Depends, File, Form, UploadFile
+from fastapi.responses import ORJSONResponse
+
+from ...analytics import AnalyticsService
+from ...config import Settings
+from ...core.service import DocsiferService
+from ...core.url_guard import validate_url
+from ...exceptions import (
+    InvalidInputError,
+    PayloadTooLargeError,
+    UnsupportedFormatError,
+    ValidationError,
+)
+from ...safety import ConversionGate, PerIPLimiter, ResourceGuard
+from ..deps import (
+    analytics_dep,
+    client_ip_dep,
+    conversion_gate_dep,
+    converter_dep,
+    per_ip_limiter_dep,
+    resource_guard_dep,
+    settings_dep,
+)
+from ..schemas import ConvertResponse, ConvertSettings, HTTPConfig, OpenAIConfig
+
+logger = logging.getLogger(__name__)
+router = APIRouter(tags=["v1"])
+
+_CHUNK_SIZE = 1024 * 1024  # 1 MB
+
+
+def _parse_json_form(name: str, raw: str | None, model: type[Any]) -> Any:
+    if not raw or not raw.strip():
+        return model()
+    try:
+        return model.model_validate_json(raw)
+    except Exception as exc:
+        raise ValidationError(
+            f"Invalid JSON in '{name}'",
+            details={"field": name, "error": str(exc)},
+        ) from exc
+
+
+def _safe_filename(raw: str | None) -> str:
+    name = (raw or "").strip()
+    safe = Path(name).name if name else ""
+    return safe or f"upload-{secrets.token_hex(4)}.bin"
+
+
+def _check_extension(name: str, allowed: set[str]) -> None:
+    suffix = Path(name).suffix.lower()
+    if not suffix:
+        return  # allow extensionless and rely on MIME sniff
+    if suffix not in allowed:
+        raise UnsupportedFormatError(
+            f"Extension '{suffix}' is not allowed",
+            details={"allowed": sorted(allowed)},
+        )
+
+
+async def _stream_to_disk(
+    upload: UploadFile,
+    dst: Path,
+    *,
+    max_bytes: int,
+) -> int:
+    """Stream the upload to ``dst`` without loading it fully into RAM."""
+
+    def _copy() -> int:
+        total = 0
+        with dst.open("wb") as fh:
+            while True:
+                chunk = upload.file.read(_CHUNK_SIZE)
+                if not chunk:
+                    break
+                total += len(chunk)
+                if total > max_bytes:
+                    raise PayloadTooLargeError(
+                        f"Body exceeds {max_bytes} bytes",
+                        details={"max_bytes": max_bytes},
+                    )
+                fh.write(chunk)
+        return total
+
+    try:
+        return await asyncio.to_thread(_copy)
+    finally:
+        await upload.close()
+
+
+@router.post(
+    "/convert",
+    response_model=ConvertResponse,
+    response_class=ORJSONResponse,
+    summary="Convert a file or URL into Markdown",
+    responses={
+        400: {"description": "Invalid input"},
+        413: {"description": "Payload too large"},
+        415: {"description": "Unsupported format"},
+        422: {"description": "Validation failed"},
+        429: {"description": "Too many requests"},
+        503: {"description": "Service unavailable"},
+    },
+)
+async def convert_document(
+    background_tasks: BackgroundTasks,
+    file: UploadFile | None = File(default=None, description="File to convert"),
+    url: str | None = Form(default=None, description="URL to convert"),
+    openai: str | None = Form(default=None, description="OpenAI config JSON"),
+    http: str | None = Form(default=None, description="HTTP config JSON"),
+    settings_form: str | None = Form(default=None, alias="settings"),
+    settings: Settings = Depends(settings_dep),
+    converter: DocsiferService = Depends(converter_dep),
+    analytics: AnalyticsService = Depends(analytics_dep),
+    gate: ConversionGate = Depends(conversion_gate_dep),
+    per_ip: PerIPLimiter = Depends(per_ip_limiter_dep),
+    guard: ResourceGuard = Depends(resource_guard_dep),
+    client_ip: str = Depends(client_ip_dep),
+) -> ConvertResponse:
+    if file is None and not (url and url.strip()):
+        raise InvalidInputError("Provide either 'file' or 'url'.")
+
+    openai_cfg = _parse_json_form("openai", openai, OpenAIConfig)
+    http_cfg = _parse_json_form("http", http, HTTPConfig)
+    convert_cfg = _parse_json_form("settings", settings_form, ConvertSettings)
+
+    async with per_ip.acquire(client_ip):
+        async with gate.acquire():
+            if file is not None:
+                guard.check(0)  # file size unknown until streamed
+                _check_extension(file.filename or "", set(settings.allowed_extensions))
+
+                tmp_root = Path(tempfile.mkdtemp(prefix="docsifer-", dir=settings.tmp_dir))
+                try:
+                    safe_name = _safe_filename(file.filename)
+                    dst = tmp_root / safe_name
+                    size = await _stream_to_disk(
+                        file, dst, max_bytes=settings.max_upload_bytes
+                    )
+                    logger.info(
+                        "Convert file received",
+                        extra={
+                            "filename": safe_name,
+                            "size": size,
+                            "client_ip": client_ip,
+                        },
+                    )
+                    guard.check(size)
+
+                    result = await asyncio.wait_for(
+                        converter.convert_file(
+                            dst,
+                            openai_config=openai_cfg.to_dict(),
+                            http_config=http_cfg.to_dict(),
+                            cleanup_html=convert_cfg.cleanup,
+                        ),
+                        timeout=settings.request_timeout_sec,
+                    )
+                finally:
+                    shutil.rmtree(tmp_root, ignore_errors=True)
+            else:
+                safe_url = validate_url(
+                    url or "",
+                    allowed_schemes=settings.url_allowed_schemes,
+                    allow_private_networks=settings.url_allow_private_networks,
+                )
+                logger.info(
+                    "Convert URL received",
+                    extra={"url": safe_url, "client_ip": client_ip},
+                )
+                guard.check(0)
+                result = await asyncio.wait_for(
+                    converter.convert_file(
+                        safe_url,
+                        openai_config=openai_cfg.to_dict(),
+                        http_config=http_cfg.to_dict(),
+                        cleanup_html=convert_cfg.cleanup,
+                    ),
+                    timeout=settings.request_timeout_sec,
+                )
+
+    background_tasks.add_task(_record_access_safe, analytics, result.token_count)
+    return ConvertResponse(filename=result.filename, markdown=result.markdown)
+
+
+async def _record_access_safe(analytics: AnalyticsService, tokens: int) -> None:
+    try:
+        await analytics.access(tokens)
+    except Exception:
+        logger.exception("Failed to record analytics access")

docsifer/api/v1/health.py

@@ -0,0 +1,42 @@
+"""Health probes (liveness + readiness) for orchestration platforms."""
+
+from __future__ import annotations
+
+from fastapi import APIRouter, Depends
+
+from ...analytics import AnalyticsService
+from ...config import Settings
+from ..deps import analytics_dep, settings_dep
+from ..schemas import HealthResponse
+
+router = APIRouter(tags=["health"])
+
+
+@router.get(
+    "/healthz",
+    response_model=HealthResponse,
+    summary="Liveness probe",
+    description="Always returns 200 if the process is responsive.",
+)
+async def healthz(settings: Settings = Depends(settings_dep)) -> HealthResponse:
+    return HealthResponse(status="ok", version=settings.app_version)
+
+
+@router.get(
+    "/readyz",
+    response_model=HealthResponse,
+    summary="Readiness probe",
+    description="Returns 200 only when the analytics backend is reachable.",
+)
+async def readyz(
+    analytics: AnalyticsService = Depends(analytics_dep),
+    settings: Settings = Depends(settings_dep),
+) -> HealthResponse:
+    healthy = await analytics.ping() if settings.analytics_persistent else True
+    if not healthy:
+        return HealthResponse(
+            status="degraded",
+            version=settings.app_version,
+            details={"analytics": "unreachable"},
+        )
+    return HealthResponse(status="ready", version=settings.app_version)

docsifer/api/v1/stats.py

@@ -0,0 +1,27 @@
+"""Usage statistics endpoint."""
+
+from __future__ import annotations
+
+from fastapi import APIRouter, Depends
+
+from ...analytics import AnalyticsService
+from ..deps import analytics_dep
+from ..schemas import StatsResponse
+
+router = APIRouter(tags=["v1"])
+
+
+@router.get(
+    "/stats",
+    response_model=StatsResponse,
+    summary="Get usage statistics",
+)
+async def get_stats(
+    analytics: AnalyticsService = Depends(analytics_dep),
+) -> StatsResponse:
+    snap = await analytics.stats()
+    return StatsResponse(
+        access=snap.get("access", {}),
+        tokens=snap.get("tokens", {}),
+        healthy=bool(snap.get("healthy", True)),
+    )

docsifer/config.py

@@ -0,0 +1,199 @@
+"""Application settings loaded from environment variables.
+
+All configuration is centralized here to avoid scattered hardcoded values.
+Environment variables are prefixed with ``DOCSIFER_`` (see ``.env.example``).
+"""
+
+from __future__ import annotations
+
+import tempfile
+from functools import lru_cache
+from pathlib import Path
+from typing import Literal
+
+from pydantic import Field, field_validator
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    """Centralized, env-driven application settings."""
+
+    model_config = SettingsConfigDict(
+        env_prefix="DOCSIFER_",
+        env_file=".env",
+        env_file_encoding="utf-8",
+        extra="ignore",
+        case_sensitive=False,
+    )
+
+    # ---------------------------------------------------------------------
+    # General
+    # ---------------------------------------------------------------------
+    app_name: str = "Docsifer"
+    app_version: str = "1.1.0"
+    environment: Literal["development", "staging", "production"] = "production"
+    log_level: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] = "INFO"
+    log_json: bool = True
+
+    # ---------------------------------------------------------------------
+    # HTTP / API
+    # ---------------------------------------------------------------------
+    cors_origins: list[str] = Field(default_factory=lambda: ["*"])
+    cors_allow_credentials: bool = False  # safe with origins=["*"]
+    request_timeout_sec: int = 55  # < HF Spaces 60s timeout
+    max_upload_bytes: int = 10 * 1024 * 1024  # 10 MB free-tier default
+    gzip_min_size: int = 1024
+    enable_security_headers: bool = True
+
+    # ---------------------------------------------------------------------
+    # Concurrency / resources
+    # ---------------------------------------------------------------------
+    max_concurrent_conversions: int = 2  # tuned for 2 vCPU
+    max_queue_depth: int = 10
+    max_per_ip_concurrent: int = 1
+    worker_pool_size: int = 4  # ThreadPoolExecutor for sync work
+    min_free_memory_mb: int = 512
+    min_free_disk_mb: int = 256
+    memory_watchdog_pct: float = 90.0
+    memory_watchdog_interval_sec: int = 30
+    enable_memory_watchdog: bool = False  # disabled by default in dev
+    disk_cleanup_interval_sec: int = 600
+    disk_cleanup_ttl_sec: int = 3600
+
+    # ---------------------------------------------------------------------
+    # Conversion
+    # ---------------------------------------------------------------------
+    token_model: str = "gpt-4o"
+    default_openai_base_url: str = "https://api.openai.com/v1"
+    default_openai_model: str = "gpt-4o-mini"
+    openai_request_timeout_sec: float = 60.0
+    openai_connect_timeout_sec: float = 10.0
+    openai_max_retries: int = 2
+
+    # LLM client cache
+    llm_cache_max_size: int = 16
+    llm_cache_ttl_sec: int = 600
+
+    # Allowed file extensions (lower-case, with leading dot)
+    allowed_extensions: list[str] = Field(
+        default_factory=lambda: [
+            ".html",
+            ".htm",
+            ".zip",
+            ".jpg",
+            ".jpeg",
+            ".png",
+            ".gif",
+            ".webp",
+            ".csv",
+            ".tsv",
+            ".ipynb",
+            ".msg",
+            ".xml",
+            ".docx",
+            ".doc",
+            ".json",
+            ".pptx",
+            ".ppt",
+            ".xls",
+            ".xlsx",
+            ".pdf",
+            ".mp3",
+            ".wav",
+            ".m4a",
+            ".txt",
+            ".md",
+            ".rtf",
+        ]
+    )
+
+    # ---------------------------------------------------------------------
+    # SSRF / URL fetch guard
+    # ---------------------------------------------------------------------
+    url_allow_private_networks: bool = False
+    url_allowed_schemes: list[str] = Field(default_factory=lambda: ["http", "https"])
+
+    # ---------------------------------------------------------------------
+    # Analytics / Redis
+    #
+    # ``redis_url`` is empty by default so fresh deployments (HF Spaces, local
+    # dev) start in in-memory analytics mode out of the box. Set
+    # ``DOCSIFER_REDIS_URL`` (and ``DOCSIFER_REDIS_TOKEN`` for Upstash) to
+    # opt into persistent analytics.
+    # ---------------------------------------------------------------------
+    redis_url: str = ""
+    redis_token: str | None = None
+    analytics_enabled: bool = True
+    analytics_sync_interval_sec: int = 1800  # 30 min
+    analytics_max_retries: int = 5
+    analytics_label: str = "docsifer"
+
+    @property
+    def analytics_persistent(self) -> bool:
+        """True when a real (non-empty, non-localhost) Redis URL is configured."""
+        if not self.analytics_enabled:
+            return False
+        url = (self.redis_url or "").strip()
+        if not url:
+            return False
+        return True
+
+    # ---------------------------------------------------------------------
+    # Quotas (anonymous, BYOK, authenticated)
+    # ---------------------------------------------------------------------
+    quota_enabled: bool = False  # enable when slowapi/redis is configured
+    quota_anon_rph: int = 10
+    quota_anon_rpd: int = 50
+    quota_byok_rph: int = 60
+    quota_byok_rpd: int = 500
+    quota_auth_keys: list[str] = Field(default_factory=list)
+
+    # ---------------------------------------------------------------------
+    # Observability
+    # ---------------------------------------------------------------------
+    enable_prometheus: bool = False
+    sentry_dsn: str | None = None
+
+    # ---------------------------------------------------------------------
+    # Filesystem
+    # ---------------------------------------------------------------------
+    tmp_dir: Path = Field(default_factory=lambda: Path(tempfile.gettempdir()))
+
+    # ---------------------------------------------------------------------
+    # Validators
+    # ---------------------------------------------------------------------
+    @field_validator("allowed_extensions", mode="before")
+    @classmethod
+    def _normalize_exts(cls, v: object) -> list[str]:
+        if isinstance(v, str):
+            v = [item.strip() for item in v.split(",") if item.strip()]
+        if not isinstance(v, list):
+            raise TypeError("allowed_extensions must be a list or comma-separated string")
+        return [e.lower() if e.startswith(".") else f".{e.lower()}" for e in v]
+
+    @field_validator("cors_origins", "url_allowed_schemes", "quota_auth_keys", mode="before")
+    @classmethod
+    def _split_csv(cls, v: object) -> list[str]:
+        if isinstance(v, str):
+            return [item.strip() for item in v.split(",") if item.strip()]
+        return list(v) if v else []
+
+    # ---------------------------------------------------------------------
+    # Convenience properties
+    # ---------------------------------------------------------------------
+    @property
+    def cors_allow_credentials_safe(self) -> bool:
+        """``allow_credentials`` is invalid together with ``allow_origins=['*']``."""
+        if "*" in self.cors_origins:
+            return False
+        return self.cors_allow_credentials
+
+    @property
+    def is_production(self) -> bool:
+        return self.environment == "production"
+
+
+@lru_cache(maxsize=1)
+def get_settings() -> Settings:
+    """Return a process-wide singleton ``Settings`` instance."""
+    return Settings()

docsifer/core/__init__.py

@@ -0,0 +1 @@
+"""Core domain layer (no FastAPI / Gradio dependencies)."""

docsifer/core/html_cleaner.py

@@ -0,0 +1,72 @@
+"""HTML cleanup utilities.
+
+Uses ``selectolax`` (≈2-3× faster than lxml/pyquery) when available with a
+graceful fallback to pure regex for environments where selectolax is missing.
+The cleaner removes scripts, styles, hidden nodes and elements relying on
+``display:none`` / ``visibility:hidden`` / ``aria-hidden``.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+
+logger = logging.getLogger(__name__)
+
+try:  # pragma: no cover - import-time only
+    from selectolax.parser import HTMLParser  # type: ignore
+
+    _HAS_SELECTOLAX = True
+except Exception:  # pragma: no cover
+    HTMLParser = None  # type: ignore[assignment]
+    _HAS_SELECTOLAX = False
+
+
+_REMOVE_SELECTORS: tuple[str, ...] = (
+    "style",
+    "script",
+    "noscript",
+    "[hidden]",
+    '[style*="display:none"]',
+    '[style*="display: none"]',
+    '[style*="visibility:hidden"]',
+    '[style*="visibility: hidden"]',
+    '[aria-hidden="true"]',
+)
+
+_FALLBACK_PATTERNS: tuple[re.Pattern[str], ...] = (
+    re.compile(r"<style\b[^>]*>.*?</style>", re.IGNORECASE | re.DOTALL),
+    re.compile(r"<script\b[^>]*>.*?</script>", re.IGNORECASE | re.DOTALL),
+    re.compile(r"<noscript\b[^>]*>.*?</noscript>", re.IGNORECASE | re.DOTALL),
+)
+
+
+def clean_html(html: str) -> str:
+    """Return ``html`` with hidden / inert nodes removed.
+
+    The function is best-effort: any parser failure logs at DEBUG and returns
+    the original input unchanged so downstream conversion can still proceed.
+    """
+    if not html:
+        return html
+
+    if _HAS_SELECTOLAX:
+        try:
+            tree = HTMLParser(html)
+            for selector in _REMOVE_SELECTORS:
+                for node in tree.css(selector):
+                    node.decompose()
+            return tree.html or ""
+        except Exception as exc:
+            logger.debug("selectolax cleanup failed: %s", exc)
+
+    cleaned = html
+    for pattern in _FALLBACK_PATTERNS:
+        cleaned = pattern.sub("", cleaned)
+    return cleaned
+
+
+def clean_html_bytes(data: bytes, encoding: str = "utf-8") -> bytes:
+    """Convenience wrapper for byte input/output (used to avoid extra decodes)."""
+    text = data.decode(encoding, errors="ignore")
+    return clean_html(text).encode(encoding, errors="ignore")

docsifer/core/llm_registry.py

@@ -0,0 +1,104 @@
+"""TTL-cached registry of MarkItDown instances configured with an LLM client.
+
+Creating an ``OpenAI`` client (and the wrapping ``MarkItDown``) is expensive
+because each ``OpenAI`` instance owns a private ``httpx.Client`` connection
+pool and triggers a TLS handshake on first use.  We cache one instance per
+``(api_key_hash, base_url, model)`` tuple so repeated requests reuse the same
+TCP/TLS session.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+import threading
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any
+
+import httpx
+from cachetools import TTLCache
+from markitdown import MarkItDown
+from openai import OpenAI
+
+if TYPE_CHECKING:
+    pass
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True, slots=True)
+class LLMConfig:
+    api_key: str
+    base_url: str
+    model: str
+
+    @property
+    def cache_key(self) -> str:
+        digest = hashlib.sha256(self.api_key.encode("utf-8")).hexdigest()[:16]
+        return f"{digest}|{self.base_url}|{self.model}"
+
+    @classmethod
+    def from_dict(
+        cls,
+        data: dict[str, Any] | None,
+        *,
+        default_base_url: str,
+        default_model: str,
+    ) -> "LLMConfig | None":
+        if not data:
+            return None
+        api_key = (data.get("api_key") or "").strip()
+        if not api_key:
+            return None
+        base_url = (data.get("base_url") or default_base_url).strip().rstrip("/")
+        model = (data.get("model") or default_model).strip()
+        return cls(api_key=api_key, base_url=base_url, model=model)
+
+
+class LLMRegistry:
+    """Thread-safe TTL cache of LLM-enabled :class:`MarkItDown` instances."""
+
+    def __init__(
+        self,
+        *,
+        max_size: int = 16,
+        ttl: int = 600,
+        request_timeout: float = 60.0,
+        connect_timeout: float = 10.0,
+        max_retries: int = 2,
+    ) -> None:
+        self._cache: TTLCache[str, MarkItDown] = TTLCache(maxsize=max_size, ttl=ttl)
+        self._lock = threading.Lock()
+        self._request_timeout = request_timeout
+        self._connect_timeout = connect_timeout
+        self._max_retries = max_retries
+
+    def get(self, config: LLMConfig) -> MarkItDown:
+        """Return a cached or freshly built :class:`MarkItDown` for ``config``."""
+        key = config.cache_key
+        with self._lock:
+            instance = self._cache.get(key)
+            if instance is not None:
+                return instance
+            instance = self._build(config)
+            self._cache[key] = instance
+            logger.info(
+                "LLM client created",
+                extra={"base_url": config.base_url, "model": config.model},
+            )
+            return instance
+
+    def _build(self, config: LLMConfig) -> MarkItDown:
+        client = OpenAI(
+            api_key=config.api_key,
+            base_url=config.base_url,
+            timeout=httpx.Timeout(
+                self._request_timeout, connect=self._connect_timeout
+            ),
+            max_retries=self._max_retries,
+        )
+        return MarkItDown(llm_client=client, llm_model=config.model)
+
+    def clear(self) -> None:
+        with self._lock:
+            self._cache.clear()

docsifer/core/mime.py

@@ -0,0 +1,79 @@
+"""MIME detection helpers.
+
+Trust the file extension when present and recognized; fall back to libmagic
+sniffing of just the first 4 KB to avoid reading entire large files into RAM.
+"""
+
+from __future__ import annotations
+
+import logging
+import mimetypes
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+# Lazy-import the optional ``magic`` dependency so the application keeps
+# working when libmagic is unavailable.
+try:  # pragma: no cover - environment dependent
+    import magic as _magic  # type: ignore
+
+    _HAS_MAGIC = True
+except Exception as exc:  # pragma: no cover
+    logger.warning("python-magic unavailable, falling back to extensions only: %s", exc)
+    _magic = None  # type: ignore[assignment]
+    _HAS_MAGIC = False
+
+_SNIFF_BYTES = 4096
+
+
+def detect_mime(path: Path, *, sniff_bytes: int = _SNIFF_BYTES) -> str | None:
+    """Return the MIME type of ``path`` or ``None`` when unknown.
+
+    Reads at most ``sniff_bytes`` from the start of the file.
+    """
+    if _HAS_MAGIC:
+        try:
+            with path.open("rb") as fh:
+                head = fh.read(sniff_bytes)
+            return _magic.from_buffer(head, mime=True) or None  # type: ignore[union-attr]
+        except Exception as exc:
+            logger.debug("magic sniff failed for %s: %s", path, exc)
+    # Fallback: derive from extension
+    guess, _ = mimetypes.guess_type(str(path))
+    return guess
+
+
+def guess_extension(mime: str | None) -> str | None:
+    """Return a canonical ``.ext`` string for ``mime`` (or ``None``)."""
+    if not mime:
+        return None
+    return mimetypes.guess_extension(mime)
+
+
+def normalize_extension(
+    path: Path,
+    *,
+    known_extensions: set[str] | None = None,
+) -> Path:
+    """Ensure ``path`` has a sensible extension.
+
+    If the current extension is in ``known_extensions``, it is trusted.
+    Otherwise we sniff the MIME type and rename the file in place using
+    :func:`os.rename` (atomic, zero-copy).
+    """
+    suffix = path.suffix.lower()
+    if known_extensions and suffix in known_extensions:
+        return path
+
+    mime = detect_mime(path)
+    ext = guess_extension(mime)
+    if not ext or ext.lower() == suffix:
+        return path
+
+    new_path = path.with_suffix(ext)
+    try:
+        path.rename(new_path)
+        return new_path
+    except OSError as exc:
+        logger.warning("Could not rename %s -> %s: %s", path, new_path, exc)
+        return path

docsifer/core/service.py

@@ -0,0 +1,254 @@
+"""Pure conversion service.
+
+The service is fully framework-agnostic so it can be invoked directly from
+the FastAPI route handlers, the Gradio UI or background workers without
+issuing self-HTTP loopback calls.
+
+Key design points:
+- Streaming MarkItDown conversion via :class:`io.BytesIO` whenever possible
+  (avoids the read → write → reread cycle of the previous implementation).
+- Cached LLM clients (see :class:`docsifer.core.llm_registry.LLMRegistry`).
+- Bounded :class:`concurrent.futures.ThreadPoolExecutor` for sync work so
+  default executor saturation can never block unrelated coroutines.
+- HTTP cookies are forwarded to MarkItDown via a dedicated
+  :class:`requests.Session`, fixing the previous broken cookie injection.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import io
+import logging
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+import requests
+from markitdown import MarkItDown
+
+from .html_cleaner import clean_html_bytes
+from .llm_registry import LLMConfig, LLMRegistry
+from .mime import normalize_extension
+from .tokenizer import TiktokenCounter
+
+logger = logging.getLogger(__name__)
+
+_HTML_EXTS = frozenset({".html", ".htm"})
+
+
+@dataclass(slots=True)
+class ConvertResult:
+    filename: str
+    markdown: str
+    token_count: int
+
+    def to_dict(self) -> dict[str, Any]:
+        return {"filename": self.filename, "markdown": self.markdown}
+
+
+class DocsiferService:
+    """Convert local files or remote URLs into Markdown."""
+
+    def __init__(
+        self,
+        *,
+        token_model: str = "gpt-4o",
+        default_openai_base_url: str = "https://api.openai.com/v1",
+        default_openai_model: str = "gpt-4o-mini",
+        worker_pool_size: int = 4,
+        llm_cache_max_size: int = 16,
+        llm_cache_ttl: int = 600,
+        openai_request_timeout: float = 60.0,
+        openai_connect_timeout: float = 10.0,
+        openai_max_retries: int = 2,
+        known_extensions: set[str] | None = None,
+    ) -> None:
+        self._basic_md = MarkItDown()
+        self._token_counter = TiktokenCounter(token_model)
+        self._llm_registry = LLMRegistry(
+            max_size=llm_cache_max_size,
+            ttl=llm_cache_ttl,
+            request_timeout=openai_request_timeout,
+            connect_timeout=openai_connect_timeout,
+            max_retries=openai_max_retries,
+        )
+        self._executor = ThreadPoolExecutor(
+            max_workers=worker_pool_size,
+            thread_name_prefix="docsifer-worker",
+        )
+        self._default_base_url = default_openai_base_url
+        self._default_model = default_openai_model
+        self._known_extensions = known_extensions or set()
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+    async def convert_file(
+        self,
+        source: str | Path,
+        *,
+        openai_config: dict[str, Any] | None = None,
+        http_config: dict[str, Any] | None = None,
+        cleanup_html: bool = True,
+    ) -> ConvertResult:
+        """Convert a local file path or HTTP(S) URL into Markdown."""
+        loop = asyncio.get_running_loop()
+        return await loop.run_in_executor(
+            self._executor,
+            self._convert_sync,
+            source,
+            openai_config,
+            http_config,
+            cleanup_html,
+        )
+
+    async def shutdown(self) -> None:
+        """Stop the worker pool gracefully (called from app lifespan)."""
+        self._llm_registry.clear()
+        await asyncio.get_running_loop().run_in_executor(
+            None, self._executor.shutdown, True
+        )
+
+    # ------------------------------------------------------------------
+    # Internal
+    # ------------------------------------------------------------------
+    def _convert_sync(
+        self,
+        source: str | Path,
+        openai_config: dict[str, Any] | None,
+        http_config: dict[str, Any] | None,
+        cleanup_html: bool,
+    ) -> ConvertResult:
+        is_url = isinstance(source, str) and source.lower().startswith(("http://", "https://"))
+
+        md_converter = self._select_converter(openai_config)
+        session = self._build_session(http_config) if http_config else None
+
+        if is_url:
+            return self._convert_url(str(source), md_converter, session)
+        return self._convert_path(Path(source), md_converter, cleanup_html)
+
+    # -- LLM / converter selection ------------------------------------------------
+    def _select_converter(self, openai_config: dict[str, Any] | None) -> MarkItDown:
+        cfg = LLMConfig.from_dict(
+            openai_config,
+            default_base_url=self._default_base_url,
+            default_model=self._default_model,
+        )
+        if cfg is None:
+            return self._basic_md
+        try:
+            return self._llm_registry.get(cfg)
+        except Exception as exc:  # pragma: no cover - defensive
+            logger.warning("Falling back to basic MarkItDown (LLM init failed): %s", exc)
+            return self._basic_md
+
+    @staticmethod
+    def _build_session(http_config: dict[str, Any]) -> requests.Session | None:
+        session = requests.Session()
+        cookies = http_config.get("cookies")
+        if isinstance(cookies, dict) and cookies:
+            session.cookies.update({str(k): str(v) for k, v in cookies.items()})
+        headers = http_config.get("headers")
+        if isinstance(headers, dict) and headers:
+            session.headers.update({str(k): str(v) for k, v in headers.items()})
+        return session
+
+    # -- URL conversion -----------------------------------------------------------
+    def _convert_url(
+        self,
+        url: str,
+        md_converter: MarkItDown,
+        session: requests.Session | None,
+    ) -> ConvertResult:
+        try:
+            if session is not None and hasattr(md_converter, "convert_url"):
+                # markitdown >= 0.0.x exposes convert_url with a session kwarg
+                result_obj = md_converter.convert_url(url, requests_session=session)  # type: ignore[arg-type]
+            else:
+                result_obj = md_converter.convert(url)
+        except Exception as exc:
+            logger.error("URL conversion failed: %s", exc)
+            raise RuntimeError(f"Conversion failed for URL '{url}': {exc}") from exc
+
+        text = getattr(result_obj, "text_content", "") or ""
+        return ConvertResult(
+            filename=self._derive_url_filename(url),
+            markdown=text,
+            token_count=self._token_counter.count(text),
+        )
+
+    @staticmethod
+    def _derive_url_filename(url: str) -> str:
+        from urllib.parse import urlparse
+
+        parsed = urlparse(url)
+        last = (parsed.path or "/").rsplit("/", 1)[-1] or "index.html"
+        if "." not in last:
+            last = f"{last}.html"
+        return last
+
+    # -- Local file conversion ----------------------------------------------------
+    def _convert_path(
+        self,
+        path: Path,
+        md_converter: MarkItDown,
+        cleanup_html: bool,
+    ) -> ConvertResult:
+        if not path.exists():
+            raise FileNotFoundError(f"File not found: {path}")
+
+        normalized = normalize_extension(path, known_extensions=self._known_extensions)
+        ext = normalized.suffix.lower()
+
+        if cleanup_html and ext in _HTML_EXTS:
+            return self._convert_html_streaming(normalized, md_converter, ext)
+
+        try:
+            result_obj = md_converter.convert(str(normalized))
+        except Exception as exc:
+            logger.error("File conversion failed: %s", exc)
+            raise RuntimeError(f"Conversion failed for '{normalized.name}': {exc}") from exc
+
+        text = getattr(result_obj, "text_content", "") or ""
+        return ConvertResult(
+            filename=normalized.name,
+            markdown=text,
+            token_count=self._token_counter.count(text),
+        )
+
+    def _convert_html_streaming(
+        self,
+        path: Path,
+        md_converter: MarkItDown,
+        ext: str,
+    ) -> ConvertResult:
+        """Read → clean → feed MarkItDown without writing back to disk."""
+        try:
+            data = path.read_bytes()
+            cleaned = clean_html_bytes(data)
+        except Exception as exc:
+            logger.warning("HTML cleanup failed (%s); falling back to raw conversion", exc)
+            cleaned = path.read_bytes()
+
+        stream = io.BytesIO(cleaned)
+        try:
+            if hasattr(md_converter, "convert_stream"):
+                result_obj = md_converter.convert_stream(stream, file_extension=ext)
+            else:  # pragma: no cover - older markitdown
+                # Fallback: write cleaned bytes to a sibling temp file
+                with contextlib.suppress(Exception):
+                    path.write_bytes(cleaned)
+                result_obj = md_converter.convert(str(path))
+        except Exception as exc:
+            logger.error("HTML conversion failed: %s", exc)
+            raise RuntimeError(f"Conversion failed for '{path.name}': {exc}") from exc
+
+        text = getattr(result_obj, "text_content", "") or ""
+        return ConvertResult(
+            filename=path.name,
+            markdown=text,
+            token_count=self._token_counter.count(text),
+        )

docsifer/core/tokenizer.py

@@ -0,0 +1,52 @@
+"""Token counting backed by ``tiktoken`` with a robust fallback chain."""
+
+from __future__ import annotations
+
+import logging
+from typing import Protocol
+
+import tiktoken
+
+logger = logging.getLogger(__name__)
+
+
+class TokenCounter(Protocol):
+    def count(self, text: str) -> int: ...
+
+
+class TiktokenCounter:
+    """Token counter backed by ``tiktoken``.
+
+    Falls back through ``cl100k_base`` and finally a whitespace heuristic so
+    that token counting never raises in production paths.
+    """
+
+    def __init__(self, model_name: str = "gpt-4o") -> None:
+        self._encoder = self._load_encoder(model_name)
+
+    @staticmethod
+    def _load_encoder(model_name: str) -> tiktoken.Encoding | None:
+        try:
+            return tiktoken.encoding_for_model(model_name)
+        except Exception as exc:  # pragma: no cover - defensive
+            logger.warning(
+                "tiktoken model '%s' unavailable (%s); using cl100k_base",
+                model_name,
+                exc,
+            )
+        try:
+            return tiktoken.get_encoding("cl100k_base")
+        except Exception as exc:  # pragma: no cover - defensive
+            logger.error("tiktoken unavailable: %s; using whitespace heuristic", exc)
+            return None
+
+    def count(self, text: str) -> int:
+        if not text:
+            return 0
+        if self._encoder is None:
+            return len(text.split())
+        try:
+            return len(self._encoder.encode(text))
+        except Exception as exc:
+            logger.warning("Token encoding failed (%s); using whitespace heuristic", exc)
+            return len(text.split())

docsifer/core/url_guard.py

@@ -0,0 +1,81 @@
+"""SSRF protection for user-supplied URLs.
+
+Validates scheme, blocks loopback / link-local / private / multicast /
+reserved networks unless explicitly allowed via configuration.
+"""
+
+from __future__ import annotations
+
+import ipaddress
+import logging
+import socket
+from urllib.parse import urlparse
+
+from ..exceptions import InvalidInputError
+
+logger = logging.getLogger(__name__)
+
+
+def _is_blocked_address(ip: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool:
+    return (
+        ip.is_loopback
+        or ip.is_link_local
+        or ip.is_multicast
+        or ip.is_reserved
+        or ip.is_private
+        or ip.is_unspecified
+    )
+
+
+def validate_url(
+    url: str,
+    *,
+    allowed_schemes: list[str] | None = None,
+    allow_private_networks: bool = False,
+) -> str:
+    """Return a sanitized URL string or raise :class:`InvalidInputError`.
+
+    When ``allow_private_networks`` is ``False`` (default) we resolve the
+    hostname and reject any address pointing at private/loopback ranges, which
+    blocks the typical SSRF vectors (e.g. AWS metadata at 169.254.169.254).
+    """
+    if not url or not url.strip():
+        raise InvalidInputError("URL must not be empty")
+
+    parsed = urlparse(url.strip())
+    scheme = (parsed.scheme or "").lower()
+    schemes = [s.lower() for s in (allowed_schemes or ["http", "https"])]
+    if scheme not in schemes:
+        raise InvalidInputError(
+            f"URL scheme '{scheme or '<none>'}' is not allowed",
+            details={"allowed_schemes": schemes},
+        )
+    if not parsed.hostname:
+        raise InvalidInputError("URL must include a hostname")
+
+    if allow_private_networks:
+        return parsed.geturl()
+
+    host = parsed.hostname
+    try:
+        infos = socket.getaddrinfo(host, None)
+    except socket.gaierror as exc:
+        raise InvalidInputError(
+            f"Could not resolve host: {host}", details={"error": str(exc)}
+        ) from exc
+
+    for info in infos:
+        sockaddr = info[4]
+        ip_str = sockaddr[0]
+        try:
+            ip = ipaddress.ip_address(ip_str)
+        except ValueError:
+            continue
+        if _is_blocked_address(ip):
+            logger.warning("Blocked SSRF target host=%s ip=%s", host, ip_str)
+            raise InvalidInputError(
+                "URL points to a non-public address",
+                details={"host": host},
+            )
+
+    return parsed.geturl()

docsifer/exceptions.py

@@ -0,0 +1,107 @@
+"""Domain-level exception hierarchy mapped to HTTP status codes.
+
+Endpoints raise ``DocsiferError`` subclasses; the global exception handler
+converts them into JSON responses with sanitized messages so internal stack
+traces never leak to clients.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+class DocsiferError(Exception):
+    """Base class for all application-level errors."""
+
+    status_code: int = 500
+    public_message: str = "Internal server error"
+
+    def __init__(
+        self,
+        message: str | None = None,
+        *,
+        public_message: str | None = None,
+        details: dict[str, Any] | None = None,
+    ) -> None:
+        super().__init__(message or self.public_message)
+        self.details = details or {}
+        if public_message is not None:
+            self.public_message = public_message
+
+
+# ---------------------------------------------------------------------------
+# 4xx — client errors
+# ---------------------------------------------------------------------------
+class InvalidInputError(DocsiferError):
+    status_code = 400
+    public_message = "Invalid input"
+
+
+class UnauthorizedError(DocsiferError):
+    status_code = 401
+    public_message = "Unauthorized"
+
+
+class ForbiddenError(DocsiferError):
+    status_code = 403
+    public_message = "Forbidden"
+
+
+class NotFoundError(DocsiferError):
+    status_code = 404
+    public_message = "Resource not found"
+
+
+class PayloadTooLargeError(DocsiferError):
+    status_code = 413
+    public_message = "Payload too large"
+
+
+class UnsupportedFormatError(DocsiferError):
+    status_code = 415
+    public_message = "Unsupported file format"
+
+
+class ValidationError(DocsiferError):
+    status_code = 422
+    public_message = "Validation failed"
+
+
+class TooManyRequestsError(DocsiferError):
+    status_code = 429
+    public_message = "Too many requests"
+
+
+# ---------------------------------------------------------------------------
+# 5xx — server errors
+# ---------------------------------------------------------------------------
+class ConversionFailedError(DocsiferError):
+    status_code = 500
+    public_message = "Conversion failed"
+
+
+class UpstreamLLMError(DocsiferError):
+    status_code = 502
+    public_message = "Upstream LLM error"
+
+
+class ServiceUnavailableError(DocsiferError):
+    status_code = 503
+    public_message = "Service unavailable"
+
+
+class ResourceExhaustedError(ServiceUnavailableError):
+    public_message = "Server resources are exhausted, please retry later"
+
+
+class CircuitOpenError(ServiceUnavailableError):
+    public_message = "Upstream temporarily unavailable"
+
+
+class QueueFullError(ServiceUnavailableError):
+    public_message = "Server is busy, please retry later"
+
+
+class GatewayTimeoutError(DocsiferError):
+    status_code = 504
+    public_message = "Upstream timeout"

docsifer/logging_config.py

@@ -0,0 +1,120 @@
+"""Logging setup with optional structured JSON output and request-id binding.
+
+Call :func:`configure_logging` exactly once at application start (inside the
+FastAPI lifespan).
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import logging.config
+import sys
+from contextvars import ContextVar
+from typing import Any
+
+#: Context variable carrying the current request id for the active task.
+request_id_var: ContextVar[str | None] = ContextVar("request_id", default=None)
+
+
+class RequestIdFilter(logging.Filter):
+    """Inject the current request id (if any) into every log record."""
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        record.request_id = request_id_var.get() or "-"
+        return True
+
+
+class JsonFormatter(logging.Formatter):
+    """Minimal JSON formatter producing a single line per record."""
+
+    _RESERVED = {
+        "args",
+        "asctime",
+        "created",
+        "exc_info",
+        "exc_text",
+        "filename",
+        "funcName",
+        "levelname",
+        "levelno",
+        "lineno",
+        "module",
+        "msecs",
+        "message",
+        "msg",
+        "name",
+        "pathname",
+        "process",
+        "processName",
+        "relativeCreated",
+        "stack_info",
+        "thread",
+        "threadName",
+    }
+
+    def format(self, record: logging.LogRecord) -> str:
+        payload: dict[str, Any] = {
+            "ts": self.formatTime(record, "%Y-%m-%dT%H:%M:%S%z"),
+            "level": record.levelname,
+            "logger": record.name,
+            "msg": record.getMessage(),
+            "request_id": getattr(record, "request_id", "-"),
+        }
+        if record.exc_info:
+            payload["exc"] = self.formatException(record.exc_info)
+        # Allow callers to attach extra fields via ``logger.info("...", extra={...})``.
+        for key, value in record.__dict__.items():
+            if key in self._RESERVED or key.startswith("_") or key == "request_id":
+                continue
+            try:
+                json.dumps(value)
+                payload[key] = value
+            except (TypeError, ValueError):
+                payload[key] = repr(value)
+        return json.dumps(payload, ensure_ascii=False, default=str)
+
+
+def configure_logging(level: str = "INFO", json_output: bool = True) -> None:
+    """Configure the root logger and library loggers in one place.
+
+    Args:
+        level: Log level name (``DEBUG``/``INFO``/...).
+        json_output: When ``True`` emit one JSON object per line; otherwise use
+            a human-readable format (useful in local development).
+    """
+    formatter: dict[str, Any]
+    if json_output:
+        formatter = {"()": "docsifer.logging_config.JsonFormatter"}
+    else:
+        formatter = {
+            "format": "%(asctime)s %(levelname)-8s %(name)s [%(request_id)s] %(message)s",
+            "datefmt": "%H:%M:%S",
+        }
+
+    config: dict[str, Any] = {
+        "version": 1,
+        "disable_existing_loggers": False,
+        "filters": {
+            "request_id": {"()": "docsifer.logging_config.RequestIdFilter"},
+        },
+        "formatters": {"default": formatter},
+        "handlers": {
+            "default": {
+                "class": "logging.StreamHandler",
+                "stream": sys.stdout,
+                "formatter": "default",
+                "filters": ["request_id"],
+            }
+        },
+        "loggers": {
+            "": {"handlers": ["default"], "level": level, "propagate": False},
+            "uvicorn": {"handlers": ["default"], "level": level, "propagate": False},
+            "uvicorn.error": {"handlers": ["default"], "level": level, "propagate": False},
+            "uvicorn.access": {"handlers": ["default"], "level": level, "propagate": False},
+            "gunicorn.error": {"handlers": ["default"], "level": level, "propagate": False},
+            "gunicorn.access": {"handlers": ["default"], "level": level, "propagate": False},
+        },
+    }
+
+    logging.config.dictConfig(config)

docsifer/main.py

@@ -0,0 +1,218 @@
+"""Application factory + ASGI entry point.
+
+This module owns the FastAPI ``app`` instance, the lifespan-managed
+singletons (``DocsiferService``, ``AnalyticsService``, safety primitives)
+and the optional Gradio UI mount.
+
+Run with::
+
+    uvicorn docsifer.main:app --host 0.0.0.0 --port 7860
+
+or in production with::
+
+    gunicorn -k uvicorn.workers.UvicornWorker -w 4 docsifer.main:app
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import logging
+from contextlib import asynccontextmanager
+from typing import AsyncIterator
+
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.middleware.gzip import GZipMiddleware
+from fastapi.responses import ORJSONResponse
+
+from .analytics import AnalyticsService, InMemoryStore, UpstashStore
+from .analytics.store import AnalyticsStore
+from .api.error_handlers import register_exception_handlers
+from .api.middleware import register_middleware
+from .api.v1 import router as v1_router
+from .config import Settings, get_settings
+from .core.service import DocsiferService
+from .logging_config import configure_logging
+from .safety import (
+    ConversionGate,
+    PerIPLimiter,
+    ResourceGuard,
+    disk_cleanup_loop,
+    memory_watchdog_loop,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def _build_analytics_store(settings: Settings) -> AnalyticsStore:
+    if not settings.analytics_persistent:
+        logger.info("Analytics: in-memory (no DOCSIFER_REDIS_URL configured)")
+        return InMemoryStore()
+    try:
+        store = UpstashStore(url=settings.redis_url, token=settings.redis_token)
+        logger.info("Analytics: Upstash store configured")
+        return store
+    except Exception as exc:  # pragma: no cover - defensive
+        logger.warning(
+            "Could not initialize Upstash store (%s); falling back to in-memory", exc
+        )
+        return InMemoryStore()
+
+
+@asynccontextmanager
+async def _lifespan(app: FastAPI) -> AsyncIterator[None]:
+    settings: Settings = app.state.settings
+    configure_logging(level=settings.log_level, json_output=settings.log_json)
+    logger.info(
+        "Starting %s v%s (env=%s)",
+        settings.app_name,
+        settings.app_version,
+        settings.environment,
+    )
+
+    # ---- core converter -------------------------------------------------
+    converter = DocsiferService(
+        token_model=settings.token_model,
+        default_openai_base_url=settings.default_openai_base_url,
+        default_openai_model=settings.default_openai_model,
+        worker_pool_size=settings.worker_pool_size,
+        llm_cache_max_size=settings.llm_cache_max_size,
+        llm_cache_ttl=settings.llm_cache_ttl_sec,
+        openai_request_timeout=settings.openai_request_timeout_sec,
+        openai_connect_timeout=settings.openai_connect_timeout_sec,
+        openai_max_retries=settings.openai_max_retries,
+        known_extensions=set(settings.allowed_extensions),
+    )
+    app.state.converter = converter
+
+    # ---- analytics ------------------------------------------------------
+    store = _build_analytics_store(settings)
+    analytics = AnalyticsService(
+        store=store,
+        sync_interval_sec=settings.analytics_sync_interval_sec,
+        max_retries=settings.analytics_max_retries,
+        label=settings.analytics_label,
+    )
+    await analytics.start()
+    app.state.analytics = analytics
+
+    # ---- safety primitives ---------------------------------------------
+    app.state.conversion_gate = ConversionGate(
+        max_concurrent=settings.max_concurrent_conversions,
+        max_queue=settings.max_queue_depth,
+    )
+    app.state.per_ip_limiter = PerIPLimiter(
+        max_per_ip=settings.max_per_ip_concurrent,
+    )
+    app.state.resource_guard = ResourceGuard(
+        min_free_memory_mb=settings.min_free_memory_mb,
+        min_free_disk_mb=settings.min_free_disk_mb,
+        tmp_dir=settings.tmp_dir,
+    )
+
+    # ---- background loops ----------------------------------------------
+    stop_event = asyncio.Event()
+    background_tasks: list[asyncio.Task[None]] = [
+        asyncio.create_task(
+            disk_cleanup_loop(
+                tmp_dir=settings.tmp_dir,
+                ttl_sec=settings.disk_cleanup_ttl_sec,
+                interval_sec=settings.disk_cleanup_interval_sec,
+                stop_event=stop_event,
+            ),
+            name="docsifer-disk-cleanup",
+        )
+    ]
+    if settings.enable_memory_watchdog:
+        background_tasks.append(
+            asyncio.create_task(
+                memory_watchdog_loop(
+                    threshold_pct=settings.memory_watchdog_pct,
+                    interval_sec=settings.memory_watchdog_interval_sec,
+                    stop_event=stop_event,
+                ),
+                name="docsifer-memory-watchdog",
+            )
+        )
+    app.state.stop_event = stop_event
+
+    try:
+        yield
+    finally:
+        logger.info("Shutting down %s", settings.app_name)
+        stop_event.set()
+        for task in background_tasks:
+            task.cancel()
+            with contextlib.suppress(asyncio.CancelledError, Exception):
+                await task
+        with contextlib.suppress(Exception):
+            await analytics.stop()
+        with contextlib.suppress(Exception):
+            await converter.shutdown()
+
+
+def create_app(settings: Settings | None = None) -> FastAPI:
+    """Build and return the FastAPI application."""
+    settings = settings or get_settings()
+
+    app = FastAPI(
+        title=f"{settings.app_name} Service API",
+        description=(
+            "Convert PDF, PowerPoint, Word, Excel, images, audio, HTML, JSON, "
+            "CSV, XML, ZIP, and more into Markdown — optionally enhanced by an LLM."
+        ),
+        version=settings.app_version,
+        docs_url="/docs",
+        redoc_url="/redoc",
+        default_response_class=ORJSONResponse,
+        lifespan=_lifespan,
+    )
+    app.state.settings = settings
+
+    # CORS — fixes Bug A9 (allow_credentials with origins='*')
+    app.add_middleware(
+        CORSMiddleware,
+        allow_origins=settings.cors_origins,
+        allow_credentials=settings.cors_allow_credentials_safe,
+        allow_methods=["*"],
+        allow_headers=["*"],
+        expose_headers=["X-Request-ID"],
+    )
+    # GZip large responses (Section B.4)
+    app.add_middleware(GZipMiddleware, minimum_size=settings.gzip_min_size)
+
+    register_middleware(app, settings)
+    register_exception_handlers(app)
+
+    # ---- routes ---------------------------------------------------------
+    app.include_router(v1_router, prefix="/v1")
+
+    # ---- optional Gradio UI --------------------------------------------
+    _maybe_mount_gradio(app, settings)
+
+    return app
+
+
+def _maybe_mount_gradio(app: FastAPI, settings: Settings) -> None:
+    try:
+        import gradio as gr  # type: ignore
+
+        mount_fn = getattr(gr, "mount_gradio_app", None)
+        if mount_fn is None:
+            from gradio.routes import mount_gradio_app as mount_fn  # type: ignore
+        from .ui.gradio_app import build_interface
+    except Exception as exc:  # pragma: no cover - optional dependency
+        logger.warning("Gradio UI disabled (%s)", exc)
+        return
+
+    try:
+        interface = build_interface(settings, app)
+        mount_fn(app, interface, path="/")
+        logger.info("Gradio UI mounted at /")
+    except Exception as exc:  # pragma: no cover - defensive
+        logger.warning("Could not mount Gradio UI (%s)", exc)
+
+
+# Public ASGI app instance ----------------------------------------------------
+app: FastAPI = create_app()

docsifer/router.py

@@ -1,129 +0,0 @@
-import logging
-import json
-import tempfile
-import os
-# import aiohttp
-from pathlib import Path
-
-from fastapi import APIRouter, HTTPException, UploadFile, File, Form, BackgroundTasks
-from pydantic import BaseModel
-# from scuid import scuid
-
-from .service import DocsiferService
-from .analytics import Analytics
-
-logger = logging.getLogger(__name__)
-router = APIRouter(tags=["v1"], responses={404: {"description": "Not found"}})
-
-# Initialize analytics (aggregated under "docsifer")
-analytics = Analytics(
-    url=os.environ.get("REDIS_URL", "redis://localhost:6379/0"),
-    token=os.environ.get("REDIS_TOKEN", "***"),
-    sync_interval=30 * 60,  # e.g. 30 minutes
-)
-
-# Initialize the Docsifer service (using "gpt-4o" for token counting)
-docsifer_service = DocsiferService(model_name="gpt-4o")
-
-
-class ConvertResponse(BaseModel):
-    filename: str
-    markdown: str
-
-
-@router.post("/convert", response_model=ConvertResponse)
-async def convert_document(
-    background_tasks: BackgroundTasks,
-    file: UploadFile = File(None, description="File to convert"),
-    url: str = Form(
-        None, description="URL to convert (used only if no file is provided)"
-    ),
-    openai: str = Form("{}", description="OpenAI config as a JSON object"),
-    http: str = Form("{}", description="HTTP config as a JSON object"),
-    settings: str = Form("{}", description="Settings as a JSON object"),
-):
-    """
-    Convert a file or an HTML page from a URL into Markdown.
-    If 'file' is provided, it takes priority over 'url'.
-
-    - 'openai' is a JSON string with keys such as {"api_key": "...", "base_url": "..."}.
-    - 'settings' is a JSON string with keys such as {"cleanup": bool}.
-    """
-    try:
-        # Parse the JSON configuration parameters.
-        try:
-            openai_config = json.loads(openai) if openai else {}
-        except json.JSONDecodeError:
-            raise ValueError("Invalid JSON in 'openai' parameter.")
-
-        try:
-            http_config = json.loads(http) if http else {}
-        except json.JSONDecodeError:
-            raise ValueError("Invalid JSON in 'http' parameter.")
-
-        try:
-            settings_config = json.loads(settings) if settings else {}
-        except json.JSONDecodeError:
-            raise ValueError("Invalid JSON in 'settings' parameter.")
-
-        cleanup = settings_config.get("cleanup", True)
-
-        # If a file is provided, use it; otherwise, fetch the content from the URL.
-        if file is not None:
-            with tempfile.TemporaryDirectory() as tmpdir:
-                temp_path = Path(tmpdir) / file.filename
-                contents = await file.read()
-                temp_path.write_bytes(contents)
-                result, token_count = await docsifer_service.convert_file(
-                    source=str(temp_path),
-                    openai_config=openai_config,
-                    http_config=http_config,
-                    cleanup=cleanup,
-                )
-        elif url:
-            # async with aiohttp.ClientSession() as session:
-            #     async with session.get(url) as resp:
-            #         if resp.status != 200:
-            #             raise ValueError(f"Failed to fetch URL: status {resp.status}")
-            #         data = await resp.read()
-            # with tempfile.TemporaryDirectory() as tmpdir:
-            #     temp_path = Path(tmpdir) / f"{scuid()}.html"
-            #     temp_path.write_bytes(data)
-            #     result, token_count = await docsifer_service.convert_file(
-            #         source=str(temp_path),
-            #         openai_config=openai_config,
-            #         cleanup=cleanup,
-            #     )
-            result, token_count = await docsifer_service.convert_file(
-                source=str(url),
-                openai_config=openai_config,
-                http_config=http_config,
-                cleanup=cleanup,
-            )
-        else:
-            raise HTTPException(
-                status_code=400, detail="Provide either 'file' or 'url'."
-            )
-
-        # Record token usage in the background.
-        background_tasks.add_task(analytics.access, token_count)
-        return ConvertResponse(**result)
-
-    except Exception as e:
-        msg = f"Failed to convert content. Error: {str(e)}"
-        logger.error(msg)
-        raise HTTPException(status_code=500, detail=msg)
-
-
-@router.get("/stats")
-async def get_stats():
-    """
-    Return usage statistics (access, tokens) from the Analytics system.
-    """
-    try:
-        data = await analytics.stats()
-        return data
-    except Exception as e:
-        msg = f"Failed to fetch analytics stats: {str(e)}"
-        logger.error(msg)
-        raise HTTPException(status_code=500, detail=msg)

docsifer/safety/__init__.py

@@ -0,0 +1,17 @@
+"""Anti-crash safety primitives (resource guard, queue gate, watchdog, …)."""
+
+from .circuit_breaker import CircuitBreaker
+from .conversion_gate import ConversionGate
+from .disk_cleanup import disk_cleanup_loop
+from .memory_watchdog import memory_watchdog_loop
+from .per_ip_limiter import PerIPLimiter
+from .resource_guard import ResourceGuard
+
+__all__ = [
+    "CircuitBreaker",
+    "ConversionGate",
+    "PerIPLimiter",
+    "ResourceGuard",
+    "disk_cleanup_loop",
+    "memory_watchdog_loop",
+]

docsifer/safety/circuit_breaker.py

@@ -0,0 +1,66 @@
+"""Circuit breaker for upstream dependencies (Section N.8)."""
+
+from __future__ import annotations
+
+import asyncio
+import time
+from typing import Awaitable, Callable, TypeVar
+
+from ..exceptions import CircuitOpenError
+
+T = TypeVar("T")
+
+
+class CircuitBreaker:
+    """Three-state circuit breaker (closed → open → half-open)."""
+
+    def __init__(
+        self,
+        *,
+        failure_threshold: int = 5,
+        reset_timeout_sec: float = 60.0,
+    ) -> None:
+        self._failure_threshold = max(1, failure_threshold)
+        self._reset_timeout = max(1.0, reset_timeout_sec)
+        self._failures = 0
+        self._opened_at: float | None = None
+        self._half_open = False
+        self._lock = asyncio.Lock()
+
+    @property
+    def state(self) -> str:
+        if self._opened_at is None:
+            return "closed"
+        if self._half_open:
+            return "half-open"
+        return "open"
+
+    async def call(self, fn: Callable[..., Awaitable[T]], *args, **kwargs) -> T:
+        async with self._lock:
+            if self._opened_at is not None and not self._half_open:
+                if time.monotonic() - self._opened_at >= self._reset_timeout:
+                    self._half_open = True
+                else:
+                    raise CircuitOpenError("Upstream is unavailable")
+
+        try:
+            result = await fn(*args, **kwargs)
+        except Exception:
+            await self._on_failure()
+            raise
+        else:
+            await self._on_success()
+            return result
+
+    async def _on_success(self) -> None:
+        async with self._lock:
+            self._failures = 0
+            self._opened_at = None
+            self._half_open = False
+
+    async def _on_failure(self) -> None:
+        async with self._lock:
+            self._failures += 1
+            if self._failures >= self._failure_threshold:
+                self._opened_at = time.monotonic()
+                self._half_open = False

docsifer/safety/conversion_gate.py

@@ -0,0 +1,62 @@
+"""Bounded global concurrency / queue (Section N.5)."""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+from typing import AsyncIterator
+
+from ..exceptions import QueueFullError
+
+
+class ConversionGate:
+    """Cap concurrent conversions and queue depth, fail fast otherwise."""
+
+    def __init__(self, *, max_concurrent: int, max_queue: int) -> None:
+        if max_concurrent <= 0:
+            raise ValueError("max_concurrent must be positive")
+        self._semaphore = asyncio.Semaphore(max_concurrent)
+        self._max_concurrent = max_concurrent
+        self._max_queue = max(0, max_queue)
+        self._waiting = 0
+        self._inflight = 0
+        self._lock = asyncio.Lock()
+
+    @property
+    def stats(self) -> dict[str, int]:
+        return {
+            "max_concurrent": self._max_concurrent,
+            "max_queue": self._max_queue,
+            "waiting": self._waiting,
+            "inflight": self._inflight,
+        }
+
+    @contextlib.asynccontextmanager
+    async def acquire(self) -> AsyncIterator[None]:
+        async with self._lock:
+            in_use = self._inflight + self._waiting
+            # Queue is engaged only after capacity is fully used.
+            queued = max(0, in_use - self._max_concurrent)
+            if queued >= self._max_queue and in_use >= self._max_concurrent:
+                raise QueueFullError(
+                    "Conversion queue full",
+                    details={"max_queue": self._max_queue},
+                )
+            self._waiting += 1
+
+        try:
+            await self._semaphore.acquire()
+        except BaseException:
+            async with self._lock:
+                self._waiting -= 1
+            raise
+
+        async with self._lock:
+            self._waiting -= 1
+            self._inflight += 1
+        try:
+            yield
+        finally:
+            async with self._lock:
+                self._inflight -= 1
+            self._semaphore.release()

docsifer/safety/disk_cleanup.py

@@ -0,0 +1,48 @@
+"""Background disk cleanup loop (Section N.11).
+
+Removes stale files matching ``docsifer-*`` in the configured tmp directory so
+long-running containers don't accumulate gigabytes of leftovers.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import time
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+
+async def disk_cleanup_loop(
+    *,
+    tmp_dir: Path,
+    ttl_sec: int = 3600,
+    interval_sec: int = 600,
+    pattern: str = "docsifer-*",
+    stop_event: asyncio.Event | None = None,
+) -> None:
+    """Sweep ``tmp_dir`` every ``interval_sec`` and unlink stale temp files."""
+    stop = stop_event or asyncio.Event()
+    while not stop.is_set():
+        try:
+            await asyncio.wait_for(stop.wait(), timeout=interval_sec)
+            return
+        except asyncio.TimeoutError:
+            pass
+
+        try:
+            now = time.time()
+            removed = 0
+            for entry in tmp_dir.glob(pattern):
+                try:
+                    if now - entry.stat().st_mtime > ttl_sec:
+                        if entry.is_file():
+                            entry.unlink(missing_ok=True)
+                            removed += 1
+                except OSError as exc:
+                    logger.debug("Could not remove %s: %s", entry, exc)
+            if removed:
+                logger.info("Disk cleanup removed %d stale file(s)", removed)
+        except Exception as exc:
+            logger.warning("Disk cleanup failed: %s", exc)

docsifer/safety/memory_watchdog.py

@@ -0,0 +1,56 @@
+"""Memory watchdog — graceful self-restart before the kernel OOM-kills us
+(Section N.13).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import os
+import signal
+
+logger = logging.getLogger(__name__)
+
+try:  # pragma: no cover - environment dependent
+    import psutil
+
+    _HAS_PSUTIL = True
+except Exception:  # pragma: no cover
+    psutil = None  # type: ignore[assignment]
+    _HAS_PSUTIL = False
+
+
+async def memory_watchdog_loop(
+    *,
+    threshold_pct: float = 90.0,
+    interval_sec: int = 30,
+    stop_event: asyncio.Event | None = None,
+) -> None:
+    """Periodically inspect process RSS and request graceful restart on overrun."""
+    if not _HAS_PSUTIL:
+        logger.info("psutil unavailable; memory watchdog disabled")
+        return
+
+    proc = psutil.Process()
+    stop = stop_event or asyncio.Event()
+    while not stop.is_set():
+        try:
+            await asyncio.wait_for(stop.wait(), timeout=interval_sec)
+            return
+        except asyncio.TimeoutError:
+            pass
+
+        try:
+            usage = proc.memory_percent()
+        except Exception as exc:  # pragma: no cover - defensive
+            logger.debug("memory_percent failed: %s", exc)
+            continue
+
+        if usage > threshold_pct:
+            logger.warning(
+                "Memory pressure %.1f%% > %.1f%% — sending SIGTERM",
+                usage,
+                threshold_pct,
+            )
+            os.kill(os.getpid(), signal.SIGTERM)
+            return

docsifer/safety/per_ip_limiter.py

@@ -0,0 +1,65 @@
+"""Per-IP fairness — limit how many conversions a single IP can run at once
+(Section N.6).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import time
+from typing import AsyncIterator
+
+from ..exceptions import TooManyRequestsError
+
+
+class _Slot:
+    __slots__ = ("active", "last_seen")
+
+    def __init__(self) -> None:
+        self.active = 0
+        self.last_seen = time.monotonic()
+
+
+class PerIPLimiter:
+    """Bound the number of concurrent in-flight requests per IP."""
+
+    def __init__(self, *, max_per_ip: int, idle_evict_sec: int = 600) -> None:
+        self._max_per_ip = max(1, max_per_ip)
+        self._slots: dict[str, _Slot] = {}
+        self._lock = asyncio.Lock()
+        self._idle_evict_sec = idle_evict_sec
+
+    @contextlib.asynccontextmanager
+    async def acquire(self, ip: str) -> AsyncIterator[None]:
+        await self._enter(ip)
+        try:
+            yield
+        finally:
+            await self._leave(ip)
+
+    async def _enter(self, ip: str) -> None:
+        async with self._lock:
+            self._evict_idle()
+            slot = self._slots.setdefault(ip, _Slot())
+            if slot.active >= self._max_per_ip:
+                raise TooManyRequestsError(
+                    "Too many concurrent requests for this client",
+                    details={"max_per_ip": self._max_per_ip},
+                )
+            slot.active += 1
+            slot.last_seen = time.monotonic()
+
+    async def _leave(self, ip: str) -> None:
+        async with self._lock:
+            slot = self._slots.get(ip)
+            if slot is None:
+                return
+            slot.active = max(0, slot.active - 1)
+            slot.last_seen = time.monotonic()
+
+    def _evict_idle(self) -> None:
+        now = time.monotonic()
+        cutoff = now - self._idle_evict_sec
+        stale = [ip for ip, s in self._slots.items() if s.active == 0 and s.last_seen < cutoff]
+        for ip in stale:
+            self._slots.pop(ip, None)

docsifer/safety/resource_guard.py

@@ -0,0 +1,67 @@
+"""Memory / disk admission control to prevent OOM (Section N.4)."""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+
+from ..exceptions import ResourceExhaustedError
+
+logger = logging.getLogger(__name__)
+
+try:  # pragma: no cover - environment dependent
+    import psutil
+
+    _HAS_PSUTIL = True
+except Exception:  # pragma: no cover
+    psutil = None  # type: ignore[assignment]
+    _HAS_PSUTIL = False
+
+
+class ResourceGuard:
+    """Reject new conversions when the host is approaching exhaustion."""
+
+    def __init__(
+        self,
+        *,
+        min_free_memory_mb: int,
+        min_free_disk_mb: int,
+        tmp_dir: Path,
+        memory_overhead_factor: int = 3,
+        memory_overhead_floor_mb: int = 256,
+    ) -> None:
+        self._min_free_memory_mb = min_free_memory_mb
+        self._min_free_disk_mb = min_free_disk_mb
+        self._tmp_dir = tmp_dir
+        self._overhead_factor = memory_overhead_factor
+        self._overhead_floor_mb = memory_overhead_floor_mb
+
+    def check(self, expected_size_bytes: int = 0) -> None:
+        """Raise :class:`ResourceExhaustedError` when limits are exceeded."""
+        if not _HAS_PSUTIL:
+            return  # cannot enforce without psutil; fail-open
+
+        # Memory
+        avail_mb = psutil.virtual_memory().available // (1024 * 1024)
+        size_mb = expected_size_bytes // (1024 * 1024)
+        needed_mb = max(
+            self._min_free_memory_mb,
+            size_mb * self._overhead_factor + self._overhead_floor_mb,
+        )
+        if avail_mb < needed_mb:
+            raise ResourceExhaustedError(
+                f"Insufficient memory: need {needed_mb} MB, have {avail_mb} MB",
+                details={"available_mb": avail_mb, "needed_mb": needed_mb},
+            )
+
+        # Disk
+        try:
+            disk_free_mb = psutil.disk_usage(str(self._tmp_dir)).free // (1024 * 1024)
+        except Exception as exc:
+            logger.debug("disk_usage check failed: %s", exc)
+            return
+        if disk_free_mb < self._min_free_disk_mb:
+            raise ResourceExhaustedError(
+                f"Insufficient disk space: {disk_free_mb} MB free",
+                details={"disk_free_mb": disk_free_mb},
+            )

docsifer/service.py

@@ -1,227 +0,0 @@
-from __future__ import annotations
-
-import asyncio
-import logging
-
-# import tempfile
-
-import requests.cookies
-import magic
-import mimetypes
-import requests
-from pathlib import Path
-from typing import Optional, Dict, Tuple, Any
-from scuid import scuid
-
-import tiktoken
-from pyquery import PyQuery as pq
-from markitdown import MarkItDown
-from openai import OpenAI
-
-logger = logging.getLogger(__name__)
-logging.basicConfig(level=logging.INFO)
-
-
-class DocsiferService:
-    """
-    A service that converts local files to Markdown using MarkItDown,
-    optionally with an OpenAI LLM for advanced extraction.
-    Token counting uses a tiktoken encoder (heuristically with the provided model).
-    """
-
-    def __init__(self, model_name: str = "gpt-4o"):
-        """
-        Initialize the DocsiferService with a basic MarkItDown instance
-        and a tiktoken encoder for counting tokens using the provided model.
-        """
-        self._basic_markitdown = MarkItDown()  # MarkItDown without LLM
-
-        # Use the given model for token counting
-        try:
-            self._encoder = tiktoken.encoding_for_model(model_name)
-        except Exception as e:
-            logger.warning(
-                "Error loading tiktoken model '%s': %s. Falling back to 'gpt-3.5-turbo-0301'.",
-                model_name,
-                e,
-            )
-            self._encoder = tiktoken.encoding_for_model("gpt-3.5-turbo-0301")
-
-        logger.info("DocsiferService initialized with token model '%s'.", model_name)
-
-    def _init_markitdown_with_llm(self, openai_config: Dict[str, Any]) -> MarkItDown:
-        """
-        Initialize a MarkItDown instance configured with an OpenAI LLM if an API key is provided.
-
-        Args:
-            openai_config: A dictionary containing OpenAI configuration (e.g., api_key, model, base_url).
-
-        Returns:
-            A MarkItDown instance configured with the OpenAI client, or the basic instance if no key is provided.
-        """
-        api_key = openai_config.get("api_key", "")
-        if not api_key:
-            logger.info("No OpenAI API key provided. Using basic MarkItDown.")
-            return self._basic_markitdown
-
-        model = openai_config.get("model", "gpt-4o-mini")
-        base_url = openai_config.get("base_url", "https://api.openai.com/v1")
-        client = OpenAI(api_key=api_key, base_url=base_url)
-
-        logger.info("Initialized OpenAI with base_url=%s", base_url)
-        return MarkItDown(llm_client=client, llm_model=model)
-
-    def _maybe_cleanup_html(self, html_file: Path) -> None:
-        """
-        If the file is HTML, remove <style> tags and hidden elements to clean up the content.
-
-        Args:
-            html_file: Path to the HTML file.
-        """
-        try:
-            content = html_file.read_text(encoding="utf-8", errors="ignore")
-            d = pq(content)
-            # Remove hidden elements and inline styles that hide content.
-            d(":hidden").remove()
-            d("[style='display:none']").remove()
-            d('*[style*="display:none"]').remove()
-            d("style").remove()
-            cleaned_html = str(d).strip()
-            html_file.write_text(cleaned_html, encoding="utf-8")
-        except Exception as e:
-            logger.error("HTML cleanup failed for %s: %s", html_file, e)
-
-    def _count_tokens(self, text: str) -> int:
-        """
-        Count tokens in the given text using the configured tiktoken encoder.
-        Falls back to a whitespace-based count if an error occurs.
-
-        Args:
-            text: The text to count tokens in.
-
-        Returns:
-            The number of tokens.
-        """
-        try:
-            return len(self._encoder.encode(text))
-        except Exception as e:
-            logger.warning(
-                "Token counting failed, fallback to whitespace. Error: %s", e
-            )
-            return len(text.split())
-
-    def _convert_sync(
-        self,
-        source: str,
-        openai_config: Optional[dict] = None,
-        http_config: Optional[dict] = None,
-        cleanup: bool = True,
-    ) -> Tuple[Dict[str, str], int]:
-        """
-        Synchronously convert a file at `file_path` to Markdown.
-        This helper method performs blocking file I/O, MIME detection, temporary file handling,
-        optional HTML cleanup, and MarkItDown conversion.
-
-        Args:
-            source: Path to the source file or URL to fetch content from.
-            openai_config: Optional dictionary with OpenAI configuration.
-            http_config: Optional dictionary with HTTP configuration.
-            cleanup: Whether to perform HTML cleanup if the file is an HTML file.
-
-        Returns:
-            A tuple containing a dictionary with keys "filename" and "markdown", and the token count.
-        """
-        if source.startswith("http"):
-            filename = f"{scuid()}.html"
-        else:
-            src = Path(source)
-            if not src.exists():
-                raise FileNotFoundError(f"File not found: {source}")
-
-            logger.info("Converting file: %s (cleanup=%s)", source, cleanup)
-
-            mime_type = magic.from_file(str(src), mime=True)
-            guessed_ext = mimetypes.guess_extension(mime_type) or ".tmp"
-            if not mime_type:
-                logger.warning(f"Could not detect file type for: {src}")
-                new_filename = src.name
-            else:
-                logger.debug(f"Detected MIME type '{mime_type}' for: {src}")
-                new_filename = f"{src.stem}{guessed_ext}"
-            tmp_path = src.parent / new_filename
-            tmp_path.write_bytes(src.read_bytes())
-            # src.unlink()
-
-            logger.info(
-                "Using temp file: %s, MIME type: %s, Guessed ext: %s, Existing: %s, Source size: %s",
-                tmp_path,
-                mime_type,
-                guessed_ext,
-                tmp_path.exists(),
-                src.stat().st_size,
-            )
-
-            # Perform HTML cleanup if requested.
-            if cleanup and guessed_ext.lower() in (".html", ".htm"):
-                self._maybe_cleanup_html(tmp_path)
-
-            filename = new_filename
-            source = tmp_path
-
-        # Decide whether to use LLM-enhanced conversion or the basic converter.
-        if openai_config and openai_config.get("api_key"):
-            md_converter = self._init_markitdown_with_llm(openai_config)
-        else:
-            md_converter = self._basic_markitdown
-
-        # Load cookies if provided in the HTTP config.
-        if http_config:
-            if "cookies" in http_config:
-                requests.cookies.cookiejar_from_dict(
-                    http_config["cookies"],
-                    requests.cookies.RequestsCookieJar,
-                    overwrite=True,
-                )
-
-        try:
-            result_obj = md_converter.convert(source)
-        except Exception as e:
-            logger.error("MarkItDown conversion failed: %s", e)
-            raise RuntimeError(f"Conversion failed for '{source}': {e}")
-
-        if isinstance(source, Path) and source.exists():
-            source.unlink()
-
-        # Count tokens in the resulting markdown text.
-        token_count = self._count_tokens(result_obj.text_content)
-
-        result_dict = {
-            "filename": filename,
-            "markdown": result_obj.text_content,
-        }
-        return result_dict, token_count
-
-    async def convert_file(
-        self,
-        source: str,
-        openai_config: Optional[dict] = None,
-        http_config: Optional[dict] = None,
-        cleanup: bool = True,
-    ) -> Tuple[Dict[str, str], int]:
-        """
-        Asynchronously convert a file at `source` to Markdown.
-        This method offloads the blocking conversion process to a separate thread.
-
-        Args:
-            source: Path to the file to convert or a URL to fetch content from.
-            openai_config: Optional OpenAI configuration dictionary.
-            http_config: Optional HTTP configuration dictionary.
-            cleanup: Whether to perform HTML cleanup if applicable.
-
-        Returns:
-            A tuple containing the result dictionary (with keys "filename" and "markdown")
-            and the token count.
-        """
-        return await asyncio.to_thread(
-            self._convert_sync, source, openai_config, http_config, cleanup
-        )

docsifer/ui/__init__.py

@@ -0,0 +1,5 @@
+"""Gradio UI layer (optional)."""
+
+from .gradio_app import build_interface
+
+__all__ = ["build_interface"]

docsifer/ui/gradio_app.py

@@ -0,0 +1,385 @@
+"""Gradio playground.
+
+A refined, focused UI: one screen for conversion, one for stats. The handlers
+call :class:`DocsiferService` directly (no HTTP loopback), so the browser
+talks to FastAPI which talks to the in-process service.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import secrets
+import shutil
+import tempfile
+from pathlib import Path
+from typing import Any
+
+import gradio as gr
+import pandas as pd
+from fastapi import FastAPI
+
+from ..analytics import AnalyticsService
+from ..config import Settings
+from ..core.service import DocsiferService
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Theme & copy
+# ---------------------------------------------------------------------------
+_THEME = gr.themes.Soft(
+    primary_hue=gr.themes.colors.indigo,
+    secondary_hue=gr.themes.colors.violet,
+    neutral_hue=gr.themes.colors.slate,
+    radius_size=gr.themes.sizes.radius_lg,
+    font=("Inter", "ui-sans-serif", "system-ui", "sans-serif"),
+)
+
+_HERO = """
+<div style="text-align:center; padding: 8px 0 24px;">
+  <h1 style="margin:0; font-weight:700; letter-spacing:-0.02em;">📚 Docsifer</h1>
+  <p style="margin:6px 0 0; opacity:0.8; font-size:1.05rem;">
+    Convert documents into clean, LLM-ready Markdown.
+  </p>
+  <p style="margin:4px 0 0; opacity:0.6; font-size:0.9rem;">
+    PDF · Word · PowerPoint · Excel · HTML · Audio · Image · CSV · JSON · ZIP
+  </p>
+</div>
+"""
+
+_FOOTER = """
+<div style="text-align:center; opacity:0.55; font-size:0.85rem; padding: 16px 0 4px;">
+  Powered by <a href="https://github.com/microsoft/markitdown" target="_blank">MarkItDown</a> ·
+  <a href="https://github.com/lh0x00/docsifer" target="_blank">Source</a> ·
+  Files are processed in-memory and discarded immediately.
+</div>
+"""
+
+_CSS = """
+.gradio-container { max-width: 1200px !important; margin: auto; }
+#convert-btn { font-weight: 600; letter-spacing: 0.01em; }
+.markdown-out textarea { font-family: ui-monospace, "JetBrains Mono", Menlo, monospace; font-size: 0.92rem; }
+.compact-tabs button.selected { font-weight: 600; }
+"""
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+def _df_from_bucket(bucket: dict[str, dict[str, int]]) -> pd.DataFrame:
+    columns = ["Model", "Total", "Daily", "Weekly", "Monthly", "Yearly"]
+    rows: list[list[Any]] = []
+    models: set[str] = set()
+    for period in ("total", "daily", "weekly", "monthly", "yearly"):
+        models.update((bucket.get(period) or {}).keys())
+    for model in sorted(models):
+        rows.append(
+            [
+                model,
+                bucket.get("total", {}).get(model, 0),
+                bucket.get("daily", {}).get(model, 0),
+                bucket.get("weekly", {}).get(model, 0),
+                bucket.get("monthly", {}).get(model, 0),
+                bucket.get("yearly", {}).get(model, 0),
+            ]
+        )
+    if not rows:
+        rows.append(["docsifer", 0, 0, 0, 0, 0])
+    return pd.DataFrame(rows, columns=columns)
+
+
+def _build_openai_dict(
+    base_url: str | None,
+    api_key: str | None,
+    model: str | None,
+) -> dict[str, Any] | None:
+    """Only forward the OpenAI config when an API key is provided (Bug A1)."""
+    api_key = (api_key or "").strip()
+    if not api_key:
+        return None
+    cfg: dict[str, Any] = {"api_key": api_key}
+    if base_url and base_url.strip():
+        cfg["base_url"] = base_url.strip()
+    if model and model.strip():
+        cfg["model"] = model.strip()
+    return cfg
+
+
+def _parse_cookies(raw: str | None) -> dict[str, Any] | None:
+    raw = (raw or "").strip()
+    if not raw:
+        return None
+    try:
+        return json.loads(raw)
+    except json.JSONDecodeError as exc:
+        raise ValueError(f"Invalid JSON for HTTP cookies: {exc}") from exc
+
+
+def _write_markdown_file(text: str, *, tmp_dir: Path, stem: str) -> str:
+    safe_stem = "".join(c for c in stem if c.isalnum() or c in "-_") or "document"
+    name = f"docsifer-{safe_stem}-{secrets.token_hex(4)}.md"
+    dst = tmp_dir / name
+    dst.write_text(text, encoding="utf-8")
+    return str(dst)
+
+
+def _status(text: str, *, level: str = "info") -> str:
+    palette = {
+        "info": "#6366f1",
+        "ok": "#16a34a",
+        "warn": "#d97706",
+        "err": "#dc2626",
+    }
+    color = palette.get(level, palette["info"])
+    return (
+        f'<div style="padding:10px 14px;border-radius:10px;'
+        f'background:{color}14;color:{color};font-size:0.9rem;'
+        f'border:1px solid {color}30;">{text}</div>'
+    )
+
+
+# ---------------------------------------------------------------------------
+# Builder
+# ---------------------------------------------------------------------------
+def build_interface(settings: Settings, app: FastAPI) -> gr.Blocks:
+    """Construct the Gradio interface bound to the running FastAPI app."""
+
+    def _service() -> DocsiferService:
+        return app.state.converter  # type: ignore[no-any-return]
+
+    def _analytics() -> AnalyticsService:
+        return app.state.analytics  # type: ignore[no-any-return]
+
+    async def _run_conversion(
+        file_path: str | None,
+        url_str: str | None,
+        base_url: str,
+        api_key: str,
+        model_id: str,
+        cleanup: bool,
+        http_cookies: str,
+    ) -> tuple[str, str | None, str]:
+        if not file_path and not (url_str and url_str.strip()):
+            return "", None, _status("Please provide a file or a URL.", level="warn")
+
+        try:
+            openai_cfg = _build_openai_dict(base_url, api_key, model_id)
+            cookies = _parse_cookies(http_cookies)
+            http_cfg = {"cookies": cookies} if cookies else None
+        except ValueError as exc:
+            return "", None, _status(str(exc), level="err")
+
+        managed_dir = Path(tempfile.mkdtemp(prefix="docsifer-ui-", dir=settings.tmp_dir))
+        try:
+            if file_path:
+                src = Path(file_path)
+                staged = managed_dir / src.name
+                shutil.copyfile(src, staged)
+                source: str | Path = staged
+                stem = src.stem
+            else:
+                source = (url_str or "").strip()
+                stem = "url"
+
+            try:
+                result = await asyncio.wait_for(
+                    _service().convert_file(
+                        source,
+                        openai_config=openai_cfg,
+                        http_config=http_cfg,
+                        cleanup_html=bool(cleanup),
+                    ),
+                    timeout=settings.request_timeout_sec,
+                )
+            except asyncio.TimeoutError:
+                return (
+                    "",
+                    None,
+                    _status(
+                        f"Conversion timed out after {settings.request_timeout_sec}s. "
+                        "Try a smaller file.",
+                        level="err",
+                    ),
+                )
+            except Exception as exc:  # noqa: BLE001
+                logger.exception("Gradio conversion failed")
+                return "", None, _status(f"Conversion failed: {exc}", level="err")
+
+            asyncio.create_task(_record_access(result.token_count))
+            md_path = _write_markdown_file(
+                result.markdown, tmp_dir=settings.tmp_dir, stem=stem
+            )
+            byte_len = len(result.markdown.encode("utf-8"))
+            ok_msg = (
+                f"Done — {len(result.markdown):,} chars · "
+                f"{byte_len/1024:.1f} KB · ~{result.token_count:,} tokens"
+            )
+            return result.markdown, md_path, _status(ok_msg, level="ok")
+        finally:
+            shutil.rmtree(managed_dir, ignore_errors=True)
+
+    async def _record_access(tokens: int) -> None:
+        try:
+            await _analytics().access(tokens)
+        except Exception:
+            logger.exception("Analytics access failed (UI)")
+
+    async def _fetch_stats() -> tuple[pd.DataFrame, pd.DataFrame, str]:
+        snap = await _analytics().stats()
+        access_df = _df_from_bucket(snap.get("access", {}))
+        tokens_df = _df_from_bucket(snap.get("tokens", {}))
+        msg = (
+            _status("Analytics backend healthy.", level="ok")
+            if snap.get("healthy", True)
+            else _status("Analytics degraded — using in-memory snapshot.", level="warn")
+        )
+        return access_df, tokens_df, msg
+
+    # ------------------------------------------------------------------
+    # Layout
+    # ------------------------------------------------------------------
+    with gr.Blocks(
+        title="Docsifer — Document to Markdown",
+        theme=_THEME,
+        css=_CSS,
+        analytics_enabled=False,
+    ) as demo:
+        gr.HTML(_HERO)
+
+        with gr.Tabs(elem_classes=["compact-tabs"]):
+            # ---- Convert tab ------------------------------------------------
+            with gr.Tab("Convert"):
+                with gr.Row(equal_height=False):
+                    with gr.Column(scale=5, min_width=320):
+                        file_input = gr.File(
+                            label="File",
+                            file_types=settings.allowed_extensions,
+                            type="filepath",
+                            file_count="single",
+                            height=120,
+                        )
+                        url_input = gr.Textbox(
+                            label="…or a URL",
+                            placeholder="https://example.com/article",
+                            lines=1,
+                        )
+
+                        with gr.Accordion("LLM (optional)", open=False):
+                            gr.Markdown(
+                                "Provide an OpenAI-compatible key for OCR / "
+                                "layout / transcription. Leave blank for basic mode.",
+                                elem_classes=["help-text"],
+                            )
+                            openai_api_key = gr.Textbox(
+                                label="API key",
+                                placeholder="sk-…",
+                                type="password",
+                            )
+                            with gr.Row():
+                                openai_base_url = gr.Textbox(
+                                    label="Base URL",
+                                    value=settings.default_openai_base_url,
+                                    scale=3,
+                                )
+                                openai_model = gr.Textbox(
+                                    label="Model",
+                                    value=settings.default_openai_model,
+                                    scale=2,
+                                )
+
+                        with gr.Accordion("Advanced", open=False):
+                            cleanup_toggle = gr.Checkbox(
+                                label="Strip script / style / hidden HTML",
+                                value=True,
+                            )
+                            http_cookies = gr.Textbox(
+                                label="HTTP cookies (JSON, optional)",
+                                placeholder='{"session": "abc..."}',
+                                lines=2,
+                            )
+
+                        convert_btn = gr.Button(
+                            "Convert to Markdown",
+                            variant="primary",
+                            elem_id="convert-btn",
+                            size="lg",
+                        )
+                        status_box = gr.HTML(value="", visible=True)
+
+                    with gr.Column(scale=7, min_width=380):
+                        output_md = gr.Textbox(
+                            label="Markdown",
+                            lines=24,
+                            max_lines=40,
+                            interactive=True,
+                            show_copy_button=True,
+                            elem_classes=["markdown-out"],
+                            placeholder="Your converted Markdown will appear here…",
+                        )
+                        download_file = gr.File(
+                            label="Download",
+                            interactive=False,
+                            height=70,
+                        )
+
+                convert_btn.click(
+                    fn=_run_conversion,
+                    inputs=[
+                        file_input,
+                        url_input,
+                        openai_base_url,
+                        openai_api_key,
+                        openai_model,
+                        cleanup_toggle,
+                        http_cookies,
+                    ],
+                    outputs=[output_md, download_file, status_box],
+                    api_name=False,
+                    show_progress="full",
+                )
+
+            # ---- Stats tab --------------------------------------------------
+            with gr.Tab("Stats"):
+                gr.Markdown(
+                    "Aggregated usage counters (per model and period).",
+                )
+                stats_status = gr.HTML(value="")
+                stats_btn = gr.Button("Refresh", variant="secondary", size="sm")
+                with gr.Row():
+                    access_df = gr.DataFrame(
+                        label="Access count",
+                        headers=["Model", "Total", "Daily", "Weekly", "Monthly", "Yearly"],
+                        interactive=False,
+                        wrap=True,
+                    )
+                    tokens_df = gr.DataFrame(
+                        label="Token usage",
+                        headers=["Model", "Total", "Daily", "Weekly", "Monthly", "Yearly"],
+                        interactive=False,
+                        wrap=True,
+                    )
+                stats_btn.click(
+                    fn=_fetch_stats,
+                    inputs=[],
+                    outputs=[access_df, tokens_df, stats_status],
+                    api_name=False,
+                )
+                # Also load on first render for a friendlier first impression.
+                demo.load(
+                    fn=_fetch_stats,
+                    inputs=[],
+                    outputs=[access_df, tokens_df, stats_status],
+                )
+
+        gr.HTML(_FOOTER)
+
+    # Bounded queue so the UI cannot saturate the underlying gate.
+    demo.queue(
+        max_size=settings.max_queue_depth,
+        default_concurrency_limit=settings.max_concurrent_conversions,
+        api_open=False,
+    )
+    return demo

pyproject.toml

@@ -1,17 +1,97 @@
+[build-system]
+requires = ["poetry-core>=1.6.0"]
+build-backend = "poetry.core.masonry.api"
+
 [tool.poetry]
 name = "docsifer"
-version = "1.0.0"
-description = "Docsifer is a powerful tool for converting various data formats into Markdown for applications such as indexing, text analysis, and more. It supports PDF, PowerPoint, Word, Excel, Images, Audio, HTML, and other text-based formats, and leverages Large Language Models (LLMs) to enhance performance."
+version = "1.1.0"
+description = "Convert PDF, PPT, Word, Excel, images, audio, HTML, JSON, CSV, XML, ZIP and more to Markdown — with optional LLM enhancement."
 authors = ["Hieu Lam <lamhieu.vk@gmail.com>"]
 readme = "README.md"
 homepage = "https://github.com/lh0x00/docsifer"
 repository = "https://github.com/lh0x00/docsifer"
 license = "MIT"
+packages = [{ include = "docsifer" }]
 
 [tool.poetry.dependencies]
 python = "^3.10"
+fastapi = ">=0.110,<1.0"
+uvicorn = { version = ">=0.27,<1.0", extras = ["standard"] }
+gunicorn = ">=21.2,<23.0"
+pydantic = ">=2.6,<3.0"
+pydantic-settings = ">=2.2,<3.0"
+orjson = ">=3.10,<4.0"
+python-multipart = ">=0.0.9"
+httpx = ">=0.27,<1.0"
+openai = ">=1.40,<2.0"
+tiktoken = ">=0.7,<1.0"
+markitdown = "0.0.1a3"
+selectolax = ">=0.3.21"
+python-magic = "0.4.27"
+upstash-redis = ">=1.2,<2.0"
+cachetools = ">=5.3,<6.0"
+psutil = ">=5.9,<7.0"
+gradio = { version = ">=4.40,<5.0", optional = true }
+pandas = { version = ">=2.0,<3.0", optional = true }
+requests = ">=2.31,<3.0"
 
+[tool.poetry.extras]
+ui = ["gradio", "pandas"]
 
-[build-system]
-requires = ["poetry-core"]
-build-backend = "poetry.core.masonry.api"
+[tool.poetry.group.dev.dependencies]
+pytest = ">=8.0,<9.0"
+pytest-asyncio = ">=0.23,<1.0"
+ruff = ">=0.6,<1.0"
+mypy = ">=1.10,<2.0"
+types-requests = "*"
+fakeredis = ">=2.23,<3.0"
+
+# ---------------------------------------------------------------------------
+# Tooling
+# ---------------------------------------------------------------------------
+[tool.ruff]
+line-length = 100
+target-version = "py310"
+extend-exclude = ["poetry.lock"]
+
+[tool.ruff.lint]
+select = [
+  "E", "F", "W",  # pycodestyle / pyflakes
+  "I",            # isort
+  "B",            # flake8-bugbear
+  "UP",           # pyupgrade
+  "SIM",          # simplify
+  "RET",          # flake8-return
+  "C4",           # comprehensions
+  "ASYNC",        # async best practices
+]
+ignore = [
+  "E501",  # line length handled by formatter
+  "B008",  # FastAPI Depends() in defaults is idiomatic
+]
+
+[tool.ruff.lint.per-file-ignores]
+"tests/**" = ["B011", "S101"]
+
+[tool.ruff.format]
+quote-style = "double"
+
+[tool.mypy]
+python_version = "3.10"
+warn_unused_configs = true
+warn_redundant_casts = true
+warn_unused_ignores = true
+disallow_untyped_defs = false
+ignore_missing_imports = true
+files = ["docsifer"]
+
+[[tool.mypy.overrides]]
+module = ["docsifer.core.*", "docsifer.analytics.*", "docsifer.safety.*"]
+disallow_untyped_defs = true
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+testpaths = ["tests"]
+filterwarnings = [
+  "ignore::DeprecationWarning",
+]

requirements.txt

@@ -1,15 +1,26 @@
-gradio
-fastapi
-uvicorn
-requests
-pydantic
-cachetools
-scuid
-markitdown @ git+https://github.com/microsoft/markitdown@519fe172
-upstash_redis==1.2.0
-openai==1.59.7
-pyquery==2.0.1
-tiktoken==0.8.0
-aiohttp==3.11.11
+# Production runtime requirements (Hugging Face Spaces / Docker).
+# Keep in sync with [tool.poetry.dependencies] in pyproject.toml. For local
+# development we recommend ``poetry install``; this file exists for build
+# pipelines that don't speak Poetry.
+
+fastapi>=0.110,<1.0
+uvicorn[standard]>=0.27,<1.0
+gunicorn>=21.2,<23.0
+pydantic>=2.6,<3.0
+pydantic-settings>=2.2,<3.0
+orjson>=3.10,<4.0
+python-multipart>=0.0.9
+httpx>=0.27,<1.0
+openai>=1.40,<2.0
+tiktoken>=0.7,<1.0
+markitdown==0.0.1a3
+selectolax>=0.3.21
 python-magic==0.4.27
-youtube-transcript-api==0.6.3
+upstash-redis>=1.2,<2.0
+cachetools>=5.3,<6.0
+psutil>=5.9,<7.0
+requests>=2.31,<3.0
+
+# Optional UI extras
+gradio>=4.40,<5.0
+pandas>=2.0,<3.0