backend/scripts/dump_env_reference.py

from __future__ import annotations

import sys
from types import UnionType
from pathlib import Path
from typing import Any, Union, get_args, get_origin

BACKEND_ROOT = Path(__file__).resolve().parents[1]
if str(BACKEND_ROOT) not in sys.path:
    sys.path.insert(0, str(BACKEND_ROOT))

from pots_shutdown_tracker.config import Settings


SECRET_ENV_VARS = {
    "POTS_TRACKER_ADMIN_API_KEY",
    "POTS_TRACKER_DB_URL",
    "POTS_TRACKER_FCC_ECFS_API_KEY",
    "POTS_TRACKER_HF_STORAGE_TOKEN",
}

FIELD_DESCRIPTION_OVERRIDES = {
    "active_window_post_target_grace_days": "Days a notice can remain active after its target date.",
    "admin_api_key": "Shared secret required to authorize admin endpoints.",
    "admin_api_key_header": "Header name expected on admin requests.",
    "api_prefix": "Base URL prefix mounted in FastAPI.",
    "area_risk_airport_promotes_to_direct": (
        "When true, `<city> Airport` notices promote to a direct match for searches on `<city>`. "
        "Set false to surface them in the nearby-municipality section instead."
    ),
    "app_name": "Human-readable application name.",
    "auto_create_schema": "Create the database schema automatically at startup.",
    "bulk_lookup_concurrent_workers": "Maximum number of background bulk lookup jobs processed concurrently.",
    "bulk_lookup_file_size_mb": "Maximum uploaded bulk lookup workbook size in megabytes.",
    "bulk_lookup_max_rows": "Maximum data rows accepted in a bulk lookup workbook.",
    "bulk_lookup_retention_days": "Days to retain bulk lookup input and output blobs before cleanup.",
    "cors_allow_origins": "Comma-separated list of allowed browser origins.",
    "db_max_overflow": "Extra SQLAlchemy pool connections allowed above the base size.",
    "db_pool_pre_ping": "Ping pooled database connections before use.",
    "db_pool_recycle_seconds": "Lifetime of pooled database connections before recycle.",
    "db_pool_size": "Base SQLAlchemy database pool size.",
    "db_pool_timeout_seconds": "Seconds to wait for a pooled database connection.",
    "db_url": "Database connection string.",
    "enable_ai": "Enable AI-backed parsing, summarization, and search helpers.",
    "enable_weekly_jobs": "Enable the weekly APScheduler job set.",
    "fcc_ecfs_api_key": "Optional ECFS API key reserved for FCC watch discovery.",
    "fcc_ecfs_base_url": "Base URL for the FCC ECFS public API.",
    "fcc_watch_lookback_months": "Historical lookback window for FCC watch backfill phases.",
    "fcc_watch_proceedings": "Comma-separated ECFS proceeding numbers for targeted FCC watch scans.",
    "fetch_max_content_length_mb": "Maximum fetched body size in megabytes.",
    "fetch_read_timeout_seconds": "Read timeout for fetched responses.",
    "fetch_timeout_seconds": "Overall fetch timeout in seconds.",
    "frontend_dist_path": "Path to the built frontend bundle served by the app.",
    "hf_storage_path_prefix": "Path prefix used for stored blobs in the dataset repo.",
    "hf_storage_require_private": "Refuse to use a public Hugging Face dataset repo.",
    "hf_storage_repo_id": "Hugging Face dataset repo that stores crawler blobs.",
    "hf_storage_revision": "Revision used for Hugging Face dataset uploads and downloads.",
    "hf_storage_token": "Write token for the Hugging Face dataset repo.",
    "lookback_months": "Active corpus lookback window in months.",
    "log_level": "Application log level.",
    "openai_api_key": "OpenAI API key used by AI features.",
    "query_embedding_cache_enabled": "Enable caching for query embeddings.",
    "query_embedding_cache_size": "Maximum number of cached query embeddings.",
    "request_timing_enabled": "Emit request timing logs.",
    "run_migrations_on_startup": "Run Alembic migrations during app startup.",
    "scheduler_enabled_instances": "Number of instances allowed to run scheduler jobs.",
    "search_candidate_limit": "Maximum number of candidate notices considered during search.",
    "search_refinement_limit": "Maximum number of notices retained after search refinement.",
    "search_prewarm_enabled": "Enable search prewarm at startup.",
    "search_prewarm_queries": "Comma-separated search queries used for prewarming.",
    "search_result_cache_enabled": "Enable caching for search responses.",
    "search_result_cache_size": "Maximum number of cached search responses.",
    "search_result_cache_ttl_seconds": "TTL for cached search responses in seconds.",
    "search_trace_enabled": "Emit detailed search trace logs.",
    "search_vector_candidate_limit": "Maximum number of vector candidates considered during search.",
    "slow_request_threshold_ms": "Threshold for logging slow requests in milliseconds.",
    "source_coverage_matrix_file": "Path to the source coverage matrix JSON file.",
    "startup_db_wait_seconds": "Maximum time to wait for the database during startup.",
    "storage_backend": "Select the storage backend (`filesystem` or `huggingface_dataset`).",
    "storage_path": "Local storage and cache root.",
    "serve_frontend": "Serve the frontend bundle from FastAPI.",
    "ops_presets_file": "Path to the curated ops preset list.",
    "timezone": "Application timezone.",
    "trust_matrix_empty_threshold": "Maximum empty-matrix ratio before queryability is disabled.",
    "trust_stale_days": "Days after which the corpus is considered stale.",
    "user_agent": "User-Agent string used for outbound HTTP requests.",
    "weekly_schedule": "Cron expression for the weekly job schedule.",
}

SPECIAL_SCOPE_OVERRIDES = {
    "POTS_TRACKER_ADMIN_API_KEY": "hosted",
    "POTS_TRACKER_CORS_ALLOW_ORIGINS": "hosted",
    "POTS_TRACKER_FCC_ECFS_API_KEY": "hosted",
    "POTS_TRACKER_FCC_ECFS_BASE_URL": "hosted",
    "POTS_TRACKER_FCC_WATCH_LOOKBACK_MONTHS": "hosted",
    "POTS_TRACKER_FCC_WATCH_PROCEEDINGS": "hosted",
    "POTS_TRACKER_FRONTEND_DIST_PATH": "hosted",
    "POTS_TRACKER_HF_STORAGE_PATH_PREFIX": "hosted",
    "POTS_TRACKER_HF_STORAGE_REPO_ID": "hosted",
    "POTS_TRACKER_HF_STORAGE_REQUIRE_PRIVATE": "hosted",
    "POTS_TRACKER_HF_STORAGE_REVISION": "hosted",
    "POTS_TRACKER_HF_STORAGE_TOKEN": "hosted",
    "POTS_TRACKER_RUN_MIGRATIONS_ON_STARTUP": "hosted",
    "POTS_TRACKER_SERVE_FRONTEND": "hosted",
}

TOKEN_DISPLAY = {
    "ai": "AI",
    "api": "API",
    "att": "AT&T",
    "clli": "CLLI",
    "db": "DB",
    "fcc": "FCC",
    "hf": "HF",
    "id": "ID",
    "ip": "IP",
    "json": "JSON",
    "lbs": "lbs",
    "ops": "ops",
    "p90": "p90",
    "png": "PNG",
    "sql": "SQL",
    "ttl": "TTL",
    "ui": "UI",
    "url": "URL",
    "urls": "URLs",
}


def _humanize_token(token: str) -> str:
    if token in TOKEN_DISPLAY:
        return TOKEN_DISPLAY[token]
    if token.isdigit():
        return token
    if token.isupper():
        return token
    return token.replace("-", " ").capitalize()


def _humanize_name(name: str) -> str:
    return " ".join(_humanize_token(part) for part in name.split("_") if part)


def _render_description(field_name: str, env_name: str) -> str:
    if field_name in FIELD_DESCRIPTION_OVERRIDES:
        return FIELD_DESCRIPTION_OVERRIDES[field_name]

    if field_name.endswith("_index_urls"):
        carrier = _humanize_name(field_name.removesuffix("_index_urls"))
        return f"Comma-separated index URLs for {carrier}."
    if field_name.endswith("_tracker_urls"):
        carrier = _humanize_name(field_name.removesuffix("_tracker_urls"))
        return f"Comma-separated tracker URLs for {carrier}."
    if field_name.endswith("_seed_documents_file"):
        carrier = _humanize_name(field_name.removesuffix("_seed_documents_file"))
        return f"Path to the seed document URL file for {carrier}."
    if field_name.endswith("_document_urls_file"):
        carrier = _humanize_name(field_name.removesuffix("_document_urls_file"))
        return f"Path to extra document URLs for {carrier}."
    if field_name.endswith("_document_urls"):
        carrier = _humanize_name(field_name.removesuffix("_document_urls"))
        return f"Comma-separated explicit document URLs for {carrier}."
    if field_name.endswith("_urls_file"):
        carrier = _humanize_name(field_name.removesuffix("_urls_file"))
        return f"Path to an additional URL file for {carrier}."
    if field_name.endswith("_urls"):
        carrier = _humanize_name(field_name.removesuffix("_urls"))
        return f"Comma-separated URLs for {carrier}."

    return f"{_humanize_name(field_name)} setting."


def _render_scope(env_name: str) -> str:
    return SPECIAL_SCOPE_OVERRIDES.get(env_name, "both")


def _render_sensitivity(env_name: str) -> str:
    return "secret" if env_name in SECRET_ENV_VARS else "non-secret"


def _render_type(annotation: Any) -> str:
    if annotation is Any:
        return "Any"
    if annotation is type(None):
        return "None"
    origin = get_origin(annotation)
    if origin in {Union, UnionType}:
        return " | ".join(_render_type(arg) for arg in get_args(annotation))
    if isinstance(annotation, type):
        return annotation.__name__
    text = str(annotation).replace("typing.", "").replace("pathlib.", "")
    return text.replace("<class '", "").replace("'>", "")


def _render_default(value: Any) -> str:
    if value is None:
        return "`unset`"
    if isinstance(value, bool):
        return f"`{str(value).lower()}`"
    if isinstance(value, Path):
        text = value.as_posix()
        if not value.is_absolute() and not text.startswith("./"):
            text = f"./{text}"
        return f"`{text}`"
    if isinstance(value, str):
        return '`""`' if value == "" else f"`{value}`"
    return f"`{value}`"


def iter_reference_rows() -> list[dict[str, str]]:
    rows: list[dict[str, str]] = []
    for field_name, field in Settings.model_fields.items():
        env_name = field.alias or field_name
        if not env_name.startswith("POTS_TRACKER_"):
            continue
        rows.append(
            {
                "name": env_name,
                "type": _render_type(field.annotation),
                "default": _render_default(field.default),
                "description": _render_description(field_name, env_name),
                "scope": _render_scope(env_name),
                "sensitivity": _render_sensitivity(env_name),
            }
        )
    return rows


def render_env_reference() -> str:
    rows = iter_reference_rows()
    lines = [
        "# Environment Reference",
        "",
        "Generated from `backend/app/pots_shutdown_tracker/config.py`. Regenerate this file with `python backend/scripts/dump_env_reference.py > docs/ENV.md` after changing `Settings`.",
        "",
        "| Name | Type | Default | Description | Scope | Sensitivity |",
        "| --- | --- | --- | --- | --- | --- |",
    ]
    for row in rows:
        lines.append(
            f"| `{row['name']}` | {row['type']} | {row['default']} | {row['description']} | {row['scope']} | {row['sensitivity']} |"
        )
    lines.append("")
    return "\n".join(lines)


def main() -> None:
    sys.stdout.write(render_env_reference())


if __name__ == "__main__":
    main()