scripts/export_hf_space.py

from __future__ import annotations

import argparse
import hashlib
import json
import shutil
import uuid
from pathlib import Path
from typing import Iterable


ROOT_DIR = Path(__file__).resolve().parent.parent
DEFAULT_OUTPUT = ROOT_DIR / "outputs" / "huggingface-space"

FILES = [
    "requirements.txt",
    "requirements-silma.txt",
    "requirements-supertonic.txt",
    "requirements-paddleocr.txt",
    "requirements-paddleocr-vl.txt",
    "requirements-qari-ocr.txt",
    "requirements-tawkeed-ocr.txt",
    "requirements-katib-ocr.txt",
    "requirements-arabic-qwen-ocr.txt",
    "requirements-arabic-glm-ocr.txt",
    "requirements-baseer-ocr.txt",
]
DIRECTORIES = [
    "app",
    "api",
    "docs",
    "static",
    "scripts",
]
EXCLUDE_NAMES = {
    "__pycache__",
    ".pytest_cache",
    ".ruff_cache",
}
EXCLUDE_SUFFIXES = {
    ".pyc",
    ".pyo",
    ".pyd",
}
MANIFEST_NAME = ".export-manifest.json"


def should_copy(path: Path) -> bool:
    if path.name in EXCLUDE_NAMES:
        return False
    if path.suffix in EXCLUDE_SUFFIXES:
        return False
    return True


def copy_tree(source: Path, destination: Path) -> None:
    if destination.exists():
        shutil.rmtree(destination)
    shutil.copytree(source, destination, ignore=lambda _dir, names: [name for name in names if not should_copy(Path(name))])


def iter_manifest_source_files(root: Path | None = None) -> Iterable[Path]:
    root = root or ROOT_DIR
    for relative in FILES:
        path = root / relative
        if path.exists():
            yield path
    dockerfile = root / "Dockerfile.worker"
    if dockerfile.exists():
        yield dockerfile
    for relative in DIRECTORIES:
        base = root / relative
        if not base.exists():
            continue
        for path in sorted(base.rglob("*")):
            if path.is_file() and should_copy(path):
                yield path


def file_sha256(path: Path) -> str:
    digest = hashlib.sha256()
    with path.open("rb") as handle:
        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
            digest.update(chunk)
    return digest.hexdigest()


def build_export_manifest(root: Path | None = None) -> dict[str, object]:
    root = root or ROOT_DIR
    files: dict[str, str] = {}
    for path in iter_manifest_source_files(root):
        relative = path.relative_to(root).as_posix()
        if relative == "Dockerfile.worker":
            relative = "Dockerfile"
        files[relative] = file_sha256(path)
    return {
        "version": 1,
        "source": "ArabicTranslator",
        "files": files,
    }


def build_hf_space_bundle(output_dir: Path) -> list[str]:
    output_dir.mkdir(parents=True)
    copied: list[str] = []
    for relative in FILES:
        source = ROOT_DIR / relative
        destination = output_dir / relative
        destination.parent.mkdir(parents=True, exist_ok=True)
        shutil.copy2(source, destination)
        copied.append(relative)

    shutil.copy2(ROOT_DIR / "Dockerfile.worker", output_dir / "Dockerfile")
    copied.append("Dockerfile")

    for relative in DIRECTORIES:
        source = ROOT_DIR / relative
        destination = output_dir / relative
        copy_tree(source, destination)
        copied.append(relative)

    (output_dir / ".dockerignore").write_text(
        "\n".join(
            [
                ".git",
                ".env",
                ".venv",
                ".venv-*",
                "__pycache__",
                ".pytest_cache",
                "outputs",
                "uploads",
                "data",
                "test_pdfs",
                "tests",
                "*.pyc",
                "*.pyo",
                "*.pyd",
                "*.log",
                "",
            ]
        ),
        encoding="utf-8",
    )
    copied.append(".dockerignore")

    write_space_readme(output_dir / "README.md")
    copied.append("README.md")
    (output_dir / MANIFEST_NAME).write_text(
        json.dumps(build_export_manifest(), indent=2, sort_keys=True) + "\n",
        encoding="utf-8",
    )
    copied.append(MANIFEST_NAME)
    (output_dir / ".export-complete").write_text("ready\n", encoding="utf-8")
    copied.append(".export-complete")
    return copied


def export_hf_space(output_dir: Path = DEFAULT_OUTPUT, force: bool = False) -> dict[str, object]:
    output_dir = output_dir.resolve()
    if output_dir.exists() and not force:
        raise FileExistsError(f"{output_dir} already exists. Use --force to replace it.")

    parent = output_dir.parent
    parent.mkdir(parents=True, exist_ok=True)
    staging_dir = parent / f".{output_dir.name}.staging-{uuid.uuid4().hex}"
    backup_dir = parent / f".{output_dir.name}.previous-{uuid.uuid4().hex}"
    copied: list[str] = []
    try:
        copied = build_hf_space_bundle(staging_dir)
        issues = validate_export(staging_dir)
        if issues:
            raise ValueError(f"Staged Hugging Face Space bundle is invalid: {', '.join(issues)}")
        if output_dir.exists():
            output_dir.rename(backup_dir)
        staging_dir.rename(output_dir)
    except Exception:
        shutil.rmtree(staging_dir, ignore_errors=True)
        if backup_dir.exists() and not output_dir.exists():
            backup_dir.rename(output_dir)
        raise
    finally:
        shutil.rmtree(backup_dir, ignore_errors=True)
    return {"outputDir": str(output_dir), "copied": copied}


def write_space_readme(path: Path) -> None:
    path.write_text(
        """---

title: Arabic Audio Reader Worker

colorFrom: green

colorTo: green

sdk: docker

app_port: 7860

---



# Arabic Audio Reader Worker



This is the Docker worker bundle for the Arabic PDF Reader.



## Hugging Face Space Settings



- SDK: Docker

- Hardware: free CPU is acceptable for demos, but cold starts and long books can be slow

- Free CPU Basic currently provides 2 vCPU, 16 GB RAM, and 50 GB non-persistent disk by default; treat generated audio as short-lived unless you add persistent/object storage

- Port: 7860

- Default build: installs SILMA, PaddleOCR Arabic, Tesseract Arabic, and eSpeak NG

- Optional fast CPU voice: set Docker build arg `INSTALL_SUPERTONIC=1` to add Supertonic 3 Arabic-capable local TTS

- Stronger OCR build: set Docker build arg `INSTALL_TAWKEED_OCR=1`, `INSTALL_KATIB_OCR=1`, `INSTALL_ARABIC_QWEN_OCR=1`, `INSTALL_ARABIC_GLM_OCR=1`, or `INSTALL_BASEER_OCR=1` for Arabic-trained models, or `INSTALL_QARI_OCR=1` for the heavier Arabic-book model



Set these Space secrets:



```text

ACCESS_CODE=1234

SECRET_KEY=<generated by outputs\\deployment-handoff.md>

CORS_ORIGINS=https://your-vercel-app.vercel.app

COOKIE_SAMESITE=none

COOKIE_SECURE=1

OCR_ENGINE=tesseract
OCR_RENDER_ZOOM=2
TESSERACT_PSM=4
DEFAULT_VOICE_ID=silma-local

OUTPUT_RETENTION_DAYS=7

OUTPUT_MAX_FILES=25

AUDIO_FORMAT=mp3

MP3_BITRATE=96k

```



Generate the deployment handoff from the main repo to get the exact `SECRET_KEY`, worker secrets, Vercel environment variables, and final proof command:



```powershell

python scripts\\deployment_handoff.py https://your-space.hf.space --origin https://your-vercel-app.vercel.app --code 1234

```



Keep `outputs\\deployment-handoff.md` private because it contains deployment secrets.



The compact process recommendation is included at `docs/recommended-free-stack.md`, with the machine-readable deployment decision card at `docs/recommended-decision-card.json` and its readable companion at `docs/recommended-decision-card.md`. The current practical default is PyMuPDF embedded text first, `OCR_ENGINE=tesseract OCR_RENDER_ZOOM=2 TESSERACT_PSM=4` for the most readable tested scanned Arabic OCR, SILMA TTS for the first clean voice, and downloadable worker audio.


Optional stronger-worker build args:



```text

INSTALL_QARI_OCR=1

INSTALL_TAWKEED_OCR=1

INSTALL_KATIB_OCR=1

INSTALL_ARABIC_QWEN_OCR=1

INSTALL_ARABIC_GLM_OCR=1

INSTALL_BASEER_OCR=1

INSTALL_PADDLEOCR_VL=1

INSTALL_SUPERTONIC=1

```



Use `INSTALL_TAWKEED_OCR=1`, `INSTALL_KATIB_OCR=1`, `INSTALL_ARABIC_QWEN_OCR=1`, `INSTALL_ARABIC_GLM_OCR=1`, or `INSTALL_BASEER_OCR=1` first when you want an Arabic-trained OCR model. Use `INSTALL_QARI_OCR=1` when you want the strongest Arabic-book OCR and the worker has enough memory/GPU. Leave heavy options at `0` on free CPU Spaces unless a short benchmark proves the stronger model is worth the cold start, build time, memory, and runtime.



After the Space builds, verify it from your main repo:



```powershell

python scripts\\verify_worker.py https://your-space.hf.space --code 1234 --origin https://your-vercel-app.vercel.app --require-cors --smoke-upload --smoke-scanned --smoke-ocr-engine arabic

```

""",
        encoding="utf-8",
    )


def validate_export(output_dir: Path) -> list[str]:
    required = [
        "Dockerfile",
        "README.md",
        ".dockerignore",
        MANIFEST_NAME,
        "requirements.txt",
        "requirements-silma.txt",
        "requirements-supertonic.txt",
        "requirements-paddleocr.txt",
        "requirements-paddleocr-vl.txt",
        "requirements-qari-ocr.txt",
        "requirements-tawkeed-ocr.txt",
        "requirements-katib-ocr.txt",
        "requirements-arabic-qwen-ocr.txt",
        "requirements-arabic-glm-ocr.txt",
        "requirements-baseer-ocr.txt",
        ".export-complete",
        "app/main.py",
        "api/index.py",
        "static/index.html",
        "scripts/setup_silma.sh",
        "scripts/setup_supertonic.sh",
        "scripts/setup_paddleocr.sh",
        "scripts/setup_paddleocr_vl.sh",
        "scripts/setup_qari_ocr.sh",
        "scripts/setup_tawkeed_ocr.sh",
        "scripts/setup_katib_ocr.sh",
        "scripts/setup_arabic_qwen_ocr.sh",
        "scripts/setup_arabic_glm_ocr.sh",
        "scripts/setup_baseer_ocr.sh",
        "scripts/qari_ocr_extract.py",
        "scripts/tawkeed_ocr_extract.py",
        "scripts/katib_ocr_extract.py",
        "scripts/arabic_qwen_ocr_extract.py",
        "scripts/arabic_glm_ocr_extract.py",
        "scripts/baseer_ocr_extract.py",
        "scripts/configure_vercel_worker.py",
        "scripts/deploy_hf_space.py",
        "scripts/finish_live_deployment.py",
        "scripts/prepare_live_deployment.py",
        "scripts/validate_deployment_env.py",
        "scripts/refresh_research_evidence.py",
        "scripts/score_voice_listening.py",
        "scripts/score_tts_preprocessor.py",
        "docs/recommended-free-stack.md",
        "docs/recommended-decision-card.md",
        "docs/recommended-decision-card.json",
    ]
    missing = [relative for relative in required if not (output_dir / relative).exists()]
    forbidden = [".env", "uploads", "outputs", "data", "test_pdfs", ".venv", ".venv-silma", ".venv-ocr"]
    present_forbidden = [relative for relative in forbidden if (output_dir / relative).exists()]
    return [f"missing:{item}" for item in missing] + [f"forbidden:{item}" for item in present_forbidden]


def main() -> None:
    parser = argparse.ArgumentParser(description="Export a clean Hugging Face Spaces Docker worker bundle.")
    parser.add_argument("--out", type=Path, default=DEFAULT_OUTPUT, help="Destination folder for the Space bundle.")
    parser.add_argument("--force", action="store_true", help="Replace the destination folder if it already exists.")
    parser.add_argument("--json", action="store_true", help="Print JSON instead of a compact summary.")
    args = parser.parse_args()

    result = export_hf_space(args.out, force=args.force)
    issues = validate_export(args.out)
    result["ready"] = not issues
    result["issues"] = issues
    if args.json:
        print(json.dumps(result, indent=2))
    else:
        print(f"Exported Hugging Face Space bundle to {result['outputDir']}")
        if issues:
            print("Issues:")
            for issue in issues:
                print(f"- {issue}")
        else:
            print("Bundle is ready to push to a Docker Space.")
    if issues:
        raise SystemExit(1)


if __name__ == "__main__":
    main()