deploy: real pdf-trainer-demo API

.gitattributes

@@ -1,35 +0,0 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -1,8 +0,0 @@
-__pycache__/
-*.pyc
-.venv/
-.env
-.env.*
-node_modules/
-dist/
-.DS_Store

Dockerfile

@@ -1,13 +1,25 @@
-FROM python:3.11-slim
+# ---- Build stage ----
+FROM node:20-alpine AS build
 
 WORKDIR /app
-ENV PYTHONUNBUFFERED=1
-ENV PIP_DISABLE_PIP_VERSION_CHECK=1
 
-COPY requirements.txt .
-RUN pip install --no-cache-dir -r requirements.txt
+COPY package.json package-lock.json ./
+RUN npm ci
 
 COPY . .
+RUN npm run build
 
-ENV PORT=7860
-CMD ["bash", "-lc", "uvicorn backend.api:app --host 0.0.0.0 --port ${PORT}"]
+# ---- Runtime stage ----
+FROM nginx:alpine
+
+# Remove default nginx config
+RUN rm /etc/nginx/conf.d/default.conf
+
+# Custom nginx config
+COPY nginx.conf /etc/nginx/conf.d/default.conf
+
+# Copy built assets
+COPY --from=build /app/dist /usr/share/nginx/html
+
+EXPOSE 7860
+CMD ["nginx", "-g", "daemon off;"]

README.md

@@ -1,10 +0,0 @@
----
-title: Pdf Trainer Api
-emoji: 🌍
-colorFrom: indigo
-colorTo: pink
-sdk: docker
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

api.py

@@ -1 +0,0 @@
-from backend.api import app

backend/api.py

@@ -1,20 +1,20 @@
 from __future__ import annotations
 
+import json
 import os
-import time
 from pathlib import Path
-from typing import Dict, Optional
-from uuid import uuid4
-from collections import deque
-from dataclasses import dataclass, asdict
+from typing import Any, Dict
 
-from fastapi import FastAPI, HTTPException, Header, Request
+from dotenv import load_dotenv
+from fastapi import FastAPI, HTTPException
 from fastapi.middleware.cors import CORSMiddleware
-from fastapi.responses import FileResponse, JSONResponse
-from pydantic import BaseModel
+from fastapi.responses import FileResponse, PlainTextResponse
+
+from backend.worker.gmail_client import GmailClient
 
 app = FastAPI(title="PDF Trainer API", version="1.0")
 
+# Allow Vite dev server
 app.add_middleware(
     CORSMiddleware,
     allow_origins=[
@@ -26,114 +26,188 @@ app.add_middleware(
     allow_headers=["*"],
 )
 
-WORKER_TOKEN = (os.getenv("WORKER_TOKEN") or "").strip()
-DATA_DIR = Path(os.getenv("DATA_DIR", "/app/data")).resolve()
-PDF_DIR = DATA_DIR / "pdfs"
-CFG_DIR = DATA_DIR / "configs"
+REPO_ROOT = Path(__file__).resolve().parents[1]
+BACKEND_DIR = REPO_ROOT / "backend"
+UPLOADS_DIR = BACKEND_DIR / "worker" / "uploads"
+
+# Load backend/.env explicitly ONCE for this process
+load_dotenv(BACKEND_DIR / ".env", override=True)
+
+CREDENTIALS_JSON = Path(os.environ.get("GMAIL_CREDENTIALS_JSON", str(BACKEND_DIR / "credentials.json")))
+TOKEN_JSON = Path(os.environ.get("GMAIL_TOKEN_JSON", str(BACKEND_DIR / "token.json")))
+
 
-PDF_DIR.mkdir(parents=True, exist_ok=True)
-CFG_DIR.mkdir(parents=True, exist_ok=True)
+def _gmail() -> GmailClient:
+    return GmailClient(CREDENTIALS_JSON, TOKEN_JSON)
+
+
+def _get_env_required(key: str) -> str:
+    v = (os.environ.get(key) or "").strip()
+    if not v:
+        raise HTTPException(status_code=500, detail=f"Server missing {key} env var")
+    return v
 
-def _require_worker_token(x_worker_token: Optional[str]):
-    if not WORKER_TOKEN:
-        raise HTTPException(status_code=500, detail="Server missing WORKER_TOKEN")
-    if not x_worker_token or x_worker_token != WORKER_TOKEN:
-        raise HTTPException(status_code=401, detail="Unauthorized worker")
 
 @app.get("/health")
-def health() -> Dict[str, bool]:
+def health():
     return {"ok": True}
 
-@app.get("/")
-def root() -> Dict[str, str]:
-    return {"service": "pdf-trainer-api", "status": "running"}
-
-def _pdf_path(pdf_id: str) -> Path:
-    safe = pdf_id.strip().replace("/", "_")
-    if not safe:
-        raise HTTPException(status_code=400, detail="Missing pdf_id")
-    if not safe.lower().endswith(".pdf"):
-        safe = safe + ".pdf"
-    return PDF_DIR / safe
-
-@app.put("/api/pdf/{pdf_id}")
-async def put_pdf(
-    pdf_id: str,
-    request: Request,
-    x_worker_token: Optional[str] = Header(default=None),
-):
-    _require_worker_token(x_worker_token)
-    body = await request.body()
-    if not body:
-        raise HTTPException(status_code=400, detail="Empty body")
-    p = _pdf_path(pdf_id)
-    p.write_bytes(body)
-    return {"ok": True, "pdf_id": p.stem, "bytes": len(body)}
 
 @app.get("/api/pdf/{pdf_id}")
 def get_pdf(pdf_id: str):
-    p = _pdf_path(pdf_id)
-    if not p.exists():
+    path = UPLOADS_DIR / f"{pdf_id}.pdf"
+    if not path.exists():
         raise HTTPException(status_code=404, detail="PDF not found")
-    return FileResponse(str(p), media_type="application/pdf", filename=p.name)
+
+    name_path = UPLOADS_DIR / f"{pdf_id}.name.txt"
+    pdf_name = name_path.read_text(encoding="utf-8").strip() if name_path.exists() else f"{pdf_id}.pdf"
+
+    resp = FileResponse(path, media_type="application/pdf", filename=pdf_name)
+    resp.headers["X-PDF-Name"] = pdf_name
+    return resp
+
 
 @app.post("/api/send-config")
-async def send_config(payload: dict):
-    pdf_id = str(payload.get("pdf_id", "")).strip()
-    template_id = str(payload.get("template_id", "")).strip()
+async def send_config(payload: Dict[str, Any]):
+    """
+    PIPELINE SUBMISSION EMAIL (after rep saves config)
+
+    REQUIRED payload:
+      - pdf_id: str
+      - template_id: str
+      - config: dict
+
+    Sends to PIPELINE inbox:
+      - PDF_PIPELINE_PIPELINE_NOTIFY_TO
+
+    Requirements:
+      - Subject includes template_id
+      - Body includes pdf_id
+      - Attachments: JSON + PDF
+    """
+    pdf_id = (payload.get("pdf_id") or "").strip()
+    template_id = (payload.get("template_id") or "").strip()
     config = payload.get("config")
-    if not pdf_id or not template_id or config is None:
-        raise HTTPException(status_code=400, detail="Missing pdf_id/template_id/config")
-    cfg_name = f"{pdf_id}_{template_id}_{uuid4().hex}.json"
-    out = CFG_DIR / cfg_name
-    import json
-    out.write_text(json.dumps(payload, indent=2), encoding="utf-8")
-    return {"ok": True, "saved": cfg_name}
-
-@dataclass
-class Job:
-    id: str
-    payload: dict
-    created_at: float
-    status: str = "queued"
-    message: str = ""
-
-JOBQ: deque[Job] = deque()
-JOBS: dict[str, Job] = {}
-
-class EnqueueReq(BaseModel):
-    payload: dict
-
-@app.post("/api/jobs/enqueue")
-def enqueue_job(req: EnqueueReq):
-    job_id = uuid4().hex
-    job = Job(id=job_id, payload=req.payload, created_at=time.time())
-    JOBQ.append(job)
-    JOBS[job_id] = job
-    return {"job_id": job_id, "status": job.status}
-
-@app.get("/api/jobs/next")
-def jobs_next(x_worker_token: Optional[str] = Header(default=None)):
-    _require_worker_token(x_worker_token)
-    while JOBQ:
-        job = JOBQ.popleft()
-        if job.status == "queued":
-            job.status = "running"
-            return asdict(job)
-    return JSONResponse(status_code=204, content=None)
-
-class StatusReq(BaseModel):
-    status: str
-    message: str = ""
-
-@app.post("/api/jobs/{job_id}/status")
-def jobs_status(job_id: str, req: StatusReq, x_worker_token: Optional[str] = Header(default=None)):
-    _require_worker_token(x_worker_token)
-    job = JOBS.get(job_id)
-    if not job:
-        raise HTTPException(status_code=404, detail="Unknown job_id")
-    if req.status not in ("running", "done", "failed"):
-        raise HTTPException(status_code=400, detail="Invalid status")
-    job.status = req.status
-    job.message = req.message
-    return {"ok": True, "job_id": job_id, "status": job.status}
+
+    if not pdf_id:
+        raise HTTPException(status_code=400, detail="Missing pdf_id")
+    if not template_id:
+        raise HTTPException(status_code=400, detail="Missing template_id")
+    if not isinstance(config, dict):
+        raise HTTPException(status_code=400, detail="Missing config object")
+
+    pipeline_to = _get_env_required("PDF_PIPELINE_PIPELINE_NOTIFY_TO")
+    notify_from = _get_env_required("PDF_PIPELINE_NOTIFY_FROM")
+    trainer_base_url = (os.environ.get("PDF_TRAINER_BASE_URL") or "http://localhost:5173").strip()
+
+    pdf_path = UPLOADS_DIR / f"{pdf_id}.pdf"
+    if not pdf_path.exists():
+        raise HTTPException(status_code=404, detail="PDF not found for pdf_id")
+
+    name_path = UPLOADS_DIR / f"{pdf_id}.name.txt"
+    pdf_name = name_path.read_text(encoding="utf-8").strip() if name_path.exists() else f"{pdf_id}.pdf"
+
+    trainer_link = f"{trainer_base_url.rstrip('/')}/?pdf_id={pdf_id}"
+
+    subject = f"PDF_TRAINER_CONFIG_SUBMITTED | template_id={template_id}"
+    body = (
+        "Hi,\n\n"
+        "A PDF Trainer configuration was submitted.\n\n"
+        f"template_id: {template_id}\n"
+        f"pdf_id: {pdf_id}\n"
+        f"trainer_link: {trainer_link}\n\n"
+        "Attachments:\n"
+        f"- trainer_config_{pdf_id}_{template_id}.json\n"
+        f"- {pdf_name}\n\n"
+        "Thank you,\n"
+        "Inserio Automation\n"
+    )
+
+    cfg_bytes = json.dumps(
+        {"pdf_id": pdf_id, "template_id": template_id, "config": config},
+        indent=2,
+    ).encode("utf-8")
+
+    attachments = [
+        (f"trainer_config_{pdf_id}_{template_id}.json", cfg_bytes),
+        (pdf_name, pdf_path.read_bytes()),
+    ]
+
+    gmail = _gmail()
+    gmail.send_email(
+        to_email=pipeline_to,
+        from_email=notify_from,
+        subject=subject,
+        body_text=body,
+        attachments=attachments,
+    )
+
+    return {"ok": True}
+
+
+@app.post("/api/notify-unknown")
+async def notify_unknown(payload: Dict[str, Any]):
+    """
+    UNKNOWN TEMPLATE NOTIFICATION (rep email)
+
+    REQUIRED payload:
+      - pdf_id: str
+    OPTIONAL:
+      - reason: str
+
+    Sends to REP inbox:
+      - PDF_PIPELINE_NOTIFY_TO
+
+    Requirements:
+      - Includes trainer link with PDF pre-loaded
+      - Attaches PDF
+      - No JSON
+    """
+    pdf_id = (payload.get("pdf_id") or "").strip()
+    reason = (payload.get("reason") or "").strip()
+
+    if not pdf_id:
+        raise HTTPException(status_code=400, detail="Missing pdf_id")
+
+    rep_to = _get_env_required("PDF_PIPELINE_NOTIFY_TO")
+    notify_from = _get_env_required("PDF_PIPELINE_NOTIFY_FROM")
+    trainer_base_url = (os.environ.get("PDF_TRAINER_BASE_URL") or "http://localhost:5173").strip()
+
+    pdf_path = UPLOADS_DIR / f"{pdf_id}.pdf"
+    if not pdf_path.exists():
+        raise HTTPException(status_code=404, detail="PDF not found for pdf_id")
+
+    name_path = UPLOADS_DIR / f"{pdf_id}.name.txt"
+    pdf_name = name_path.read_text(encoding="utf-8").strip() if name_path.exists() else f"{pdf_id}.pdf"
+
+    trainer_link = f"{trainer_base_url.rstrip('/')}/?pdf_id={pdf_id}"
+
+    subject = "Action required: Unknown PDF format (template not found)"
+    body = (
+        "Hi,\n\n"
+        "We received a PDF that does not match any existing templates in the system.\n\n"
+        + (f"Reason: {reason}\n\n" if reason else "")
+        + "Please open the PDF Trainer using the link below and create or update the template configuration:\n"
+        f"{trainer_link}\n\n"
+        "The original PDF is attached for reference.\n\n"
+        "Thank you,\n"
+        "Inserio Automation\n"
+    )
+
+    attachments = [(pdf_name, pdf_path.read_bytes())]
+
+    gmail = _gmail()
+    gmail.send_email(
+        to_email=rep_to,
+        from_email=notify_from,
+        subject=subject,
+        body_text=body,
+        attachments=attachments,
+    )
+
+    return {"ok": True}
+
+
+@app.get("/", response_class=PlainTextResponse)
+def root():
+    return "PDF Trainer API. Use /health"

backend/sftp_store.py

@@ -1,42 +0,0 @@
-import os
-import socket
-import posixpath
-import paramiko
-
-
-def _env(name: str) -> str:
-    v = (os.getenv(name) or "").strip()
-    if not v:
-        raise RuntimeError(f"Missing env var: {name}")
-    return v
-
-
-def download_bytes(remote_path: str) -> bytes:
-    """
-    Downloads a file from SFTP_ROOT + remote_path.
-    remote_path should be relative like 'pdfs/<id>.pdf' (no leading slash).
-    """
-    host = _env("SFTP_HOST")
-    port = int(_env("SFTP_PORT"))
-    user = _env("SFTP_USER")
-    pw = _env("SFTP_PASS")
-    root = (_env("SFTP_ROOT").rstrip("/") or "/")
-
-    rp = remote_path.lstrip("/")
-
-    transport = paramiko.Transport(socket.create_connection((host, port), timeout=10))
-    transport.banner_timeout = 10
-    transport.auth_timeout = 10
-    transport.connect(username=user, password=pw)
-    sftp = paramiko.SFTPClient.from_transport(transport)
-
-    try:
-        sftp.chdir(root)
-        full = posixpath.join(".", rp)
-        with sftp.open(full, "rb") as f:
-            return f.read()
-    finally:
-        try:
-            sftp.close()
-        finally:
-            transport.close()

requirements.txt

@@ -1,4 +1,18 @@
-fastapi
-uvicorn[standard]
-python-dotenv
-pydantic
+# Google / Gmail
+google-api-python-client==2.111.0
+google-auth==2.27.0
+google-auth-oauthlib==1.2.0
+
+# OpenAI
+openai==1.12.0
+
+# PDF -> image
+PyMuPDF==1.23.26
+Pillow==10.2.0
+
+# Utilities
+python-dotenv==1.0.1
+requests==2.31.0
+
+fastapi==0.115.6
+uvicorn==0.30.6