deploy: real pdf-trainer-demo worker

.gitattributes

@@ -1,35 +0,0 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -1,5 +1,5 @@
 __pycache__/
 *.pyc
-*.pyo
-*.pyd
 .DS_Store
+.env
+_local_secrets/

Dockerfile

@@ -1,11 +1,25 @@
-FROM python:3.11-slim
+# ---- Build stage ----
+FROM node:20-alpine AS build
+
 WORKDIR /app
 
-COPY requirements.txt .
-RUN pip install --no-cache-dir -r requirements.txt
+COPY package.json package-lock.json ./
+RUN npm ci
 
-# IMPORTANT: copy everything (worker.py + sftp_store.py + future files)
 COPY . .
+RUN npm run build
+
+# ---- Runtime stage ----
+FROM nginx:alpine
+
+# Remove default nginx config
+RUN rm /etc/nginx/conf.d/default.conf
+
+# Custom nginx config
+COPY nginx.conf /etc/nginx/conf.d/default.conf
+
+# Copy built assets
+COPY --from=build /app/dist /usr/share/nginx/html
 
-ENV PYTHONUNBUFFERED=1
-CMD ["python", "worker.py"]
+EXPOSE 7860
+CMD ["nginx", "-g", "daemon off;"]

README.md

@@ -1,10 +0,0 @@
----
-title: Pdf Trainer Worker
-emoji: 🏆
-colorFrom: indigo
-colorTo: red
-sdk: docker
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

requirements.txt

@@ -1,3 +1,18 @@
-requests
-python-dotenv
-paramiko
+# Google / Gmail
+google-api-python-client==2.111.0
+google-auth==2.27.0
+google-auth-oauthlib==1.2.0
+
+# OpenAI
+openai==1.12.0
+
+# PDF -> image
+PyMuPDF==1.23.26
+Pillow==10.2.0
+
+# Utilities
+python-dotenv==1.0.1
+requests==2.31.0
+
+fastapi==0.115.6
+uvicorn==0.30.6

sftp_store.py

@@ -1,89 +0,0 @@
-import os
-import posixpath
-import paramiko
-
-
-def _env(name: str) -> str:
-    v = (os.getenv(name) or "").strip()
-    if not v:
-        raise RuntimeError(f"Missing env var: {name}")
-    return v
-
-
-def sftp_test() -> str:
-    host = _env("SFTP_HOST")
-    port = int(_env("SFTP_PORT"))
-    user = _env("SFTP_USER")
-    pw = _env("SFTP_PASS")
-    root = _env("SFTP_ROOT").rstrip("/") or "/"
-
-    transport = paramiko.Transport((host, port))
-    transport.connect(username=user, password=pw)
-    sftp = paramiko.SFTPClient.from_transport(transport)
-
-    try:
-        sftp.chdir(root)
-        sftp.listdir(".")
-        return "sftp ok"
-    finally:
-        try:
-            sftp.close()
-        finally:
-            transport.close()
-
-
-def upload_bytes(remote_path: str, data: bytes) -> None:
-    """
-    Upload to SFTP_ROOT/<remote_path>.
-    remote_path should be RELATIVE (e.g. "pdfs/abc.pdf").
-    If someone passes "/inserio/pdfs/abc.pdf", we normalize it.
-    """
-    host = _env("SFTP_HOST")
-    port = int(_env("SFTP_PORT"))
-    user = _env("SFTP_USER")
-    pw = _env("SFTP_PASS")
-    root = _env("SFTP_ROOT").rstrip("/") or "/"
-
-    transport = paramiko.Transport((host, port))
-    transport.connect(username=user, password=pw)
-    sftp = paramiko.SFTPClient.from_transport(transport)
-
-    try:
-        # go to the shared root first
-        sftp.chdir(root)
-
-        # normalize remote_path relative to root (avoid /inserio/inserio/... double prefix)
-        root_rel = root.strip("/")
-        rp = remote_path.lstrip("/")
-        if root_rel and rp.startswith(root_rel + "/"):
-            rp = rp[len(root_rel) + 1 :]
-        remote_path = rp
-
-        # ensure directories exist (relative to root)
-        parts = remote_path.split("/")[:-1]
-        cur = "."
-        for part in parts:
-            if not part:
-                continue
-            cur = posixpath.join(cur, part)
-            try:
-                sftp.mkdir(cur)
-            except Exception:
-                pass
-
-        # write file relative to root
-        full = posixpath.join(".", remote_path)
-        with sftp.open(full, "wb") as f:
-            f.write(data)
-    finally:
-        try:
-            sftp.close()
-        finally:
-            transport.close()
-
-
-def upload_pdf(pdf_id: str, data: bytes) -> str:
-    # Stores under: pdfs/{pdf_id}.pdf (relative to SFTP_ROOT)
-    remote_path = f"pdfs/{pdf_id}.pdf"
-    upload_bytes(remote_path, data)
-    return remote_path

worker.py

@@ -1,173 +0,0 @@
-import os
-import time
-import base64
-import threading
-import requests
-from http.server import BaseHTTPRequestHandler, HTTPServer
-
-from sftp_store import upload_pdf
-from sftp_store import sftp_test
-
-import sftp_store
-
-API_BASE = os.getenv("API_BASE", "").rstrip("/")
-WORKER_TOKEN = os.getenv("WORKER_TOKEN", "")
-PORT = int(os.getenv("PORT", "7860"))
-POLL = int(os.getenv("POLL_INTERVAL_SECONDS", "10"))
-
-if not API_BASE or not WORKER_TOKEN:
-    raise RuntimeError("Missing API_BASE or WORKER_TOKEN")
-
-
-class Handler(BaseHTTPRequestHandler):
-
-    def do_HEAD(self):
-        path = self.path.split("?", 1)[0]
-
-        if path in ("/", "/health"):
-            self.send_response(200)
-            self.end_headers()
-            return
-
-        if path == "/sftp-test":
-            # no body on HEAD
-            self.send_response(200)
-            self.end_headers()
-            return
-
-        # if you have /api/pdf routes in worker, add them here too
-        self.send_response(404)
-        self.end_headers()
-
-    def do_GET(self):
-        path = self.path.split('?', 1)[0]
-
-        if path in ('/', '/health'):
-            self.send_response(200)
-            self.end_headers()
-            self.wfile.write(b"ok\n")
-            return
-
-        if path == '/sftp-test':
-            try:
-                msg = sftp_store.sftp_test()
-                self.send_response(200)
-                self.end_headers()
-                self.wfile.write((msg + "\n").encode('utf-8'))
-            except Exception as e:
-                self.send_response(500)
-                self.end_headers()
-                self.wfile.write((f"sftp fail: {e}\n").encode('utf-8'))
-            return
-
-        self.send_response(404)
-        self.end_headers()
-
-
-def serve_http():
-    HTTPServer(("0.0.0.0", PORT), Handler).serve_forever()
-
-
-def _api_headers() -> dict:
-    # API expects X-Worker-Token (NOT Authorization Bearer)
-    return {"X-Worker-Token": WORKER_TOKEN}
-
-def _job_status(job_id: str, status: str, message: str = "") -> None:
-    try:
-        requests.post(
-            f"{API_BASE}/api/jobs/{job_id}/status",
-            headers=_api_headers(),
-            json={"status": status, "message": message},
-            timeout=10,
-        )
-    except Exception as e:
-        print(f"status update failed: {e}", flush=True)
-
-def process_job(job: dict) -> None:
-    # API returns job dict (asdict(Job)) => id field is the canonical one
-    jid = job.get("id") or job.get("job_id")
-    if not jid:
-        raise RuntimeError("job missing id/job_id")
-
-    payload = job.get("payload") or {}
-    action = payload.get("action")
-
-    if action == "upload_pdf":
-        pdf_id = payload.get("pdf_id")
-        pdf_b64 = payload.get("pdf_b64")
-        if not pdf_id or not pdf_b64:
-            raise RuntimeError("upload_pdf requires payload.pdf_id and payload.pdf_b64")
-
-        # decode
-        try:
-            pdf_bytes = base64.b64decode(pdf_b64)
-        except Exception as e:
-            raise RuntimeError(f"invalid base64 in pdf_b64: {e}")
-
-        # upload
-        print(f"uploading pdf {pdf_id} ({len(pdf_bytes)} bytes)", flush=True)
-        remote_path = upload_pdf(pdf_id, pdf_bytes)
-        print(f"uploaded to {remote_path}", flush=True)
-        _job_status(jid, "done", f"uploaded:{remote_path}")
-        print(f"job {jid} done -> {remote_path}", flush=True)
-        return
-
-    # unknown action -> fail but don't crash loop
-    _job_status(jid, "failed", f"unknown action: {action}")
-    print(f"job {jid} failed: unknown action {action}", flush=True)
-
-
-def poll_loop():
-    print("Worker polling started", flush=True)
-
-    while True:
-        try:
-            r = requests.get(
-                f"{API_BASE}/api/jobs/next",
-                headers={"X-Worker-Token": WORKER_TOKEN},
-                timeout=10,
-            )
-
-            print(f"poll /api/jobs/next -> {r.status_code}", flush=True)
-
-            if r.status_code == 200:
-                job = r.json()
-                job_id = job.get("job_id") or job.get("id")
-                print(f"claimed job {job_id}", flush=True)
-
-                payload = job.get("payload", {})
-
-                try:
-                    action = payload.get("action")
-
-                    if action == "upload_pdf":
-                        pdf_id = payload["pdf_id"]
-                        pdf_b64 = payload["pdf_b64"]
-
-                        import base64
-                        from sftp_store import upload_bytes
-
-                        data = base64.b64decode(pdf_b64)
-                        remote_path = f"pdfs/{pdf_id}.pdf"
-
-                        print(f"uploading pdf {pdf_id} ({len(data)} bytes)", flush=True)
-                        upload_bytes(remote_path, data)
-                        print(f"uploaded to {remote_path}", flush=True)
-
-                    else:
-                        print(f"unknown action: {action}", flush=True)
-
-                except Exception as e:
-                    print(f"job {job_id} failed: {e}", flush=True)
-                # processing placeholder
-
-        except Exception as e:
-            print(f"poll error: {e}", flush=True)
-
-        time.sleep(POLL)
-
-
-print("PDF Trainer Worker booted", flush=True)
-
-threading.Thread(target=serve_http, daemon=True).start()
-poll_loop()