Spaces:
Runtime error
Runtime error
Upload folder using huggingface_hub
Browse files- .dockerignore +4 -0
- .env.example +16 -0
- Dockerfile +1 -0
- README.md +97 -4
- app.py +4 -16
- requirements.txt +2 -0
- space_worker/__init__.py +1 -0
- space_worker/config.py +55 -0
- space_worker/http.py +11 -0
- space_worker/logging_setup.py +8 -0
- space_worker/messages.py +60 -0
- space_worker/runtime.py +28 -0
- space_worker/supabase.py +142 -0
- space_worker/worker.py +69 -0
.dockerignore
CHANGED
|
@@ -1,5 +1,9 @@
|
|
| 1 |
__pycache__/
|
|
|
|
| 2 |
*.py[cod]
|
|
|
|
| 3 |
.git/
|
| 4 |
.gitignore
|
| 5 |
.env
|
|
|
|
|
|
|
|
|
| 1 |
__pycache__/
|
| 2 |
+
**/__pycache__/
|
| 3 |
*.py[cod]
|
| 4 |
+
**/*.py[cod]
|
| 5 |
.git/
|
| 6 |
.gitignore
|
| 7 |
.env
|
| 8 |
+
.env.*
|
| 9 |
+
!.env.example
|
.env.example
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
SUPABASE_URL=https://dkpixedlmpkjthvhlcwu.supabase.co
|
| 2 |
+
SUPABASE_SERVICE_KEY=
|
| 3 |
+
|
| 4 |
+
# Space A listens here for user messages.
|
| 5 |
+
SUPABASE_SPACE_INBOX_TOPIC=space-a
|
| 6 |
+
SUPABASE_REQUEST_EVENT=message
|
| 7 |
+
SUPABASE_SPACE_INBOX_PRIVATE=true
|
| 8 |
+
|
| 9 |
+
# Space A replies to user-specific channels.
|
| 10 |
+
SUPABASE_REPLY_CHANNEL_TEMPLATE=user-{user_id}
|
| 11 |
+
SUPABASE_RESPONSE_EVENT=message
|
| 12 |
+
SUPABASE_USER_CHANNEL_PRIVATE=true
|
| 13 |
+
|
| 14 |
+
# Set false only for local debugging. Hugging Face should keep this true if
|
| 15 |
+
# you want the Space to remain in the Starting state.
|
| 16 |
+
SPACE_HANG_AFTER_WORKER_START=true
|
Dockerfile
CHANGED
|
@@ -3,6 +3,7 @@ FROM python:3.9
|
|
| 3 |
RUN useradd -m -u 1000 user
|
| 4 |
USER user
|
| 5 |
ENV PATH="/home/user/.local/bin:$PATH"
|
|
|
|
| 6 |
|
| 7 |
WORKDIR /app
|
| 8 |
|
|
|
|
| 3 |
RUN useradd -m -u 1000 user
|
| 4 |
USER user
|
| 5 |
ENV PATH="/home/user/.local/bin:$PATH"
|
| 6 |
+
ENV PYTHONUNBUFFERED=1
|
| 7 |
|
| 8 |
WORKDIR /app
|
| 9 |
|
README.md
CHANGED
|
@@ -1,15 +1,108 @@
|
|
| 1 |
---
|
| 2 |
-
title:
|
| 3 |
emoji: 🐳
|
| 4 |
colorFrom: blue
|
| 5 |
colorTo: gray
|
| 6 |
sdk: docker
|
|
|
|
|
|
|
| 7 |
pinned: false
|
| 8 |
---
|
| 9 |
|
| 10 |
-
#
|
| 11 |
|
| 12 |
This Space is configured as a Docker-based Hugging Face Space.
|
| 13 |
|
| 14 |
-
The container installs
|
| 15 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
+
title: Celery Space
|
| 3 |
emoji: 🐳
|
| 4 |
colorFrom: blue
|
| 5 |
colorTo: gray
|
| 6 |
sdk: docker
|
| 7 |
+
app_port: 7860
|
| 8 |
+
startup_duration_timeout: 3h
|
| 9 |
pinned: false
|
| 10 |
---
|
| 11 |
|
| 12 |
+
# Celery Docker Space
|
| 13 |
|
| 14 |
This Space is configured as a Docker-based Hugging Face Space.
|
| 15 |
|
| 16 |
+
The container installs FastAPI, Uvicorn, and `websocket-client`. `app.py` is a
|
| 17 |
+
small entrypoint; the worker code lives in `space_worker/`. During ASGI import,
|
| 18 |
+
it starts a Supabase Realtime worker thread and then intentionally blocks so the
|
| 19 |
+
Space stays in the Starting state instead of binding to port `7860`.
|
| 20 |
+
|
| 21 |
+
## Layout
|
| 22 |
+
|
| 23 |
+
```text
|
| 24 |
+
app.py
|
| 25 |
+
space_worker/
|
| 26 |
+
config.py
|
| 27 |
+
http.py
|
| 28 |
+
logging_setup.py
|
| 29 |
+
messages.py
|
| 30 |
+
runtime.py
|
| 31 |
+
supabase.py
|
| 32 |
+
worker.py
|
| 33 |
+
```
|
| 34 |
+
|
| 35 |
+
## Flow
|
| 36 |
+
|
| 37 |
+
1. Space A connects to Supabase Realtime.
|
| 38 |
+
2. Space A subscribes to `SUPABASE_SPACE_INBOX_TOPIC`, default `space-a`.
|
| 39 |
+
3. User `ABC` sends broadcast event `message` to channel `space-a`.
|
| 40 |
+
4. Space A receives the message, builds `Hello ABC`, and broadcasts it to
|
| 41 |
+
`user-ABC`.
|
| 42 |
+
5. User `ABC` receives the response from their own channel.
|
| 43 |
+
|
| 44 |
+
Kafka is not used in this flow.
|
| 45 |
+
|
| 46 |
+
## User Request
|
| 47 |
+
|
| 48 |
+
The user sends this to channel `space-a`, event `message`:
|
| 49 |
+
|
| 50 |
+
```json
|
| 51 |
+
{
|
| 52 |
+
"user_id": "ABC",
|
| 53 |
+
"message": "hello",
|
| 54 |
+
"correlation_id": "request_uuid"
|
| 55 |
+
}
|
| 56 |
+
```
|
| 57 |
+
|
| 58 |
+
## Space Response
|
| 59 |
+
|
| 60 |
+
Space A sends this to channel `user-ABC`, event `message`:
|
| 61 |
+
|
| 62 |
+
```json
|
| 63 |
+
{
|
| 64 |
+
"type": "space.response",
|
| 65 |
+
"ok": true,
|
| 66 |
+
"user_id": "ABC",
|
| 67 |
+
"session_id": null,
|
| 68 |
+
"correlation_id": "request_uuid",
|
| 69 |
+
"request": {
|
| 70 |
+
"message": "hello"
|
| 71 |
+
},
|
| 72 |
+
"message": "Hello ABC",
|
| 73 |
+
"created_at": "2026-06-26T00:00:00+00:00"
|
| 74 |
+
}
|
| 75 |
+
```
|
| 76 |
+
|
| 77 |
+
## JavaScript Client Sketch
|
| 78 |
+
|
| 79 |
+
```js
|
| 80 |
+
const userId = 'ABC'
|
| 81 |
+
const inbox = supabase.channel('space-a', {
|
| 82 |
+
config: { private: true, broadcast: { ack: true } },
|
| 83 |
+
})
|
| 84 |
+
const replies = supabase.channel(`user-${userId}`, {
|
| 85 |
+
config: { private: true },
|
| 86 |
+
})
|
| 87 |
+
|
| 88 |
+
replies.on('broadcast', { event: 'message' }, ({ payload }) => {
|
| 89 |
+
console.log(payload.message)
|
| 90 |
+
})
|
| 91 |
+
|
| 92 |
+
await replies.subscribe()
|
| 93 |
+
await inbox.subscribe()
|
| 94 |
+
|
| 95 |
+
await inbox.send({
|
| 96 |
+
type: 'broadcast',
|
| 97 |
+
event: 'message',
|
| 98 |
+
payload: {
|
| 99 |
+
user_id: userId,
|
| 100 |
+
message: 'hello',
|
| 101 |
+
correlation_id: crypto.randomUUID(),
|
| 102 |
+
},
|
| 103 |
+
})
|
| 104 |
+
```
|
| 105 |
+
|
| 106 |
+
Private channels require Supabase Realtime authorization policies. For a quick
|
| 107 |
+
Realtime Inspector test, you can temporarily set both private flags to `false`,
|
| 108 |
+
then switch them back to `true` for real users.
|
app.py
CHANGED
|
@@ -1,18 +1,6 @@
|
|
| 1 |
-
import
|
|
|
|
| 2 |
|
| 3 |
-
from fastapi import FastAPI
|
| 4 |
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
@app.get("/")
|
| 10 |
-
def greet_json():
|
| 11 |
-
return {"status": "this route is intentionally unreachable"}
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
# Intentionally block module import so Uvicorn never finishes starting.
|
| 15 |
-
# Hugging Face will install dependencies, run the container, then keep waiting
|
| 16 |
-
# because the app never binds to port 7860.
|
| 17 |
-
while True:
|
| 18 |
-
time.sleep(3600)
|
|
|
|
| 1 |
+
from space_worker.http import create_app
|
| 2 |
+
from space_worker.runtime import start_worker_and_block_if_needed
|
| 3 |
|
|
|
|
| 4 |
|
| 5 |
+
app = create_app()
|
| 6 |
+
start_worker_and_block_if_needed()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
requirements.txt
CHANGED
|
@@ -1,2 +1,4 @@
|
|
| 1 |
fastapi
|
| 2 |
uvicorn[standard]
|
|
|
|
|
|
|
|
|
| 1 |
fastapi
|
| 2 |
uvicorn[standard]
|
| 3 |
+
websocket-client
|
| 4 |
+
python-dotenv
|
space_worker/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
"""Runtime package for the Celery Space worker."""
|
space_worker/config.py
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
from dataclasses import dataclass
|
| 3 |
+
from pathlib import Path
|
| 4 |
+
|
| 5 |
+
from dotenv import load_dotenv
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
TEMPLATE_ROOT = Path(__file__).resolve().parents[1]
|
| 9 |
+
load_dotenv(TEMPLATE_ROOT / ".env")
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def env_bool(name, default=False):
|
| 13 |
+
value = os.getenv(name)
|
| 14 |
+
if value is None:
|
| 15 |
+
return default
|
| 16 |
+
return value.strip().lower() in {"1", "true", "yes", "on"}
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
@dataclass(frozen=True)
|
| 20 |
+
class Settings:
|
| 21 |
+
supabase_url: str
|
| 22 |
+
supabase_key: str
|
| 23 |
+
inbox_topic: str
|
| 24 |
+
request_event: str
|
| 25 |
+
response_event: str
|
| 26 |
+
inbox_private: bool
|
| 27 |
+
response_private: bool
|
| 28 |
+
reply_channel_template: str
|
| 29 |
+
heartbeat_seconds: int
|
| 30 |
+
reconnect_seconds: int
|
| 31 |
+
hang_after_worker_start: bool
|
| 32 |
+
log_level: str
|
| 33 |
+
|
| 34 |
+
@classmethod
|
| 35 |
+
def from_env(cls):
|
| 36 |
+
return cls(
|
| 37 |
+
supabase_url=os.getenv("SUPABASE_URL", "").rstrip("/"),
|
| 38 |
+
supabase_key=os.getenv("SUPABASE_SERVICE_KEY") or os.getenv("SUPABASE_ANON_KEY", ""),
|
| 39 |
+
inbox_topic=os.getenv("SUPABASE_SPACE_INBOX_TOPIC", "space-a"),
|
| 40 |
+
request_event=os.getenv("SUPABASE_REQUEST_EVENT", "message"),
|
| 41 |
+
response_event=os.getenv("SUPABASE_RESPONSE_EVENT", "message"),
|
| 42 |
+
inbox_private=env_bool("SUPABASE_SPACE_INBOX_PRIVATE", True),
|
| 43 |
+
response_private=env_bool("SUPABASE_USER_CHANNEL_PRIVATE", True),
|
| 44 |
+
reply_channel_template=os.getenv("SUPABASE_REPLY_CHANNEL_TEMPLATE", "user-{user_id}"),
|
| 45 |
+
heartbeat_seconds=int(os.getenv("SUPABASE_HEARTBEAT_SECONDS", "20")),
|
| 46 |
+
reconnect_seconds=int(os.getenv("SUPABASE_RECONNECT_SECONDS", "5")),
|
| 47 |
+
hang_after_worker_start=env_bool("SPACE_HANG_AFTER_WORKER_START", True),
|
| 48 |
+
log_level=os.getenv("LOG_LEVEL", "INFO").upper(),
|
| 49 |
+
)
|
| 50 |
+
|
| 51 |
+
def require_supabase(self):
|
| 52 |
+
if not self.supabase_url:
|
| 53 |
+
raise RuntimeError("SUPABASE_URL is required")
|
| 54 |
+
if not self.supabase_key:
|
| 55 |
+
raise RuntimeError("SUPABASE_SERVICE_KEY or SUPABASE_ANON_KEY is required")
|
space_worker/http.py
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from fastapi import FastAPI
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
def create_app():
|
| 5 |
+
app = FastAPI()
|
| 6 |
+
|
| 7 |
+
@app.get("/")
|
| 8 |
+
def status():
|
| 9 |
+
return {"status": "supabase-worker-started-http-intentionally-unreachable"}
|
| 10 |
+
|
| 11 |
+
return app
|
space_worker/logging_setup.py
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import logging
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
def configure_logging(level):
|
| 5 |
+
logging.basicConfig(
|
| 6 |
+
level=level,
|
| 7 |
+
format="%(asctime)s %(levelname)s %(threadName)s %(message)s",
|
| 8 |
+
)
|
space_worker/messages.py
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import re
|
| 2 |
+
import uuid
|
| 3 |
+
from datetime import datetime, timezone
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
def utc_now():
|
| 7 |
+
return datetime.now(timezone.utc).isoformat()
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
def sanitize_channel_part(value):
|
| 11 |
+
return re.sub(r"[^A-Za-z0-9_.-]+", "-", str(value)).strip("-") or "unknown"
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class ResponseBuilder:
|
| 15 |
+
def __init__(self, settings):
|
| 16 |
+
self.settings = settings
|
| 17 |
+
|
| 18 |
+
def reply_channel_for(self, payload):
|
| 19 |
+
explicit = payload.get("reply_channel") or payload.get("reply_topic")
|
| 20 |
+
if explicit:
|
| 21 |
+
return explicit
|
| 22 |
+
|
| 23 |
+
user_id = sanitize_channel_part(payload.get("user_id") or payload.get("id") or "unknown")
|
| 24 |
+
session_id = sanitize_channel_part(payload.get("session_id") or "default")
|
| 25 |
+
return self.settings.reply_channel_template.format(user_id=user_id, session_id=session_id)
|
| 26 |
+
|
| 27 |
+
def build_success(self, payload):
|
| 28 |
+
user_id = payload.get("user_id") or payload.get("id")
|
| 29 |
+
if not user_id:
|
| 30 |
+
raise ValueError("payload must include user_id")
|
| 31 |
+
|
| 32 |
+
text = payload.get("message")
|
| 33 |
+
if text is None:
|
| 34 |
+
raise ValueError("payload must include message")
|
| 35 |
+
|
| 36 |
+
display_name = payload.get("name") or payload.get("username") or user_id
|
| 37 |
+
correlation_id = payload.get("correlation_id") or str(uuid.uuid4())
|
| 38 |
+
|
| 39 |
+
return {
|
| 40 |
+
"type": "space.response",
|
| 41 |
+
"ok": True,
|
| 42 |
+
"user_id": user_id,
|
| 43 |
+
"session_id": payload.get("session_id"),
|
| 44 |
+
"correlation_id": correlation_id,
|
| 45 |
+
"request": {"message": text},
|
| 46 |
+
"message": f"Hello {display_name}",
|
| 47 |
+
"created_at": utc_now(),
|
| 48 |
+
}
|
| 49 |
+
|
| 50 |
+
def build_error(self, payload, error):
|
| 51 |
+
user_id = payload.get("user_id") or payload.get("id") or "unknown"
|
| 52 |
+
return {
|
| 53 |
+
"type": "space.response",
|
| 54 |
+
"ok": False,
|
| 55 |
+
"user_id": user_id,
|
| 56 |
+
"session_id": payload.get("session_id"),
|
| 57 |
+
"correlation_id": payload.get("correlation_id") or str(uuid.uuid4()),
|
| 58 |
+
"error": str(error),
|
| 59 |
+
"created_at": utc_now(),
|
| 60 |
+
}
|
space_worker/runtime.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import logging
|
| 2 |
+
import threading
|
| 3 |
+
import time
|
| 4 |
+
|
| 5 |
+
from .config import Settings
|
| 6 |
+
from .logging_setup import configure_logging
|
| 7 |
+
from .worker import RealtimeWorker
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
logger = logging.getLogger("celery-space.runtime")
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def start_worker(settings):
|
| 14 |
+
worker = RealtimeWorker(settings)
|
| 15 |
+
thread = threading.Thread(target=worker.run_forever, name="supabase-realtime-worker", daemon=True)
|
| 16 |
+
thread.start()
|
| 17 |
+
return thread
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def start_worker_and_block_if_needed():
|
| 21 |
+
settings = Settings.from_env()
|
| 22 |
+
configure_logging(settings.log_level)
|
| 23 |
+
start_worker(settings)
|
| 24 |
+
|
| 25 |
+
if settings.hang_after_worker_start:
|
| 26 |
+
logger.info("worker thread started; intentionally blocking ASGI import so Space remains Starting")
|
| 27 |
+
while True:
|
| 28 |
+
time.sleep(3600)
|
space_worker/supabase.py
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
import logging
|
| 3 |
+
import time
|
| 4 |
+
from urllib import parse, request
|
| 5 |
+
|
| 6 |
+
import websocket
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
logger = logging.getLogger("celery-space.supabase")
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def json_bytes(payload):
|
| 13 |
+
return json.dumps(payload, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def realtime_topic(topic):
|
| 17 |
+
if topic.startswith("realtime:"):
|
| 18 |
+
return topic
|
| 19 |
+
return f"realtime:{topic}"
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def extract_broadcast(message):
|
| 23 |
+
if message.get("event") != "broadcast":
|
| 24 |
+
return None
|
| 25 |
+
|
| 26 |
+
payload = message.get("payload") or {}
|
| 27 |
+
event = payload.get("event") or payload.get("type")
|
| 28 |
+
body = payload.get("payload", payload)
|
| 29 |
+
return event, body
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
class RefCounter:
|
| 33 |
+
def __init__(self):
|
| 34 |
+
self.value = 0
|
| 35 |
+
|
| 36 |
+
def next(self):
|
| 37 |
+
self.value += 1
|
| 38 |
+
return str(self.value)
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
class SupabaseBroadcaster:
|
| 42 |
+
def __init__(self, settings):
|
| 43 |
+
self.settings = settings
|
| 44 |
+
|
| 45 |
+
def send(self, topic, event, payload, private=True):
|
| 46 |
+
self.settings.require_supabase()
|
| 47 |
+
encoded_topic = parse.quote(topic, safe="")
|
| 48 |
+
encoded_event = parse.quote(event, safe="")
|
| 49 |
+
private_query = "true" if private else "false"
|
| 50 |
+
url = (
|
| 51 |
+
f"{self.settings.supabase_url}/realtime/v1/api/broadcast/"
|
| 52 |
+
f"{encoded_topic}/events/{encoded_event}?private={private_query}"
|
| 53 |
+
)
|
| 54 |
+
req = request.Request(
|
| 55 |
+
url,
|
| 56 |
+
data=json_bytes(payload),
|
| 57 |
+
method="POST",
|
| 58 |
+
headers={
|
| 59 |
+
"apikey": self.settings.supabase_key,
|
| 60 |
+
"Authorization": f"Bearer {self.settings.supabase_key}",
|
| 61 |
+
"Content-Type": "application/json",
|
| 62 |
+
},
|
| 63 |
+
)
|
| 64 |
+
|
| 65 |
+
with request.urlopen(req, timeout=10) as response:
|
| 66 |
+
logger.info("broadcast delivered topic=%s event=%s status=%s", topic, event, response.status)
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
class RealtimeConnection:
|
| 70 |
+
def __init__(self, settings):
|
| 71 |
+
self.settings = settings
|
| 72 |
+
self.refs = RefCounter()
|
| 73 |
+
self.ws = None
|
| 74 |
+
|
| 75 |
+
def __enter__(self):
|
| 76 |
+
self.settings.require_supabase()
|
| 77 |
+
self.ws = websocket.create_connection(self.websocket_url(), timeout=30)
|
| 78 |
+
self.ws.settimeout(1)
|
| 79 |
+
self.join_inbox()
|
| 80 |
+
return self
|
| 81 |
+
|
| 82 |
+
def __exit__(self, exc_type, exc, traceback):
|
| 83 |
+
if self.ws is not None:
|
| 84 |
+
self.ws.close()
|
| 85 |
+
|
| 86 |
+
def websocket_url(self):
|
| 87 |
+
parsed = parse.urlparse(self.settings.supabase_url)
|
| 88 |
+
if parsed.scheme == "https":
|
| 89 |
+
scheme = "wss"
|
| 90 |
+
elif parsed.scheme == "http":
|
| 91 |
+
scheme = "ws"
|
| 92 |
+
else:
|
| 93 |
+
raise RuntimeError("SUPABASE_URL must start with http:// or https://")
|
| 94 |
+
|
| 95 |
+
query = parse.urlencode({"apikey": self.settings.supabase_key, "vsn": "1.0.0"})
|
| 96 |
+
return parse.urlunparse((scheme, parsed.netloc, "/realtime/v1/websocket", "", query, ""))
|
| 97 |
+
|
| 98 |
+
def send(self, topic, event, payload, join_ref=None, ref=None):
|
| 99 |
+
if ref is None:
|
| 100 |
+
ref = self.refs.next()
|
| 101 |
+
message = {
|
| 102 |
+
"topic": topic,
|
| 103 |
+
"event": event,
|
| 104 |
+
"payload": payload,
|
| 105 |
+
"ref": ref,
|
| 106 |
+
"join_ref": join_ref,
|
| 107 |
+
}
|
| 108 |
+
self.ws.send(json.dumps(message, separators=(",", ":"), ensure_ascii=False))
|
| 109 |
+
return ref
|
| 110 |
+
|
| 111 |
+
def join_inbox(self):
|
| 112 |
+
topic = realtime_topic(self.settings.inbox_topic)
|
| 113 |
+
join_ref = self.refs.next()
|
| 114 |
+
payload = {
|
| 115 |
+
"config": {
|
| 116 |
+
"broadcast": {"ack": True, "self": False},
|
| 117 |
+
"presence": {"enabled": False},
|
| 118 |
+
"postgres_changes": [],
|
| 119 |
+
"private": self.settings.inbox_private,
|
| 120 |
+
},
|
| 121 |
+
"access_token": self.settings.supabase_key,
|
| 122 |
+
}
|
| 123 |
+
self.send(topic, "phx_join", payload, join_ref=join_ref, ref=join_ref)
|
| 124 |
+
logger.info("joining supabase topic=%s private=%s", self.settings.inbox_topic, self.settings.inbox_private)
|
| 125 |
+
|
| 126 |
+
def send_heartbeat(self):
|
| 127 |
+
self.send("phoenix", "heartbeat", {})
|
| 128 |
+
|
| 129 |
+
def iter_messages(self):
|
| 130 |
+
last_heartbeat = 0
|
| 131 |
+
while True:
|
| 132 |
+
now = time.monotonic()
|
| 133 |
+
if now - last_heartbeat >= self.settings.heartbeat_seconds:
|
| 134 |
+
self.send_heartbeat()
|
| 135 |
+
last_heartbeat = now
|
| 136 |
+
|
| 137 |
+
try:
|
| 138 |
+
raw_message = self.ws.recv()
|
| 139 |
+
except websocket.WebSocketTimeoutException:
|
| 140 |
+
continue
|
| 141 |
+
|
| 142 |
+
yield json.loads(raw_message)
|
space_worker/worker.py
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import logging
|
| 2 |
+
import time
|
| 3 |
+
from urllib.error import URLError
|
| 4 |
+
|
| 5 |
+
from .messages import ResponseBuilder
|
| 6 |
+
from .supabase import RealtimeConnection, SupabaseBroadcaster, extract_broadcast
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
logger = logging.getLogger("celery-space.worker")
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class RealtimeWorker:
|
| 13 |
+
def __init__(self, settings):
|
| 14 |
+
self.settings = settings
|
| 15 |
+
self.responses = ResponseBuilder(settings)
|
| 16 |
+
self.broadcaster = SupabaseBroadcaster(settings)
|
| 17 |
+
|
| 18 |
+
def run_forever(self):
|
| 19 |
+
while True:
|
| 20 |
+
try:
|
| 21 |
+
logger.info("connecting supabase realtime url=%s", self.settings.supabase_url)
|
| 22 |
+
with RealtimeConnection(self.settings) as connection:
|
| 23 |
+
self.consume(connection)
|
| 24 |
+
except Exception:
|
| 25 |
+
logger.exception(
|
| 26 |
+
"supabase realtime worker disconnected; retrying in %s seconds",
|
| 27 |
+
self.settings.reconnect_seconds,
|
| 28 |
+
)
|
| 29 |
+
time.sleep(self.settings.reconnect_seconds)
|
| 30 |
+
|
| 31 |
+
def consume(self, connection):
|
| 32 |
+
for message in connection.iter_messages():
|
| 33 |
+
broadcast = extract_broadcast(message)
|
| 34 |
+
if not broadcast:
|
| 35 |
+
continue
|
| 36 |
+
|
| 37 |
+
event, payload = broadcast
|
| 38 |
+
if event != self.settings.request_event:
|
| 39 |
+
continue
|
| 40 |
+
if not isinstance(payload, dict):
|
| 41 |
+
logger.warning("ignored non-object realtime payload event=%s", event)
|
| 42 |
+
continue
|
| 43 |
+
|
| 44 |
+
self.handle_payload(payload)
|
| 45 |
+
|
| 46 |
+
def handle_payload(self, payload):
|
| 47 |
+
try:
|
| 48 |
+
response = self.responses.build_success(payload)
|
| 49 |
+
except Exception as exc:
|
| 50 |
+
response = self.responses.build_error(payload, exc)
|
| 51 |
+
|
| 52 |
+
reply_channel = self.responses.reply_channel_for(payload)
|
| 53 |
+
try:
|
| 54 |
+
self.broadcaster.send(
|
| 55 |
+
reply_channel,
|
| 56 |
+
self.settings.response_event,
|
| 57 |
+
response,
|
| 58 |
+
private=self.settings.response_private,
|
| 59 |
+
)
|
| 60 |
+
except URLError as exc:
|
| 61 |
+
logger.warning("broadcast failed reply_channel=%s error=%s", reply_channel, exc)
|
| 62 |
+
return
|
| 63 |
+
|
| 64 |
+
logger.info(
|
| 65 |
+
"processed realtime message user_id=%s reply_channel=%s correlation_id=%s",
|
| 66 |
+
response.get("user_id"),
|
| 67 |
+
reply_channel,
|
| 68 |
+
response.get("correlation_id"),
|
| 69 |
+
)
|