Upload folder using huggingface_hub

examples/redis_space/Dockerfile

@@ -0,0 +1,20 @@
+FROM python:3.10-slim
+
+# Install Redis
+RUN apt-get update && apt-get install -y redis-server && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+# Install Python dependencies
+RUN pip install fastapi uvicorn websockets huggingface_hub schedule
+
+COPY . .
+
+# Set up Redis data directory
+RUN mkdir -p /data
+RUN sed -i 's/^dir .*/dir \/data/' /etc/redis/redis.conf
+
+# Add start script
+RUN chmod +x start.sh
+
+CMD ["./start.sh"]

examples/redis_space/app.py

@@ -0,0 +1,60 @@
+import asyncio
+from fastapi import FastAPI, WebSocket
+from fastapi.responses import HTMLResponse
+import logging
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+app = FastAPI()
+
+@app.get("/")
+def read_root():
+    return HTMLResponse("<h1>Redis WebSocket Proxy Running</h1>")
+
+@app.websocket("/ws")
+async def websocket_endpoint(websocket: WebSocket):
+    await websocket.accept()
+    logger.info("WebSocket connection accepted.")
+    
+    try:
+        # Connect to local Redis TCP port
+        reader, writer = await asyncio.open_connection("127.0.0.1", 6379)
+        logger.info("Connected to local Redis.")
+    except Exception as e:
+        logger.error(f"Failed to connect to local Redis: {e}")
+        await websocket.close()
+        return
+
+    async def pipe_ws_to_tcp():
+        try:
+            while True:
+                data = await websocket.receive_bytes()
+                writer.write(data)
+                await writer.drain()
+        except Exception as e:
+            logger.info(f"ws_to_tcp closed: {e}")
+
+    async def pipe_tcp_to_ws():
+        try:
+            while True:
+                data = await reader.read(4096)
+                if not data:
+                    break
+                await websocket.send_bytes(data)
+        except Exception as e:
+            logger.info(f"tcp_to_ws closed: {e}")
+
+    try:
+        await asyncio.gather(
+            pipe_ws_to_tcp(),
+            pipe_tcp_to_ws(),
+            return_exceptions=True
+        )
+    finally:
+        writer.close()
+        try:
+            await websocket.close()
+        except:
+            pass
+        logger.info("Connection closed.")

examples/redis_space/backup_manager.py

@@ -0,0 +1,68 @@
+import os
+import time
+import schedule
+from huggingface_hub import HfApi, hf_hub_download
+import subprocess
+import logging
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+HF_TOKEN = os.getenv("HF_TOKEN")
+DATASET_REPO = os.getenv("DATASET_REPO", "augment17/agentscope-redis-backup")
+DUMP_FILE = "/data/dump.rdb"
+LOCAL_DUMP = "dump.rdb"
+
+api = HfApi(token=HF_TOKEN)
+
+def ensure_repo():
+    try:
+        api.repo_info(repo_id=DATASET_REPO, repo_type="dataset")
+    except Exception:
+        logger.info(f"Creating dataset repo {DATASET_REPO}...")
+        api.create_repo(repo_id=DATASET_REPO, repo_type="dataset", private=True)
+
+def restore_backup():
+    ensure_repo()
+    try:
+        logger.info("Checking for existing Redis backup...")
+        downloaded_file = hf_hub_download(
+            repo_id=DATASET_REPO, 
+            repo_type="dataset", 
+            filename=LOCAL_DUMP,
+            token=HF_TOKEN,
+            local_dir="/data"
+        )
+        logger.info(f"Restored backup from {downloaded_file}")
+    except Exception as e:
+        logger.info(f"No existing backup found or failed to restore: {e}")
+
+def perform_backup():
+    logger.info("Performing Redis backup...")
+    try:
+        # Trigger Redis bgsave
+        subprocess.run(["redis-cli", "save"], check=True)
+        if os.path.exists(DUMP_FILE):
+            api.upload_file(
+                path_or_fileobj=DUMP_FILE,
+                path_in_repo=LOCAL_DUMP,
+                repo_id=DATASET_REPO,
+                repo_type="dataset",
+                token=HF_TOKEN
+            )
+            logger.info("Backup uploaded to Hugging Face Datasets successfully.")
+        else:
+            logger.error("dump.rdb not found after save!")
+    except Exception as e:
+        logger.error(f"Backup failed: {e}")
+
+if __name__ == "__main__":
+    mode = os.getenv("MODE", "restore")
+    if mode == "restore":
+        restore_backup()
+    elif mode == "cron":
+        logger.info("Starting backup cron schedule (every 30 minutes)...")
+        schedule.every(30).minutes.do(perform_backup)
+        while True:
+            schedule.run_pending()
+            time.sleep(60)

examples/redis_space/start.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+set -e
+
+# Restore backup from HF Datasets
+echo "Restoring database from Hugging Face Datasets..."
+MODE=restore python backup_manager.py
+
+# Start Redis Server in background
+echo "Starting Redis server..."
+redis-server /etc/redis/redis.conf &
+
+# Start backup cron loop in background
+echo "Starting background backup loop..."
+MODE=cron python backup_manager.py &
+
+# Start FastAPI WebSocket Proxy in foreground (port 7860 for HF Spaces)
+echo "Starting FastAPI WebSocket proxy..."
+uvicorn app:app --host 0.0.0.0 --port 7860

local_ws_proxy.py

@@ -0,0 +1,63 @@
+import asyncio
+import os
+import websockets
+import logging
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# The remote HF Space websocket URL (e.g., wss://augment17-redis.hf.space/ws)
+REMOTE_WS_URL = os.getenv("REMOTE_REDIS_WS_URL")
+LOCAL_PORT = int(os.getenv("LOCAL_REDIS_PORT", 6379))
+
+async def handle_client(reader, writer):
+    if not REMOTE_WS_URL:
+        logger.error("REMOTE_REDIS_WS_URL is not set!")
+        writer.close()
+        return
+
+    logger.info("New local TCP connection. Connecting to remote WebSocket...")
+    try:
+        async with websockets.connect(REMOTE_WS_URL) as ws:
+            logger.info("Connected to remote WebSocket.")
+
+            async def pipe_tcp_to_ws():
+                try:
+                    while True:
+                        data = await reader.read(4096)
+                        if not data:
+                            break
+                        await ws.send(data)
+                except Exception as e:
+                    logger.info(f"tcp_to_ws closed: {e}")
+
+            async def pipe_ws_to_tcp():
+                try:
+                    while True:
+                        data = await ws.recv()
+                        writer.write(data)
+                        await writer.drain()
+                except Exception as e:
+                    logger.info(f"ws_to_tcp closed: {e}")
+
+            await asyncio.gather(
+                pipe_tcp_to_ws(),
+                pipe_ws_to_tcp(),
+                return_exceptions=True
+            )
+    except Exception as e:
+        logger.error(f"WebSocket connection failed: {e}")
+    finally:
+        writer.close()
+        logger.info("Local TCP connection closed.")
+
+async def main():
+    server = await asyncio.start_server(handle_client, '127.0.0.1', LOCAL_PORT)
+    addr = server.sockets[0].getsockname()
+    logger.info(f"Local WebSocket-to-TCP proxy listening on {addr}")
+
+    async with server:
+        await server.serve_forever()
+
+if __name__ == '__main__':
+    asyncio.run(main())