Spaces:

Subham9126
/

redpanda

Sleeping

App Files Files Community

Subham9126 commited on Feb 23

Commit

0122e2f

verified ·

1 Parent(s): 453ef85

Upload 5 files

Browse files

Files changed (5) hide show

Dockerfile +30 -0
init-topics.sh +38 -0
nginx.conf +115 -0
redpanda.yaml +66 -0
start.sh +71 -0

Dockerfile ADDED Viewed

	@@ -0,0 +1,30 @@

+FROM docker.redpanda.com/redpandadata/redpanda:v23.3.11 AS redpanda
+FROM nginx:1.25-alpine AS final
+# ── Install Redpanda into nginx image ─────────────────────────────────────────
+COPY --from=redpanda /usr/bin/redpanda         /usr/bin/redpanda
+COPY --from=redpanda /usr/bin/rpk              /usr/bin/rpk
+COPY --from=redpanda /usr/lib/redpanda         /usr/lib/redpanda
+COPY --from=redpanda /etc/redpanda             /etc/redpanda
+# ── Install runtime deps ───────────────────────────────────────────────────────
+RUN apk add --no-cache bash curl netcat-openbsd gettext
+# ── Copy all config files from root ───────────────────────────────────────────
+COPY redpanda.yaml    /etc/redpanda/redpanda.yaml
+COPY nginx.conf       /etc/nginx/nginx.conf
+COPY start.sh         /start.sh
+COPY init-topics.sh   /init-topics.sh
+RUN chmod +x /start.sh /init-topics.sh
+# ── Data dir ──────────────────────────────────────────────────────────────────
+RUN mkdir -p /var/lib/redpanda/data && \
+    chown -R 1000:1000 /var/lib/redpanda /etc/redpanda /start.sh /init-topics.sh
+# HF Spaces only exposes 7860 — everything goes through nginx
+EXPOSE 7860
+USER 1000
+ENTRYPOINT ["/start.sh"]

init-topics.sh ADDED Viewed

	@@ -0,0 +1,38 @@

+#!/bin/bash
+set -euo pipefail
+RPK="rpk --brokers 127.0.0.1:9092"
+# ── Edit this array to define your topics ────────────────────────────────────
+# Format: "topic-name:partitions"
+TOPICS=(
+    "events:4"
+    "raw-data:4"
+    "processed:4"
+    "dead-letter:1"
+)
+RETENTION_MS=3600000       # 1 hour — consume and store yourself
+RETENTION_BYTES=536870912  # 512MB per partition max
+for entry in "${TOPICS[@]}"; do
+    IFS=':' read -r name partitions <<< "$entry"
+    if $RPK topic list 2>/dev/null | grep -q "^${name}"; then
+        echo "  ↩  ${name} already exists"
+        continue
+    fi
+    $RPK topic create "$name" \
+        --partitions "$partitions" \
+        --replicas 1 \
+        --topic-config "retention.ms=${RETENTION_MS}" \
+        --topic-config "retention.bytes=${RETENTION_BYTES}" \
+        --topic-config "cleanup.policy=delete" \
+        --topic-config "compression.type=snappy"
+    echo "  ✓  ${name} (${partitions} partitions)"
+done
+echo "Topics ready:"
+$RPK topic list

nginx.conf ADDED Viewed

	@@ -0,0 +1,115 @@

+worker_processes auto;
+error_log /dev/stderr warn;
+pid /tmp/nginx.pid;
+events {
+    worker_connections 1024;
+}
+http {
+    access_log /dev/stdout;
+    client_max_body_size 100M;
+    # ── Status dashboard at / ──────────────────────────────────────────────────
+    server {
+        listen 7860;
+        # ── Root → HTML dashboard ──────────────────────────────────────────────
+        location = / {
+            default_type text/html;
+            return 200 '<!DOCTYPE html>
+<html>
+<head>
+  <title>Redpanda</title>
+  <meta charset="utf-8">
+  <meta http-equiv="refresh" content="15">
+  <style>
+    *{box-sizing:border-box;margin:0;padding:0}
+    body{font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif;background:#0a0a0a;color:#e5e5e5;padding:40px 32px}
+    h1{font-size:1.9rem;font-weight:700;margin-bottom:6px}
+    .sub{color:#666;margin-bottom:28px;font-size:.95rem}
+    .grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(190px,1fr));gap:14px;margin-bottom:28px}
+    .card{background:#141414;border:1px solid #242424;border-radius:10px;padding:18px}
+    .card h3{font-size:.7rem;text-transform:uppercase;letter-spacing:.08em;color:#555;margin-bottom:10px}
+    .card .val{font-size:1.05rem;font-weight:600;color:#fff;font-family:monospace}
+    .card .desc{font-size:.78rem;color:#555;margin-top:4px}
+    table{width:100%;border-collapse:collapse;background:#141414;border:1px solid #242424;border-radius:10px;overflow:hidden;margin-bottom:28px}
+    th{text-align:left;padding:12px 16px;font-size:.75rem;text-transform:uppercase;letter-spacing:.07em;color:#555;border-bottom:1px solid #242424}
+    td{padding:11px 16px;font-size:.88rem;border-bottom:1px solid #1a1a1a}
+    tr:last-child td{border-bottom:none}
+    code{background:#1e1e1e;padding:2px 8px;border-radius:5px;font-size:.82rem;color:#a78bfa;font-family:monospace}
+    .badge{display:inline-block;padding:2px 10px;border-radius:999px;font-size:.75rem;font-weight:600;background:#22c55e18;color:#22c55e;border:1px solid #22c55e44}
+  </style>
+</head>
+<body>
+  <h1>🔴 Redpanda Broker</h1>
+  <p class="sub">All services accessible via path-based routing on this port</p>
+  <div class="grid">
+    <div class="card"><h3>Kafka REST Proxy</h3><div class="val">/kafka/</div><div class="desc">Produce &amp; consume over HTTP</div></div>
+    <div class="card"><h3>Admin API</h3><div class="val">/admin/</div><div class="desc">Cluster management &amp; metrics</div></div>
+    <div class="card"><h3>Schema Registry</h3><div class="val">/schema/</div><div class="desc">Avro / Protobuf schemas</div></div>
+    <div class="card"><h3>Health</h3><div class="val">/health</div><div class="desc">Broker ready check</div></div>
+  </div>
+  <table>
+    <tr><th>Action</th><th>Method</th><th>Path</th></tr>
+    <tr><td>List topics</td><td><code>GET</code></td><td><code>/admin/v1/topics</code></td></tr>
+    <tr><td>Produce messages</td><td><code>POST</code></td><td><code>/kafka/topics/{topic}</code></td></tr>
+    <tr><td>Create consumer</td><td><code>POST</code></td><td><code>/kafka/consumers/{group}</code></td></tr>
+    <tr><td>Subscribe consumer</td><td><code>POST</code></td><td><code>/kafka/consumers/{group}/instances/{id}/subscription</code></td></tr>
+    <tr><td>Consume records</td><td><code>GET</code></td><td><code>/kafka/consumers/{group}/instances/{id}/records</code></td></tr>
+    <tr><td>List schemas</td><td><code>GET</code></td><td><code>/schema/subjects</code></td></tr>
+    <tr><td>Cluster health</td><td><code>GET</code></td><td><code>/health</code></td></tr>
+  </table>
+</body>
+</html>';
+        }
+        # ── /health → Redpanda ready check ────────────────────────────────────
+        location = /health {
+            proxy_pass         http://127.0.0.1:9644/v1/status/ready;
+            proxy_set_header   Host $host;
+        }
+        # ── /kafka/ → Pandaproxy HTTP REST (port 8082) ────────────────────────
+        location /kafka/ {
+            rewrite            ^/kafka/(.*) /$1 break;
+            proxy_pass         http://127.0.0.1:8082;
+            proxy_set_header   Host $host;
+            proxy_set_header   X-Real-IP $remote_addr;
+            proxy_read_timeout 120s;
+            proxy_buffering    off;
+        }
+        # ── /admin/ → Redpanda Admin API (port 9644) ──────────────────────────
+        location /admin/ {
+            rewrite            ^/admin/(.*) /$1 break;
+            proxy_pass         http://127.0.0.1:9644;
+            proxy_set_header   Host $host;
+            proxy_set_header   X-Real-IP $remote_addr;
+            proxy_read_timeout 120s;
+        }
+        # ── /schema/ → Schema Registry (port 8081) ────────────────────────────
+        location /schema/ {
+            rewrite            ^/schema/(.*) /$1 break;
+            proxy_pass         http://127.0.0.1:8081;
+            proxy_set_header   Host $host;
+            proxy_set_header   X-Real-IP $remote_addr;
+            proxy_read_timeout 60s;
+        }
+    }
+}
+# ── TCP stream proxy for native Kafka protocol (port 9092) ────────────────────
+# NOTE: HF Spaces only exposes port 7860 via HTTPS externally.
+# The Kafka binary protocol (TCP) requires a direct TCP connection.
+# Use the HTTP proxy paths above for external access.
+# For same-network (another HF Space or internal service), connect to :9092 directly.
+stream {
+    server {
+        listen     7861;
+        proxy_pass 127.0.0.1:9092;
+    }
+}

redpanda.yaml ADDED Viewed

	@@ -0,0 +1,66 @@

+redpanda:
+  node_id: 0
+  empty_seed_starts_cluster: true
+  data_directory: /var/lib/redpanda/data
+  kafka_api:
+    - address: 0.0.0.0
+      port: 9092
+      name: internal
+  advertised_kafka_api:
+    - address: 0.0.0.0
+      port: 9092
+      name: internal
+  rpc_server:
+    address: 0.0.0.0
+    port: 33145
+  admin:
+    - address: 0.0.0.0
+      port: 9644
+  # ── HF Spaces resource limits ───────────────────────────────────────────────
+  developer_mode: true
+  overprovisioned: true
+  # ── Retention: short since you store data yourself ──────────────────────────
+  log_retention_ms: 3600000        # 1 hour
+  log_retention_bytes: 536870912   # 512MB per partition
+  log_segment_size: 67108864       # 64MB segments
+  log_cleanup_policy: delete
+rpk:
+  kafka_api:
+    brokers:
+      - 127.0.0.1:9092
+  admin_api:
+    addresses:
+      - 127.0.0.1:9644
+pandaproxy:
+  pandaproxy_api:
+    - address: 0.0.0.0
+      port: 8082
+      name: internal
+  advertised_pandaproxy_api:
+    - address: 0.0.0.0
+      port: 8082
+      name: internal
+pandaproxy_client:
+  brokers:
+    - address: 127.0.0.1
+      port: 9092
+schema_registry:
+  schema_registry_api:
+    - address: 0.0.0.0
+      port: 8081
+      name: internal
+schema_registry_client:
+  brokers:
+    - address: 127.0.0.1
+      port: 9092

start.sh ADDED Viewed

	@@ -0,0 +1,71 @@

+#!/bin/bash
+set -euo pipefail
+echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+echo "  Redpanda → HF Spaces"
+echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+# ── 1. Start Redpanda ─────────────────────────────────────────────────────────
+echo "[1/4] Starting Redpanda..."
+/usr/bin/redpanda start \
+    --overprovisioned \
+    --smp 1 \
+    --memory 512M \
+    --reserve-memory 0M \
+    --node-id 0 \
+    --check=false \
+    --kafka-addr PLAINTEXT://0.0.0.0:9092 \
+    --advertise-kafka-addr PLAINTEXT://0.0.0.0:9092 \
+    --pandaproxy-addr 0.0.0.0:8082 \
+    --advertise-pandaproxy-addr 0.0.0.0:8082 \
+    --schema-registry-addr 0.0.0.0:8081 \
+    --rpc-addr 0.0.0.0:33145 \
+    --advertise-rpc-addr 0.0.0.0:33145 \
+    --mode dev-container &
+REDPANDA_PID=$!
+# ── 2. Wait for broker ready ──────────────────────────────────────────────────
+echo "[2/4] Waiting for Redpanda..."
+ATTEMPTS=0
+until curl -sf http://127.0.0.1:9644/v1/status/ready > /dev/null 2>&1; do
+    ATTEMPTS=$((ATTEMPTS+1))
+    if [ $ATTEMPTS -gt 60 ]; then
+        echo "ERROR: Redpanda failed to start after 120s"
+        exit 1
+    fi
+    echo "  ... waiting (${ATTEMPTS}/60)"
+    sleep 2
+done
+echo "  ✓ Redpanda ready"
+# ── 3. Create topics ──────────────────────────────────────────────────────────
+echo "[3/4] Initializing topics..."
+/init-topics.sh
+# ── 4. Start nginx (routes everything through port 7860) ─────────────────────
+echo "[4/4] Starting nginx on :7860..."
+nginx -g "daemon off;" &
+NGINX_PID=$!
+echo ""
+echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+echo "  ✅ All services up on port 7860"
+echo ""
+echo "  GET  /health              → broker status"
+echo "  *    /kafka/...           → HTTP REST proxy"
+echo "  *    /admin/...           → admin API"
+echo "  *    /schema/...          → schema registry"
+echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+# ── Trap shutdown signals ─────────────────────────────────────────────────────
+cleanup() {
+    echo "Shutting down..."
+    kill $NGINX_PID 2>/dev/null || true
+    kill $REDPANDA_PID 2>/dev/null || true
+    exit 0
+}
+trap cleanup SIGTERM SIGINT
+# ── Keep alive ────────────────────────────────────────────────────────────────
+wait $REDPANDA_PID