Upload 5 files

Dockerfile

@@ -1,27 +1,32 @@
 FROM ubuntu:22.04
 
-# Avoid tzdata interactive prompt during apt-get install
 ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update && apt-get install -y \
-    git curl python3 python3-pip
+    git curl python3 python3-pip \
+    && rm -rf /var/lib/apt/lists/*
 
-# Install ClickHouse
+# Install ClickHouse static binary
 RUN curl https://clickhouse.com/ | sh && \
-    mv clickhouse /usr/local/bin/
+    mv clickhouse /usr/local/bin/ && \
+    chmod +x /usr/local/bin/clickhouse
 
-# Install Python requirements
+WORKDIR /app
+
+# Install Python deps first (layer cache)
 COPY requirements.txt .
 RUN pip3 install --no-cache-dir -r requirements.txt
 
-WORKDIR /app
-# Copy application files to root level as requested
+# Copy all app files
 COPY . .
 
-# IMPORTANT: Make /app writable by any user, as HF Spaces runs containers as user 1000 by default
-RUN chmod 777 -R /app && chmod +x init_clickhouse.sh
+# Create all directories ClickHouse needs — writable by uid 1000 (HF default user)
+RUN mkdir -p /app/ch/data /app/ch/tmp /app/ch/user_files /app/ch/format_schemas \
+             /app/ch/access /app/ch/log /app/ch/metadata /app/ch/store \
+             /app/data && \
+    chmod -R 777 /app
 
-# HF Spaces run on port 7860 by default
+# HF Spaces only exposes this port
 EXPOSE 7860
 
 CMD ["bash", "init_clickhouse.sh"]

clickhouse-config.xml

@@ -0,0 +1,36 @@
+<clickhouse>
+    <!-- All paths under /app/ch which we chmod 777 at build time -->
+    <path>/app/ch/data/</path>
+    <tmp_path>/app/ch/tmp/</tmp_path>
+    <user_files_path>/app/ch/user_files/</user_files_path>
+    <format_schema_path>/app/ch/format_schemas/</format_schema_path>
+    <access_control_path>/app/ch/access/</access_control_path>
+
+    <logger>
+        <log>/app/ch/log/clickhouse-server.log</log>
+        <errorlog>/app/ch/log/clickhouse-server.err.log</errorlog>
+        <level>warning</level>
+        <console>0</console>
+    </logger>
+
+    <!-- Only listen locally — HF exposes only 7860, we proxy internally -->
+    <listen_host>127.0.0.1</listen_host>
+    <http_port>8123</http_port>
+    <tcp_port>9000</tcp_port>
+
+    <!-- Disable interserver and other unneeded ports -->
+    <mysql_port remove="remove"/>
+    <postgresql_port remove="remove"/>
+    <interserver_http_port remove="remove"/>
+    <grpc_port remove="remove"/>
+
+    <!-- Relax security for single-user embedded mode -->
+    <openSSL>
+        <server>
+            <verificationMode>none</verificationMode>
+        </server>
+    </openSSL>
+
+    <mark_cache_size>1073741824</mark_cache_size>
+    <max_concurrent_queries>10</max_concurrent_queries>
+</clickhouse>

init_clickhouse.sh

@@ -1,39 +1,55 @@
 #!/bin/bash
+set -e
 
-# Define data directory locally on HF Spaces persistent/ephemeral storage
-DATA_DIR="/app/data"
-mkdir -p $DATA_DIR/user_files
-cd $DATA_DIR
+echo "=== Market-Data Observatory Boot Sequence ==="
 
-# Clone the repository if it doesn't exist
-if [ ! -d "ohlc_data" ]; then
-  git clone https://github.com/subhamgiri460/ohlc_data.git
+# ── 1. Clone / refresh the parquet repo ──
+echo "[1/4] Syncing parquet data from GitHub..."
+if [ ! -d "/app/data/ohlc_data" ]; then
+  git clone --depth 1 https://github.com/subhamgiri460/ohlc_data.git /app/data/ohlc_data
 else
-  # Perform a pull if it does exist
-  cd ohlc_data
-  git pull
-  cd ..
+  cd /app/data/ohlc_data && git pull && cd /app
 fi
 
-# Link it to user_files
-ln -sfn $DATA_DIR/ohlc_data $DATA_DIR/user_files/ohlc_data
+# ── 2. Symlink repo into ClickHouse user_files ──
+echo "[2/4] Linking data into ClickHouse user_files..."
+ln -sfn /app/data/ohlc_data /app/ch/user_files/ohlc_data
 
-# Start ClickHouse server in the background
-clickhouse server --daemon --user_files_path=$DATA_DIR/user_files
+# ── 3. Start ClickHouse in background (NOT --daemon, use & for reliability) ──
+echo "[3/4] Starting ClickHouse server..."
+clickhouse server --config-file=/app/clickhouse-config.xml &
+CH_PID=$!
 
-# Give ClickHouse a little time to spin up
-echo "Waiting for ClickHouse to start..."
-sleep 5
+# Wait until ClickHouse is genuinely ready (poll the HTTP port)
+echo "       Waiting for ClickHouse to become ready..."
+for i in $(seq 1 30); do
+  if curl -sf http://127.0.0.1:8123/ping > /dev/null 2>&1; then
+    echo "       ClickHouse is UP (took ${i}s)"
+    break
+  fi
+  if [ $i -eq 30 ]; then
+    echo "       ERROR: ClickHouse failed to start within 30s"
+    echo "       Last log lines:"
+    tail -20 /app/ch/log/clickhouse-server.err.log 2>/dev/null || true
+    exit 1
+  fi
+  sleep 1
+done
 
-# Create the view pointing to the parquet files
-# Note: Parquet files inside directories like year=.../month=... 
-# will be successfully read by ClickHouse format engine
+# ── 4. Create the lakehouse view ──
+echo "[4/4] Creating OHLC view over parquet files..."
 clickhouse client --query "
 CREATE VIEW IF NOT EXISTS ohlc AS
 SELECT *
 FROM file('ohlc_data/**/*.parquet', Parquet)
 "
 
-# Start the FastAPI server on Hugging Face Spaces port 7860
-cd /app
-uvicorn server:app --host 0.0.0.0 --port 7860
+echo ""
+echo "=== Observatory is LIVE on port 7860 ==="
+echo "    /play      → ClickHouse SQL playground"
+echo "    /dashboard → ClickHouse dashboard"
+echo "    POST /refresh → git pull latest data"
+echo ""
+
+# Start FastAPI — this is the foreground process that keeps the container alive
+exec uvicorn server:app --host 0.0.0.0 --port 7860

server.py

@@ -1,54 +1,68 @@
 from fastapi import FastAPI, Request, HTTPException
-from fastapi.responses import StreamingResponse
+from fastapi.responses import StreamingResponse, HTMLResponse
 import subprocess
 import httpx
 
 app = FastAPI(title="Market-Data Observatory")
 
-# HTTP client for proxying to ClickHouse
-ch_client = httpx.AsyncClient(base_url="http://127.0.0.1:8123")
+# Async HTTP client for proxying to ClickHouse's internal HTTP interface
+ch_client = httpx.AsyncClient(base_url="http://127.0.0.1:8123", timeout=60.0)
 
+
+# ── Refresh endpoint (git pull) ──
 @app.post("/refresh")
 def refresh():
     try:
-        # Simply cd into the local clone and git pull to hydrate the "lake"
-        result = subprocess.check_output("cd /app/data/ohlc_data && git pull", shell=True)
-        return {"status": "updated", "details": result.decode()}
+        result = subprocess.check_output(
+            "cd /app/data/ohlc_data && git pull",
+            shell=True, stderr=subprocess.STDOUT
+        )
+        return {"status": "updated", "details": result.decode().strip()}
     except subprocess.CalledProcessError as e:
-        error_msg = e.output.decode() if getattr(e, 'output', None) else str(e)
-        raise HTTPException(status_code=500, detail={"error": "Refresh Failed", "message": error_msg})
+        msg = e.output.decode() if e.output else str(e)
+        raise HTTPException(status_code=500, detail={"error": "Refresh failed", "message": msg})
+
+
+# ── Health check ──
+@app.get("/health")
+async def health():
+    try:
+        r = await ch_client.get("/ping")
+        return {"clickhouse": r.text.strip(), "status": "ok"}
+    except Exception as e:
+        raise HTTPException(status_code=503, detail={"status": "unhealthy", "error": str(e)})
+
 
-# Reverse proxy all other routes to ClickHouse so you get the full UI
+# ── Catch-all reverse proxy to ClickHouse ──
+# This gives you /play, /dashboard, native HTTP API, everything.
 @app.api_route("/{path:path}", methods=["GET", "POST", "PUT", "DELETE", "OPTIONS", "HEAD", "PATCH"])
-async def proxy_clickhouse(request: Request, path: str):
-    # Reconstruct the URL for the proxy
-    url = httpx.URL(path=request.url.path, query=request.url.query.encode("utf-8"))
-    
-    # Forward the headers, stripping any Hop-by-hop headers
+async def proxy_to_clickhouse(request: Request, path: str):
+    url = httpx.URL(path=f"/{path}", query=request.url.query.encode("utf-8"))
+
+    # Forward headers, strip hop-by-hop
     headers = dict(request.headers)
-    headers.pop("host", None)
-    headers.pop("content-length", None)
+    for h in ("host", "content-length", "transfer-encoding"):
+        headers.pop(h, None)
 
     try:
-        # Stream the request body to ClickHouse
+        body = await request.body()
         req = ch_client.build_request(
-            request.method,
-            url,
+            method=request.method,
+            url=url,
             headers=headers,
-            content=request.stream()
+            content=body,
         )
-        # Send the proxy request
         r = await ch_client.send(req, stream=True)
-        
-        # Prepare response headers
+
+        # Pass through ClickHouse response headers
         resp_headers = dict(r.headers)
-        resp_headers.pop("content-length", None)
-        resp_headers.pop("content-encoding", None)
+        for h in ("content-length", "content-encoding", "transfer-encoding"):
+            resp_headers.pop(h, None)
 
         return StreamingResponse(
             r.aiter_raw(),
             status_code=r.status_code,
-            headers=resp_headers
+            headers=resp_headers,
         )
     except httpx.RequestError as exc:
-        raise HTTPException(status_code=502, detail=f"Proxy error to ClickHouse: {exc}")
+        raise HTTPException(status_code=502, detail=f"ClickHouse proxy error: {exc}")