docker/scripts/start_space.sh

#!/bin/bash

# Fail on error
set -e

# Ensure DVC is initialized (in case .dvc folder was not copied)
if [ ! -d ".dvc" ]; then
    echo "Initializing DVC..."
    dvc init --no-scm
    dvc remote add -d origin https://dagshub.com/se4ai2526-uniba/Hopcroft.dvc
fi

# Determine credentials
# Prefer specific DAGSHUB vars, fallback to MLFLOW vars (often the same for DagsHub)
USER=${DAGSHUB_USERNAME:-$MLFLOW_TRACKING_USERNAME}
PASS=${DAGSHUB_TOKEN:-$MLFLOW_TRACKING_PASSWORD}

if [ -n "$USER" ] && [ -n "$PASS" ]; then
    echo "$(date) - Configuring DVC authentication for DagsHub..."
    # Configure local config (not committed)
    dvc remote modify origin --local auth basic
    dvc remote modify origin --local user "$USER"
    dvc remote modify origin --local password "$PASS"
else
    echo "$(date) - WARNING: No DagsHub credentials found. DVC pull might fail if the remote is private."
fi

echo "$(date) - Pulling models from DVC..."
# Pull only the necessary files for inference
dvc pull models/random_forest_tfidf_gridsearch.pkl.dvc \
         models/tfidf_vectorizer.pkl.dvc \
         models/label_names.pkl.dvc || echo "DVC pull failed, but continuing..."

# Create Nginx temp directories
mkdir -p /tmp/client_temp /tmp/proxy_temp /tmp/fastcgi_temp /tmp/uwsgi_temp /tmp/scgi_temp

echo "$(date) - Checking models existence..."
ls -la models/

echo "$(date) - Starting FastAPI application in background..."
# Using 0.0.0.0 to be safe
uvicorn hopcroft_skill_classification_tool_competition.main:app --host 0.0.0.0 --port 8000 >> /tmp/fastapi.log 2>&1 &

# Wait for API to start
echo "$(date) - Waiting for API to start (30s)..."
for i in {1..30}; do
    if curl -s http://127.0.0.1:8000/health > /dev/null; then
        echo "$(date) - API is UP!"
        break
    fi
    echo "$(date) - Waiting... ($i/30)"
    sleep 2
done

echo "$(date) - Configuring and starting Prometheus..."
# Patch Grafana Datasource for Localhost (HF Space) and fix URL path
# Replace prometheus:9090 with 127.0.0.1:9090/prometheus in all datasource configs
find /app/monitoring/grafana/provisioning/datasources -name '*.yml' -exec sed -i 's/prometheus:9090/127.0.0.1:9090\/prometheus/g' {} +

# Copy production configs to /tmp for modification
cp /etc/prometheus/prometheus.yml /tmp/prometheus.yml
cp /etc/prometheus/alert_rules.yml /tmp/alert_rules.yml
cp /etc/alertmanager/config.yml /tmp/alertmanager.yml

# Modify Prometheus config for local execution (replace docker-compose service names with localhost)
# hopcroft-api:8080 -> 127.0.0.1:8000 (API runs on 8000 in Space)
sed -i 's/hopcroft-api:8080/127.0.0.1:8000/g' /tmp/prometheus.yml
# Alertmanager: hopcroft-api:8080 -> 127.0.0.1:8000
sed -i 's/hopcroft-api:8080/127.0.0.1:8000/g' /tmp/alertmanager.yml
# alertmanager:9093 -> 127.0.0.1:9093
sed -i 's/alertmanager:9093/127.0.0.1:9093/g' /tmp/prometheus.yml
# pushgateway:9091 -> 127.0.0.1:9091
sed -i 's/pushgateway:9091/127.0.0.1:9091/g' /tmp/prometheus.yml
# Fix alert_rules path to be absolute or relative to execution
sed -i 's|"alert_rules.yml"|"/tmp/alert_rules.yml"|g' /tmp/prometheus.yml

# FIX: Add path prefixes to match --web.route-prefix arguments
# Add metrics_path for self-scraping prometheus
sed -i 's/job_name: "prometheus"/job_name: "prometheus"\n    metrics_path: "\/prometheus\/metrics"/g' /tmp/prometheus.yml
# Add metrics_path for pushgateway
sed -i 's/job_name: "pushgateway"/job_name: "pushgateway"\n    metrics_path: "\/pushgateway\/metrics"/g' /tmp/prometheus.yml
# Add path_prefix for Alertmanager
sed -i 's/    - static_configs:/    - path_prefix: "\/alertmanager\/"\n      static_configs:/g' /tmp/prometheus.yml

echo "$(date) - Starting Alertmanager..."
alertmanager \
    --config.file=/tmp/alertmanager.yml \
    --storage.path=/tmp/alertmanager_data \
    --web.route-prefix=/alertmanager/ \
    >> /tmp/alertmanager.log 2>&1 &

echo "$(date) - Starting Pushgateway..."
pushgateway \
    --persistence.file=/tmp/pushgateway_data \
    --web.route-prefix=/pushgateway/ \
    >> /tmp/pushgateway.log 2>&1 &

# Determine Prometheus External URL
# Always use relative path so it works on both huggingface.co and .hf.space domains
PROM_EXTERNAL_URL="/prometheus/"

# Start Prometheus
# --web.external-url needs to match the public URL for correct link generation
# --web.route-prefix needs to match the path Nginx proxies to (/prometheus/)
prometheus \
    --config.file=/tmp/prometheus.yml \
    --storage.tsdb.path=/tmp/prometheus_data \
    --web.listen-address=0.0.0.0:9090 \
    --web.external-url=$PROM_EXTERNAL_URL \
    --web.route-prefix=/prometheus/ \
    >> /tmp/prometheus.log 2>&1 &

# Start Grafana
if [ -n "$SPACE_ID" ]; then
    # Parse username and space name from SPACE_ID (user/space)
    # This allows us to construct the .hf.space domain which avoids CORS/Asset loading issues
    SPACE_AUTHOR=$(echo $SPACE_ID | cut -d'/' -f1)
    SPACE_NAME=$(echo $SPACE_ID | cut -d'/' -f2)
    SPACE_HOST="${SPACE_AUTHOR}-${SPACE_NAME}.hf.space"
    
    echo "$(date) - Detected HF Space environment (ID: $SPACE_ID). Configured Host: $SPACE_HOST"
    GRAFANA_ROOT_URL="https://$SPACE_HOST/grafana/"
else
    echo "$(date) - No SPACE_ID found. Defaulting Grafana to localhost."
    GRAFANA_ROOT_URL="http://localhost:3000/grafana/"
fi

# Locate Grafana binary
GRAFANA_BIN=$(which grafana-server || echo "/usr/sbin/grafana-server")
echo "$(date) - Found Grafana binary at: $GRAFANA_BIN"

echo "$(date) - Starting Grafana with Root URL: $GRAFANA_ROOT_URL"

# Use the project's grafana.ini which we have permissions to read
$GRAFANA_BIN --homepath=/usr/share/grafana \
    --config=/app/monitoring/grafana/grafana.ini \
    cfg:paths.data=/tmp/grafana_data \
    cfg:paths.logs=/tmp/grafana_logs \
    cfg:paths.plugins=/usr/share/grafana/plugins \
    cfg:paths.provisioning=/app/monitoring/grafana/provisioning \
    cfg:server.root_url="$GRAFANA_ROOT_URL" \
    cfg:server.serve_from_sub_path=true \
    cfg:server.http_port=3000 \
    > /tmp/grafana.log 2>&1 &

# Wait for Grafana to start
echo "$(date) - Waiting for Grafana (20s)..."
for i in {1..20}; do
    if curl -s http://127.0.0.1:3000/api/health > /dev/null; then
        echo "$(date) - Grafana is UP!"
        # Debug: Check what Grafana responds at root
        echo "$(date) - VERIFYING GRAFANA ROOT RESPONSE:"
        curl -v http://127.0.0.1:3000/ 2>&1 | head -n 20
        break
    fi
    sleep 1
done

# If Grafana is still down, print logs
if ! curl -s http://127.0.0.1:3000/api/health > /dev/null; then
    echo "$(date) - ERROR: Grafana failed to start within 20 seconds. Dumping logs:"
    cat /tmp/grafana.log
fi


echo "$(date) - Starting Nginx reverse proxy..."
if ! command -v nginx &> /dev/null; then
    echo "$(date) - ERROR: nginx not found in PATH"
    exit 1
fi
nginx -c /app/docker/nginx.conf -g "daemon off;" >> /tmp/nginx_startup.log 2>&1 &

echo "$(date) - Waiting for Nginx to initialize..."
sleep 5

# Check if Nginx is running
if ps aux | grep -v grep | grep -q "nginx"; then
    echo "$(date) - Nginx is running."
else
    echo "$(date) - ERROR: Nginx failed to start. Logs:"
    cat /tmp/nginx_startup.log
fi

echo "$(date) - Final backend check before starting Streamlit..."
curl -v http://127.0.0.1:8000/health || echo "FastAPI health check failed!"

echo "$(date) - Starting Streamlit application on 127.0.0.1:8501..."
export API_BASE_URL="http://127.0.0.1:8000"
streamlit run hopcroft_skill_classification_tool_competition/streamlit_app.py \
    --server.port 8501 \
    --server.address 127.0.0.1 \
    --server.enableCORS=false \
    --server.enableXsrfProtection=false \
    --server.headless true &

# Wait for Streamlit to start
echo "$(date) - Waiting for Streamlit to start (30s)..."
for i in {1..30}; do
    if curl -s http://127.0.0.1:8501/healthz > /dev/null; then
        echo "$(date) - Streamlit is UP!"
        break
    fi
    echo "$(date) - Waiting for Streamlit... ($i/30)"
    sleep 2
done

echo "$(date) - Process started. Tailing logs for debug..."
tail -f /tmp/nginx_startup.log /tmp/fastapi.log /tmp/grafana.log /tmp/prometheus.log