Upload 7 files

Dockerfile

@@ -1,24 +1,37 @@
-FROM apache/airflow:2.9.2-python3.11
+FROM ubuntu:22.04
 
-USER root
-RUN apt-get update && apt-get install -y git && apt-get clean && rm -rf /var/lib/apt/lists/*
+ENV DEBIAN_FRONTEND=noninteractive
+ENV AIRFLOW_HOME=/opt/airflow
 
-COPY startup.sh /startup.sh
-RUN chmod +x /startup.sh
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    python3 \
+    python3-pip \
+    python3-venv \
+    nginx \
+    git \
+    supervisor \
+    curl \
+    && rm -rf /var/lib/apt/lists/* \
+    && useradd -m -u 1000 airflow
+
+RUN mkdir -p /var/log/supervisor /var/log/nginx /run/nginx /var/lib/nginx /opt/airflow/dags \
+    && chown -R airflow:airflow /opt/airflow /var/log/supervisor /var/log/nginx /run/nginx /var/lib/nginx
 
 USER airflow
 
-ENV AIRFLOW__CORE__LOAD_EXAMPLES=False
-ENV AIRFLOW__CORE__EXECUTOR=LocalExecutor
-ENV AIRFLOW__CORE__DAGS_FOLDER=/opt/airflow/dags
-ENV AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL=30
-ENV AIRFLOW__LOGGING__LOGGING_LEVEL=ERROR
-ENV AIRFLOW__LOGGING__BASE_LOG_FOLDER=/tmp/airflow/logs
-ENV AIRFLOW__LOGGING__DAG_PROCESSOR_MANAGER_LOG_LOCATION=/tmp/airflow/logs/dag_processor_manager/dag_processor_manager.log
-ENV AIRFLOW__SCHEDULER__CHILD_PROCESS_LOG_DIRECTORY=/tmp/airflow/logs/scheduler
-ENV AIRFLOW__WEBSERVER__EXPOSE_CONFIG=False
-ENV AIRFLOW__WEBSERVER__WEB_SERVER_PORT=7860
+ENV PATH="/home/airflow/.local/bin:${PATH}"
+
+COPY --chown=airflow:airflow requirements.txt /tmp/requirements.txt
+RUN pip install --no-cache-dir -r /tmp/requirements.txt
+
+COPY --chown=airflow:airflow config.sh /config.sh
+COPY --chown=airflow:airflow supervisord.conf /etc/supervisord.conf
+COPY --chown=airflow:airflow nginx.conf /etc/nginx/nginx.conf
+COPY --chown=airflow:airflow entrypoint.sh /entrypoint.sh
+COPY --chown=airflow:airflow refresh_service.py /refresh_service.py
+
+RUN chmod +x /entrypoint.sh /config.sh
 
 EXPOSE 7860
 
-ENTRYPOINT ["/startup.sh"]
+ENTRYPOINT ["/bin/bash", "/entrypoint.sh"]

config.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+# ===========================================
+# Airflow Configuration - SINGLE SOURCE OF TRUTH
+# ===========================================
+# All configurable values are defined here with defaults
+# Override by setting environment variables or HF Secrets
+
+# ---- Directory Paths ----
+export AIRFLOW_HOME="${AIRFLOW_HOME:-/opt/airflow}"
+export DAGS_DIR="${AIRFLOW_HOME}/dags"
+export LOGS_DIR="${AIRFLOW_HOME}/logs"
+
+# ---- Database Connection (Set via HF Secret for production) ----
+export AIRFLOW__DATABASE__SQL_ALCHEMY_CONN="${AIRFLOW__DATABASE__SQL_ALCHEMY_CONN:-postgresql://postgres.ffetkispaxytayrnmxvj:uKInpRkBXsCUfVg1@aws-1-ap-south-1.pooler.supabase.com:5432/postgres}"
+
+# ---- DAG Repository ----
+export DAG_REPO_URL="${DAG_REPO_URL:-https://github.com/subhamgiri460/myworkflows.git}"
+export DAG_REPO_BRANCH="${DAG_REPO_BRANCH:-main}"
+
+# ---- Admin Credentials (Set via HF Secrets for production) ----
+export AIRFLOW_ADMIN_USER="${AIRFLOW_ADMIN_USER:-admin}"
+export AIRFLOW_ADMIN_PASSWORD="${AIRFLOW_ADMIN_PASSWORD:-admin}"
+
+# ---- Internal Ports ----
+export AIRFLOW_WEB_PORT="${AIRFLOW_WEB_PORT:-8080}"
+export REFRESH_SERVICE_PORT="${REFRESH_SERVICE_PORT:-5000}"
+export NGINX_PORT="${NGINX_PORT:-7860}"
+
+# ---- Airflow Core Settings ----
+export AIRFLOW__CORE__LOAD_EXAMPLES="${AIRFLOW__CORE__LOAD_EXAMPLES:-False}"
+export AIRFLOW__CORE__EXECUTOR="${AIRFLOW__CORE__EXECUTOR:-LocalExecutor}"
+
+# ---- Airflow Webserver Settings ----
+export AIRFLOW__WEBSERVER__WEB_SERVER_PORT="${AIRFLOW_WEB_PORT}"
+export AIRFLOW__WEBSERVER__WEB_SERVER_MASTER_TIMEOUT="${AIRFLOW__WEBSERVER__WEB_SERVER_MASTER_TIMEOUT:-300}"
+export AIRFLOW__WEBSERVER__WORKER_CLASS="${AIRFLOW__WEBSERVER__WORKER_CLASS:-gevent}"
+
+# ---- Airflow Scheduler Settings ----
+export AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL="${AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL:-30}"

entrypoint.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+set -e
+
+echo "=== Loading configuration ==="
+source /config.sh
+
+echo "Configuration:"
+echo "  DAG_REPO_URL: ${DAG_REPO_URL}"
+echo "  DAG_REPO_BRANCH: ${DAG_REPO_BRANCH}"
+echo "  DAGS_DIR: ${DAGS_DIR}"
+echo "  AIRFLOW_WEB_PORT: ${AIRFLOW_WEB_PORT}"
+echo "  REFRESH_SERVICE_PORT: ${REFRESH_SERVICE_PORT}"
+
+echo ""
+echo "=== Syncing DAG repository ==="
+if [ ! -d "${DAGS_DIR}/.git" ]; then
+    rm -rf "${DAGS_DIR:?}"/* 2>/dev/null || true
+    git clone --depth 1 --branch "${DAG_REPO_BRANCH}" "${DAG_REPO_URL}" "${DAGS_DIR}"
+    echo "Repository cloned successfully"
+else
+    cd "${DAGS_DIR}"
+    git fetch origin "${DAG_REPO_BRANCH}" && git reset --hard "origin/${DAG_REPO_BRANCH}"
+    echo "Repository updated successfully"
+fi
+
+echo ""
+echo "=== Running database migrations ==="
+airflow db migrate || true
+
+echo ""
+echo "=== Creating admin user ==="
+airflow users create \
+    --username "${AIRFLOW_ADMIN_USER}" \
+    --firstname Admin \
+    --lastname User \
+    --role Admin \
+    --email admin@example.com \
+    --password "${AIRFLOW_ADMIN_PASSWORD}" || true
+
+echo ""
+echo "=== Starting services ==="
+exec /usr/bin/supervisord -c /etc/supervisord.conf

nginx.conf

@@ -0,0 +1,65 @@
+worker_processes 1;
+error_log /var/log/nginx/error.log warn;
+pid /run/nginx/nginx.pid;
+
+events {
+    worker_connections 1024;
+}
+
+http {
+    include /etc/nginx/mime.types;
+    default_type application/octet-stream;
+    
+    log_format main '$remote_addr - $remote_user [$time_local] "$request" '
+                    '$status $body_bytes_sent "$http_referer" "$http_user_agent"';
+    
+    access_log /var/log/nginx/access.log main;
+    sendfile on;
+    keepalive_timeout 65;
+    
+    client_body_temp_path /run/nginx/client_temp;
+    proxy_temp_path       /run/nginx/proxy_temp;
+    fastcgi_temp_path     /run/nginx/fastcgi_temp;
+    uwsgi_temp_path       /run/nginx/uwsgi_temp;
+    scgi_temp_path        /run/nginx/scgi_temp;
+    
+    upstream airflow {
+        server 127.0.0.1:8080;
+    }
+    
+    upstream refresh {
+        server 127.0.0.1:5000;
+    }
+    
+    server {
+        listen 7860;
+        server_name _;
+        client_max_body_size 100M;
+        
+        location /refresh {
+            proxy_pass http://refresh;
+            proxy_set_header Host $host;
+            proxy_read_timeout 120s;
+        }
+        
+        location /config {
+            proxy_pass http://refresh;
+            proxy_set_header Host $host;
+        }
+        
+        location /health {
+            proxy_pass http://refresh;
+            proxy_set_header Host $host;
+        }
+        
+        location / {
+            proxy_pass http://airflow;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_read_timeout 300s;
+            proxy_connect_timeout 75s;
+        }
+    }
+}

refresh_service.py

@@ -0,0 +1,76 @@
+import os
+from flask import Flask, jsonify
+import subprocess
+
+app = Flask(__name__)
+
+DAGS_DIR = os.environ.get('AIRFLOW_HOME', '/opt/airflow') + '/dags'
+DAG_REPO_URL = os.environ.get('DAG_REPO_URL', 'https://github.com/subhamgiri460/myworkflows.git')
+DAG_REPO_BRANCH = os.environ.get('DAG_REPO_BRANCH', 'main')
+
+
+@app.route('/refresh', methods=['GET', 'POST'])
+def refresh_dags():
+    try:
+        if os.path.exists(os.path.join(DAGS_DIR, '.git')):
+            fetch_result = subprocess.run(
+                ['git', 'fetch', 'origin', DAG_REPO_BRANCH],
+                cwd=DAGS_DIR, capture_output=True, text=True, timeout=60
+            )
+            if fetch_result.returncode != 0:
+                return jsonify({
+                    'status': 'error',
+                    'message': f'Git fetch failed: {fetch_result.stderr}'
+                }), 500
+            
+            reset_result = subprocess.run(
+                ['git', 'reset', '--hard', f'origin/{DAG_REPO_BRANCH}'],
+                cwd=DAGS_DIR, capture_output=True, text=True, timeout=60
+            )
+            result = reset_result
+            action = 'updated'
+        else:
+            result = subprocess.run(
+                ['git', 'clone', '--depth', '1', '--branch', DAG_REPO_BRANCH, DAG_REPO_URL, DAGS_DIR],
+                capture_output=True, text=True, timeout=120
+            )
+            action = 'cloned'
+        
+        if result.returncode == 0:
+            return jsonify({
+                'status': 'success',
+                'message': f'DAGs {action} successfully',
+                'repo': DAG_REPO_URL,
+                'branch': DAG_REPO_BRANCH
+            }), 200
+        else:
+            return jsonify({
+                'status': 'error',
+                'message': result.stderr or 'Unknown error'
+            }), 500
+            
+    except subprocess.TimeoutExpired:
+        return jsonify({'status': 'error', 'message': 'Operation timed out'}), 500
+    except Exception as e:
+        return jsonify({'status': 'error', 'message': str(e)}), 500
+
+
+@app.route('/health')
+def health():
+    return jsonify({'status': 'healthy'}), 200
+
+
+@app.route('/config')
+def show_config():
+    return jsonify({
+        'dags_dir': DAGS_DIR,
+        'repo_url': DAG_REPO_URL,
+        'branch': DAG_REPO_BRANCH,
+        'airflow_web_port': os.environ.get('AIRFLOW_WEB_PORT', '8080'),
+        'refresh_service_port': os.environ.get('REFRESH_SERVICE_PORT', '5000')
+    }), 200
+
+
+if __name__ == '__main__':
+    port = int(os.environ.get('REFRESH_SERVICE_PORT', 5000))
+    app.run(host='0.0.0.0', port=port)

requirements.txt

@@ -1,2 +1,7 @@
-apache-airflow-providers-http
-apache-airflow-providers-common-sql
+apache-airflow[postgres]==2.10.3
+apache-airflow-providers-postgres
+apache-airflow-providers-ssh
+flask
+gunicorn
+gevent
+requests

supervisord.conf

@@ -0,0 +1,38 @@
+[supervisord]
+nodaemon=true
+user=airflow
+logfile=/var/log/supervisor/supervisord.log
+pidfile=/run/nginx/supervisord.pid
+loglevel=info
+
+[program:nginx]
+command=/usr/sbin/nginx -g "daemon off;"
+autostart=true
+autorestart=true
+stderr_logfile=/var/log/supervisor/nginx.err.log
+stdout_logfile=/var/log/supervisor/nginx.out.log
+priority=5
+
+[program:airflow-webserver]
+command=airflow webserver
+autostart=true
+autorestart=true
+stderr_logfile=/var/log/supervisor/webserver.err.log
+stdout_logfile=/var/log/supervisor/webserver.out.log
+priority=20
+
+[program:airflow-scheduler]
+command=airflow scheduler
+autostart=true
+autorestart=true
+stderr_logfile=/var/log/supervisor/scheduler.err.log
+stdout_logfile=/var/log/supervisor/scheduler.out.log
+priority=30
+
+[program:refresh-service]
+command=gunicorn --bind 127.0.0.1:5000 --workers 1 --worker-class gevent --chdir / refresh_service:app
+autostart=true
+autorestart=true
+stderr_logfile=/var/log/supervisor/refresh.err.log
+stdout_logfile=/var/log/supervisor/refresh.out.log
+priority=10