File size: 6,803 Bytes
e391a84 a2d73c6 3bb4e96 aa0e77f a2d73c6 3bb4e96 a2d73c6 aa0e77f 3bb4e96 aa0e77f 3bb4e96 aa0e77f 3bb4e96 aa0e77f 3bb4e96 aa0e77f 3bb4e96 aa0e77f a2d73c6 aa0e77f a2d73c6 aa0e77f a2d73c6 3bb4e96 aa0e77f a2d73c6 aa0e77f a2d73c6 aa0e77f a2d73c6 e391a84 ce458f9 e391a84 083898e a2d73c6 e391a84 a2d73c6 e391a84 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 | import os
import shutil
import json
def build_deploy_dir():
# Paths
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
deploy_dir = os.path.join(base_dir, "kaggle_deploy")
# Clean and recreate deploy_dir
if os.path.exists(deploy_dir):
shutil.rmtree(deploy_dir)
os.makedirs(deploy_dir, exist_ok=True)
# Copy src, pyproject.toml
shutil.copytree(os.path.join(base_dir, "src"), os.path.join(deploy_dir, "src"))
shutil.copy(os.path.join(base_dir, "pyproject.toml"), os.path.join(deploy_dir, "pyproject.toml"))
if os.path.exists(os.path.join(base_dir, "README.md")):
shutil.copy(os.path.join(base_dir, "README.md"), os.path.join(deploy_dir, "README.md"))
# Generate run_consumer_kaggle.ipynb (Jupyter Notebook format)
notebook_content = {
"cells": [
{
"cell_type": "code",
"execution_count": None,
"metadata": {},
"outputs": [],
"source": [
"# 1. GPU & CUDA Check\n",
"import torch\n",
"import sys\n",
"import os\n",
"import subprocess\n",
"import shutil\n",
"\n",
"print(\"=== Kaggle GPU & System Check ===\")\n",
"print(f\"Python Version: {sys.version}\")\n",
"print(f\"PyTorch Version: {torch.__version__}\")\n",
"print(f\"CUDA Available: {torch.cuda.is_available()}\")\n",
"if torch.cuda.is_available():\n",
" print(f\"GPU Device: {torch.cuda.get_device_name(0)}\")\n",
"else:\n",
" print(\"WARNING: GPU is not active. Please enable GPU in the Kaggle settings for optimal performance.\")\n",
"\n",
"# 2. Load Kaggle User Secrets\n",
"print(\"\\n=== Kaggle Secrets Validation ===\")\n",
"gh_token = None\n",
"try:\n",
" from kaggle_secrets import UserSecretsClient\n",
" user_secrets = UserSecretsClient()\n",
" db_url = user_secrets.get_secret(\"DATABASE_URL\")\n",
" rabbitmq_url = user_secrets.get_secret(\"RABBITMQ_URL\")\n",
" gh_token = user_secrets.get_secret(\"GH_TOKEN\")\n",
" print(f\"DATABASE_URL secret: {'FOUND' if db_url else 'MISSING'}\")\n",
" print(f\"RABBITMQ_URL secret: {'FOUND' if rabbitmq_url else 'MISSING'}\")\n",
" print(f\"GH_TOKEN secret: {'FOUND' if gh_token else 'MISSING'}\")\n",
"except Exception as e:\n",
" print(f\"Warning: Could not check secrets: {e}\")\n",
"\n",
"# 3. Clone Repository (Since Kaggle CLI only uploads the notebook itself)\n",
"print(\"\\n=== Cloning Repository ===\")\n",
"repo_dir = \"/kaggle/working/ETL-Ingestion\"\n",
"if os.path.exists(repo_dir):\n",
" print(f\"Removing existing repository directory at {repo_dir}...\")\n",
" shutil.rmtree(repo_dir)\n",
"\n",
"if gh_token:\n",
" print(\"Cloning private repository using GH_TOKEN...\")\n",
" clone_url = f\"https://x-access-token:{gh_token}@github.com/FP-KKA/ETL-Ingestion.git\"\n",
"else:\n",
" print(\"WARNING: GH_TOKEN secret not found. Attempting public clone...\")\n",
" clone_url = \"https://github.com/FP-KKA/ETL-Ingestion.git\"\n",
"\n",
"try:\n",
" subprocess.check_call([\"git\", \"clone\", clone_url, repo_dir])\n",
" print(\"Repository cloned successfully.\")\n",
"except Exception as e:\n",
" print(f\"Error cloning repository: {e}\")\n",
" sys.exit(1)\n",
"\n",
"# 4. Change Directory & Install Dependencies\n",
"os.chdir(repo_dir)\n",
"print(f\"Changed working directory to: {os.getcwd()}\")\n",
"\n",
"print(\"\\n=== Installing Clean Architecture Package ===\")\n",
"try:\n",
" subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", \"-e\", \".[all]\"])\n",
" print(\"Package & dependencies installed successfully.\")\n",
"except Exception as e:\n",
" print(f\"Error installing package dependencies: {e}\")\n",
" sys.exit(1)\n",
"\n",
"# 5. Launch Consumer Daemon\n",
"print(\"\\n=== Launching Consumer Daemon ===\")\n",
"try:\n",
" subprocess.check_call([sys.executable, \"-m\", \"src.interface.consumer.run_consumer\"])\n",
"except KeyboardInterrupt:\n",
" print(\"Consumer stopped.\")\n",
"except Exception as e:\n",
" print(f\"Consumer failed to execute: {e}\")\n",
" sys.exit(1)"
]
}
],
"metadata": {
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
with open(os.path.join(deploy_dir, "run_consumer_kaggle.ipynb"), "w", encoding="utf-8") as f:
json.dump(notebook_content, f, indent=4)
# Generate kernel-metadata.json
username = os.environ.get("KAGGLE_USERNAME", "benedictusryugunawan")
slug = os.environ.get("KAGGLE_SLUG", "libre")
metadata = {
"id": f"{username}/{slug}",
"title": slug,
"code_file": "run_consumer_kaggle.ipynb",
"language": "python",
"kernel_type": "notebook",
"is_private": "true",
"enable_gpu": "true",
"enable_tpu": "false",
"enable_internet": "true",
"dataset_sources": [],
"competition_sources": [],
"kernel_sources": []
}
with open(os.path.join(deploy_dir, "kernel-metadata.json"), "w", encoding="utf-8") as f:
json.dump(metadata, f, indent=4)
print(f"Kaggle deployment bundle successfully created in: {deploy_dir}")
if __name__ == "__main__":
build_deploy_dir()
|