{ "cells": [ { "cell_type": "markdown", "id": "bc13dda0", "metadata": {}, "source": [ "# 07 - Deployment Artifacts\n", "\n", "## CyberForge AI - Final Deployment Package\n", "\n", "This notebook creates final deployment artifacts for:\n", "- Hugging Face Hub upload\n", "- Production deployment\n", "- Model versioning and documentation\n", "\n", "### Outputs:\n", "- Complete model package for HF Hub\n", "- Docker configuration for ML services\n", "- Environment configuration\n", "- Deployment documentation" ] }, { "cell_type": "code", "execution_count": null, "id": "9263214e", "metadata": {}, "outputs": [], "source": [ "import json\n", "import os\n", "import time\n", "import shutil\n", "from pathlib import Path\n", "from typing import Dict, List, Any\n", "import warnings\n", "warnings.filterwarnings('ignore')\n", "\n", "# Configuration\n", "config_path = Path(\"notebook_config.json\")\n", "if not config_path.exists():\n", " config_path = Path(\"/home/user/app/notebooks/notebook_config.json\")\n", "with open(config_path) as f:\n", " CONFIG = json.load(f)\n", "\n", "MODELS_DIR = Path(CONFIG[\"datasets_dir\"]).parent / \"models\"\n", "BACKEND_DIR = MODELS_DIR.parent / \"backend_package\"\n", "DEPLOY_DIR = MODELS_DIR.parent / \"deployment\"\n", "DEPLOY_DIR.mkdir(exist_ok=True)\n", "\n", "print(f\"✓ Configuration loaded\")\n", "print(f\"✓ Deployment output: {DEPLOY_DIR}\")" ] }, { "cell_type": "markdown", "id": "ea513b54", "metadata": {}, "source": [ "## 1. Hugging Face Hub Upload Preparation" ] }, { "cell_type": "code", "execution_count": null, "id": "0409ab79", "metadata": {}, "outputs": [], "source": [ "try:\n", " from huggingface_hub import HfApi, login\n", " HF_AVAILABLE = True\n", "except ImportError:\n", " import subprocess\n", " subprocess.run(['pip', 'install', 'huggingface_hub', '-q'])\n", " from huggingface_hub import HfApi, login\n", " HF_AVAILABLE = True\n", "\n", "class HuggingFaceDeployer:\n", " \"\"\"\n", " Deploy models and artifacts to Hugging Face Hub.\n", " \"\"\"\n", " \n", " def __init__(self, token: str = None):\n", " self.token = token or CONFIG.get('hf_token') or os.environ.get('HF_TOKEN')\n", " self.api = HfApi(token=self.token) if self.token else None\n", " self.repo_id = CONFIG.get('hf_repo', 'Che237/cyberforge-models')\n", " \n", " def is_authenticated(self) -> bool:\n", " \"\"\"Check if authenticated to Hugging Face\"\"\"\n", " if not self.api:\n", " return False\n", " try:\n", " self.api.whoami()\n", " return True\n", " except:\n", " return False\n", " \n", " def create_model_card(self, models_info: Dict) -> str:\n", " \"\"\"Create MODEL_CARD.md for Hugging Face\"\"\"\n", " card = f\"\"\"\n", "---\n", "license: mit\n", "tags:\n", " - cybersecurity\n", " - threat-detection\n", " - phishing\n", " - malware\n", " - security\n", "language:\n", " - en\n", "---\n", "\n", "# CyberForge AI Models\n", "\n", "Production-ready machine learning models for cybersecurity threat detection.\n", "\n", "## Models Included\n", "\n", "| Model | Task | Accuracy | F1 Score | Inference Time |\n", "|-------|------|----------|----------|----------------|\n", "\"\"\"\n", " \n", " for name, info in models_info.items():\n", " card += f\"| {name} | {info.get('type', 'classification')} | {info.get('accuracy', 0):.4f} | {info.get('f1_score', 0):.4f} | {info.get('inference_time_ms', 0):.2f}ms |\\n\"\n", " \n", " card += f\"\"\"\n", "\n", "## Usage\n", "\n", "### Python\n", "\n", "```python\n", "from inference import CyberForgeInference\n", "\n", "inference = CyberForgeInference('./models')\n", "result = inference.predict('phishing_detection', features)\n", "```\n", "\n", "### API\n", "\n", "```python\n", "import requests\n", "\n", "response = requests.post(\n", " 'https://huggingface.co/spaces/Che237/cyberforge/predict',\n", " json={{'model_name': 'phishing_detection', 'features': features}}\n", ")\n", "```\n", "\n", "## Model Details\n", "\n", "- **Framework**: scikit-learn\n", "- **Python Version**: 3.11+\n", "- **License**: MIT\n", "\n", "## Citation\n", "\n", "If you use these models, please cite:\n", "\n", "```bibtex\n", "@software{{cyberforge2024,\n", " title = {{CyberForge AI Security Models}},\n", " year = {{2024}},\n", " url = {{https://huggingface.co/Che237/cyberforge-models}}\n", "}}\n", "```\n", "\"\"\"\n", " return card\n", " \n", " def prepare_upload_package(self, source_dir: Path, output_dir: Path) -> Path:\n", " \"\"\"Prepare package for HF upload\"\"\"\n", " hf_dir = output_dir / \"huggingface_upload\"\n", " hf_dir.mkdir(exist_ok=True)\n", " \n", " # Copy all files from backend package\n", " if source_dir.exists():\n", " for item in source_dir.iterdir():\n", " if item.is_file():\n", " shutil.copy(item, hf_dir / item.name)\n", " elif item.is_dir():\n", " shutil.copytree(item, hf_dir / item.name, dirs_exist_ok=True)\n", " \n", " return hf_dir\n", "\n", "hf_deployer = HuggingFaceDeployer()\n", "print(f\"✓ HuggingFace Deployer initialized\")\n", "print(f\" Authenticated: {hf_deployer.is_authenticated()}\")" ] }, { "cell_type": "markdown", "id": "2ca3a2e2", "metadata": {}, "source": [ "## 2. Create Deployment Package" ] }, { "cell_type": "code", "execution_count": null, "id": "057ab810", "metadata": {}, "outputs": [], "source": [ "# Create HF upload package\n", "print(\"Creating deployment package...\\n\")\n", "\n", "hf_package_dir = hf_deployer.prepare_upload_package(BACKEND_DIR, DEPLOY_DIR)\n", "print(f\"✓ Package prepared at: {hf_package_dir}\")\n", "\n", "# Load models info\n", "manifest_path = BACKEND_DIR / \"manifest.json\"\n", "if manifest_path.exists():\n", " with open(manifest_path) as f:\n", " manifest = json.load(f)\n", " models_info = manifest.get('models', {})\n", "else:\n", " models_info = {}\n", "\n", "# Create model card\n", "model_card = hf_deployer.create_model_card(models_info)\n", "model_card_path = hf_package_dir / \"README.md\"\n", "with open(model_card_path, 'w') as f:\n", " f.write(model_card)\n", "\n", "print(f\"✓ Model card saved\")" ] }, { "cell_type": "markdown", "id": "5168e806", "metadata": {}, "source": [ "## 3. Docker Configuration" ] }, { "cell_type": "code", "execution_count": null, "id": "c756998d", "metadata": {}, "outputs": [], "source": [ "# Generate Dockerfile for ML services\n", "dockerfile_content = '''\n", "# CyberForge ML Services Dockerfile\n", "FROM python:3.11-slim\n", "\n", "WORKDIR /app\n", "\n", "# Install dependencies\n", "COPY requirements.txt .\n", "RUN pip install --no-cache-dir -r requirements.txt\n", "\n", "# Copy models and code\n", "COPY . .\n", "\n", "# Expose port\n", "EXPOSE 8001\n", "\n", "# Health check\n", "HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \\\n", " CMD curl -f http://localhost:8001/health || exit 1\n", "\n", "# Run server\n", "CMD [\"uvicorn\", \"inference:app\", \"--host\", \"0.0.0.0\", \"--port\", \"8001\"]\n", "'''\n", "\n", "dockerfile_path = DEPLOY_DIR / \"Dockerfile\"\n", "with open(dockerfile_path, 'w') as f:\n", " f.write(dockerfile_content)\n", "\n", "print(f\"✓ Dockerfile saved to: {dockerfile_path}\")" ] }, { "cell_type": "code", "execution_count": null, "id": "afea07f6", "metadata": {}, "outputs": [], "source": [ "# Generate requirements.txt for deployment\n", "requirements_content = '''\n", "# CyberForge ML Requirements\n", "fastapi>=0.104.0\n", "uvicorn>=0.24.0\n", "pydantic>=2.0.0\n", "numpy>=1.24.0\n", "pandas>=2.0.0\n", "scikit-learn>=1.3.0\n", "joblib>=1.3.0\n", "python-multipart>=0.0.6\n", "huggingface-hub>=0.19.0\n", "google-genai>=1.0.0\n", "requests>=2.31.0\n", "pyarrow>=14.0.0\n", "'''\n", "\n", "requirements_path = DEPLOY_DIR / \"requirements.txt\"\n", "with open(requirements_path, 'w') as f:\n", " f.write(requirements_content)\n", "\n", "print(f\"✓ requirements.txt saved\")\n" ] }, { "cell_type": "code", "execution_count": null, "id": "6586c940", "metadata": {}, "outputs": [], "source": [ "# Generate docker-compose.yml\n", "docker_compose_content = '''\n", "version: \"3.8\"\n", "\n", "services:\n", " ml-services:\n", " build:\n", " context: .\n", " dockerfile: Dockerfile\n", " ports:\n", " - \"8001:8001\"\n", " environment:\n", " - GEMINI_API_KEY=${GEMINI_API_KEY}\n", " - HF_TOKEN=${HF_TOKEN}\n", " volumes:\n", " - ./models:/app/models:ro\n", " restart: unless-stopped\n", " healthcheck:\n", " test: [\"CMD\", \"curl\", \"-f\", \"http://localhost:8001/health\"]\n", " interval: 30s\n", " timeout: 10s\n", " retries: 3\n", "'''\n", "\n", "compose_path = DEPLOY_DIR / \"docker-compose.yml\"\n", "with open(compose_path, 'w') as f:\n", " f.write(docker_compose_content)\n", "\n", "print(f\"✓ docker-compose.yml saved\")" ] }, { "cell_type": "markdown", "id": "665ec48b", "metadata": {}, "source": [ "## 4. Environment Configuration" ] }, { "cell_type": "code", "execution_count": null, "id": "61531f5c", "metadata": {}, "outputs": [], "source": [ "# Generate .env.example\n", "env_example = '''\n", "# CyberForge ML Services Environment Configuration\n", "\n", "# Hugging Face\n", "HF_TOKEN=your_huggingface_token_here\n", "HF_REPO=Che237/cyberforge-models\n", "\n", "# Gemini API\n", "GEMINI_API_KEY=your_gemini_api_key_here\n", "\n", "# WebScraper API\n", "WEBSCRAPER_API_KEY=your_webscraper_api_key_here\n", "WEBSCRAPER_API_URL=http://webscrapper.live/api/scrape\n", "\n", "# Server Configuration\n", "ML_SERVICE_PORT=8001\n", "ML_SERVICE_HOST=0.0.0.0\n", "\n", "# Model Configuration\n", "MODELS_DIR=./models\n", "MAX_INFERENCE_TIME_MS=100\n", "CONFIDENCE_THRESHOLD=0.7\n", "'''\n", "\n", "env_path = DEPLOY_DIR / \".env.example\"\n", "with open(env_path, 'w') as f:\n", " f.write(env_example)\n", "\n", "print(f\"✓ .env.example saved\")" ] }, { "cell_type": "markdown", "id": "86c547c3", "metadata": {}, "source": [ "## 5. Upload Script" ] }, { "cell_type": "code", "execution_count": null, "id": "21703867", "metadata": {}, "outputs": [], "source": [ "# Generate HuggingFace upload script\n", "upload_script = '''\n", "#!/usr/bin/env python3\n", "\"\"\"\n", "CyberForge - Upload to Hugging Face Hub\n", "\"\"\"\n", "\n", "import os\n", "import sys\n", "from pathlib import Path\n", "from huggingface_hub import HfApi, login\n", "\n", "def main():\n", " # Get token\n", " token = os.environ.get('HF_TOKEN')\n", " if not token:\n", " print(\"Error: HF_TOKEN environment variable not set\")\n", " sys.exit(1)\n", " \n", " # Login\n", " login(token=token)\n", " api = HfApi()\n", " \n", " # Configuration\n", " repo_id = os.environ.get('HF_REPO', 'Che237/cyberforge-models')\n", " upload_dir = Path('./huggingface_upload')\n", " \n", " if not upload_dir.exists():\n", " print(f\"Error: Upload directory not found: {upload_dir}\")\n", " sys.exit(1)\n", " \n", " print(f\"Uploading to: {repo_id}\")\n", " \n", " # Upload\n", " try:\n", " api.upload_folder(\n", " folder_path=str(upload_dir),\n", " repo_id=repo_id,\n", " repo_type=\"model\",\n", " commit_message=\"Update CyberForge ML models\"\n", " )\n", " print(f\"✓ Upload complete: https://huggingface.co/{repo_id}\")\n", " except Exception as e:\n", " print(f\"Error: {e}\")\n", " sys.exit(1)\n", "\n", "if __name__ == \"__main__\":\n", " main()\n", "'''\n", "\n", "upload_script_path = DEPLOY_DIR / \"upload_to_hf.py\"\n", "with open(upload_script_path, 'w') as f:\n", " f.write(upload_script)\n", "\n", "os.chmod(upload_script_path, 0o755)\n", "print(f\"✓ Upload script saved\")" ] }, { "cell_type": "markdown", "id": "289852a7", "metadata": {}, "source": [ "## 6. Deployment Documentation" ] }, { "cell_type": "code", "execution_count": null, "id": "4b823e84", "metadata": {}, "outputs": [], "source": [ "# Generate deployment guide\n", "deployment_guide = f'''\n", "# CyberForge ML Deployment Guide\n", "\n", "## Quick Start\n", "\n", "### 1. Local Deployment\n", "\n", "```bash\n", "# Install dependencies\n", "pip install -r requirements.txt\n", "\n", "# Start server\n", "uvicorn inference:app --host 0.0.0.0 --port 8001\n", "```\n", "\n", "### 2. Docker Deployment\n", "\n", "```bash\n", "# Build and run\n", "docker-compose up -d\n", "\n", "# Check logs\n", "docker-compose logs -f\n", "```\n", "\n", "### 3. Hugging Face Deployment\n", "\n", "```bash\n", "# Set token\n", "export HF_TOKEN=your_token_here\n", "\n", "# Upload\n", "python upload_to_hf.py\n", "```\n", "\n", "## API Endpoints\n", "\n", "### Prediction\n", "\n", "```\n", "POST /predict\n", "Content-Type: application/json\n", "\n", "{{\n", " \"model_name\": \"phishing_detection\",\n", " \"features\": {{\n", " \"url_length\": 50,\n", " \"has_https\": true,\n", " ...\n", " }}\n", "}}\n", "```\n", "\n", "### List Models\n", "\n", "```\n", "GET /models\n", "```\n", "\n", "### Health Check\n", "\n", "```\n", "GET /health\n", "```\n", "\n", "## Environment Variables\n", "\n", "| Variable | Description | Required |\n", "|----------|-------------|----------|\n", "| HF_TOKEN | Hugging Face API token | For HF upload |\n", "| GEMINI_API_KEY | Gemini API key | For AI reasoning |\n", "| WEBSCRAPER_API_KEY | WebScraper API key | For data collection |\n", "\n", "## Monitoring\n", "\n", "- Health endpoint: `/health`\n", "- Metrics endpoint: `/metrics`\n", "- Logs: `docker-compose logs -f ml-services`\n", "\n", "## Troubleshooting\n", "\n", "### Model not found\n", "- Check model files exist in `models/` directory\n", "- Verify manifest.json includes the model\n", "\n", "### Slow inference\n", "- Check model size\n", "- Consider using lighter model variant\n", "- Verify no resource contention\n", "\n", "### API errors\n", "- Check logs for stack traces\n", "- Verify input format matches expected schema\n", "- Ensure all dependencies installed\n", "\n", "## Support\n", "\n", "For issues and feature requests, please open an issue on GitHub.\n", "'''\n", "\n", "guide_path = DEPLOY_DIR / \"DEPLOYMENT.md\"\n", "with open(guide_path, 'w') as f:\n", " f.write(deployment_guide)\n", "\n", "print(f\"✓ Deployment guide saved\")" ] }, { "cell_type": "markdown", "id": "54cad563", "metadata": {}, "source": [ "## 7. Final Package Verification" ] }, { "cell_type": "code", "execution_count": null, "id": "502eb389", "metadata": {}, "outputs": [], "source": [ "# Verify deployment package\n", "required_files = [\n", " 'Dockerfile',\n", " 'docker-compose.yml',\n", " 'requirements.txt',\n", " '.env.example',\n", " 'upload_to_hf.py',\n", " 'DEPLOYMENT.md',\n", " 'huggingface_upload/README.md'\n", "]\n", "\n", "print(\"Verifying deployment package...\\n\")\n", "\n", "all_present = True\n", "for file in required_files:\n", " path = DEPLOY_DIR / file\n", " exists = path.exists()\n", " status = \"✓\" if exists else \"✗\"\n", " print(f\" {status} {file}\")\n", " if not exists:\n", " all_present = False\n", "\n", "print(f\"\\n{'✓ All files present' if all_present else '✗ Some files missing'}\")" ] }, { "cell_type": "markdown", "id": "4eea48b3", "metadata": {}, "source": [ "## 8. Summary" ] }, { "cell_type": "code", "execution_count": null, "id": "0371e31b", "metadata": {}, "outputs": [], "source": [ "# Calculate package stats\n", "total_files = len(list(DEPLOY_DIR.rglob('*')))\n", "total_size = sum(f.stat().st_size for f in DEPLOY_DIR.rglob('*') if f.is_file())\n", "\n", "print(\"\\n\" + \"=\" * 60)\n", "print(\"DEPLOYMENT ARTIFACTS COMPLETE\")\n", "print(\"=\" * 60)\n", "\n", "print(f\"\"\"\n", "🚀 Deployment Package Ready:\n", " - Location: {DEPLOY_DIR}\n", " - Files: {total_files}\n", " - Total size: {total_size / (1024*1024):.2f} MB\n", "\n", "📦 Package Contents:\n", " - Dockerfile: Container configuration\n", " - docker-compose.yml: Multi-service orchestration\n", " - requirements.txt: Python dependencies\n", " - .env.example: Environment template\n", " - upload_to_hf.py: HuggingFace upload script\n", " - DEPLOYMENT.md: Deployment guide\n", " - huggingface_upload/: HF Hub package\n", "\n", "🔧 Deployment Options:\n", " 1. Local: uvicorn inference:app --port 8001\n", " 2. Docker: docker-compose up -d\n", " 3. HuggingFace: python upload_to_hf.py\n", "\n", "📋 Next Steps:\n", " 1. Set environment variables (copy .env.example to .env)\n", " 2. Choose deployment method\n", " 3. Verify health endpoint\n", " 4. Integrate with backend services\n", "\n", "🎉 CyberForge ML Pipeline Complete!\n", "\"\"\")\n", "print(\"=\" * 60)" ] }, { "cell_type": "markdown", "id": "2ee1fdf8", "metadata": {}, "source": [ "## 9. Optional: Upload to Hugging Face" ] }, { "cell_type": "code", "execution_count": null, "id": "7ce9155a", "metadata": {}, "outputs": [], "source": [ "# Upload trained models and artifacts to Hugging Face Hub\n", "if hf_deployer.is_authenticated():\n", " print(\"Uploading to Hugging Face Hub...\")\n", " try:\n", " hf_deployer.api.upload_folder(\n", " folder_path=str(hf_package_dir),\n", " repo_id=hf_deployer.repo_id,\n", " repo_type=\"model\",\n", " commit_message=\"Update CyberForge ML models and deployment artifacts\"\n", " )\n", " print(f\"✓ Uploaded to: https://huggingface.co/{hf_deployer.repo_id}\")\n", " except Exception as e:\n", " print(f\"⚠ Upload error: {e}\")\n", "else:\n", " print(\"⚠ Not authenticated to HuggingFace. Models saved locally only.\")\n", " print(\" Set HF_TOKEN environment variable or hf_token in notebook_config.json to upload.\")" ] } ], "metadata": { "language_info": { "name": "python" } }, "nbformat": 4, "nbformat_minor": 5 }