Che237 commited on
Commit
aef1511
Β·
verified Β·
1 Parent(s): 52f9d9a

Add 07_deployment_artifacts.ipynb

Browse files
notebooks/07_deployment_artifacts.ipynb ADDED
@@ -0,0 +1,724 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "cells": [
3
+ {
4
+ "cell_type": "markdown",
5
+ "id": "bc13dda0",
6
+ "metadata": {},
7
+ "source": [
8
+ "# 07 - Deployment Artifacts\n",
9
+ "\n",
10
+ "## CyberForge AI - Final Deployment Package\n",
11
+ "\n",
12
+ "This notebook creates final deployment artifacts for:\n",
13
+ "- Hugging Face Hub upload\n",
14
+ "- Production deployment\n",
15
+ "- Model versioning and documentation\n",
16
+ "\n",
17
+ "### Outputs:\n",
18
+ "- Complete model package for HF Hub\n",
19
+ "- Docker configuration for ML services\n",
20
+ "- Environment configuration\n",
21
+ "- Deployment documentation"
22
+ ]
23
+ },
24
+ {
25
+ "cell_type": "code",
26
+ "execution_count": null,
27
+ "id": "9263214e",
28
+ "metadata": {},
29
+ "outputs": [],
30
+ "source": [
31
+ "import json\n",
32
+ "import os\n",
33
+ "import time\n",
34
+ "import shutil\n",
35
+ "from pathlib import Path\n",
36
+ "from typing import Dict, List, Any\n",
37
+ "import warnings\n",
38
+ "warnings.filterwarnings('ignore')\n",
39
+ "\n",
40
+ "# Configuration\n",
41
+ "config_path = Path(\"../notebook_config.json\")\n",
42
+ "with open(config_path) as f:\n",
43
+ " CONFIG = json.load(f)\n",
44
+ "\n",
45
+ "MODELS_DIR = Path(CONFIG[\"datasets_dir\"]).parent / \"models\"\n",
46
+ "BACKEND_DIR = MODELS_DIR.parent / \"backend_package\"\n",
47
+ "DEPLOY_DIR = MODELS_DIR.parent / \"deployment\"\n",
48
+ "DEPLOY_DIR.mkdir(exist_ok=True)\n",
49
+ "\n",
50
+ "print(f\"βœ“ Configuration loaded\")\n",
51
+ "print(f\"βœ“ Deployment output: {DEPLOY_DIR}\")"
52
+ ]
53
+ },
54
+ {
55
+ "cell_type": "markdown",
56
+ "id": "ea513b54",
57
+ "metadata": {},
58
+ "source": [
59
+ "## 1. Hugging Face Hub Upload Preparation"
60
+ ]
61
+ },
62
+ {
63
+ "cell_type": "code",
64
+ "execution_count": null,
65
+ "id": "0409ab79",
66
+ "metadata": {},
67
+ "outputs": [],
68
+ "source": [
69
+ "try:\n",
70
+ " from huggingface_hub import HfApi, login\n",
71
+ " HF_AVAILABLE = True\n",
72
+ "except ImportError:\n",
73
+ " import subprocess\n",
74
+ " subprocess.run(['pip', 'install', 'huggingface_hub', '-q'])\n",
75
+ " from huggingface_hub import HfApi, login\n",
76
+ " HF_AVAILABLE = True\n",
77
+ "\n",
78
+ "class HuggingFaceDeployer:\n",
79
+ " \"\"\"\n",
80
+ " Deploy models and artifacts to Hugging Face Hub.\n",
81
+ " \"\"\"\n",
82
+ " \n",
83
+ " def __init__(self, token: str = None):\n",
84
+ " self.token = token or CONFIG.get('hf_token') or os.environ.get('HF_TOKEN')\n",
85
+ " self.api = HfApi(token=self.token) if self.token else None\n",
86
+ " self.repo_id = CONFIG.get('hf_repo', 'Che237/cyberforge-models')\n",
87
+ " \n",
88
+ " def is_authenticated(self) -> bool:\n",
89
+ " \"\"\"Check if authenticated to Hugging Face\"\"\"\n",
90
+ " if not self.api:\n",
91
+ " return False\n",
92
+ " try:\n",
93
+ " self.api.whoami()\n",
94
+ " return True\n",
95
+ " except:\n",
96
+ " return False\n",
97
+ " \n",
98
+ " def create_model_card(self, models_info: Dict) -> str:\n",
99
+ " \"\"\"Create MODEL_CARD.md for Hugging Face\"\"\"\n",
100
+ " card = f\"\"\"\n",
101
+ "---\n",
102
+ "license: mit\n",
103
+ "tags:\n",
104
+ " - cybersecurity\n",
105
+ " - threat-detection\n",
106
+ " - phishing\n",
107
+ " - malware\n",
108
+ " - security\n",
109
+ "language:\n",
110
+ " - en\n",
111
+ "---\n",
112
+ "\n",
113
+ "# CyberForge AI Models\n",
114
+ "\n",
115
+ "Production-ready machine learning models for cybersecurity threat detection.\n",
116
+ "\n",
117
+ "## Models Included\n",
118
+ "\n",
119
+ "| Model | Task | Accuracy | F1 Score | Inference Time |\n",
120
+ "|-------|------|----------|----------|----------------|\n",
121
+ "\"\"\"\n",
122
+ " \n",
123
+ " for name, info in models_info.items():\n",
124
+ " card += f\"| {name} | {info.get('type', 'classification')} | {info.get('accuracy', 0):.4f} | {info.get('f1_score', 0):.4f} | {info.get('inference_time_ms', 0):.2f}ms |\\n\"\n",
125
+ " \n",
126
+ " card += f\"\"\"\n",
127
+ "\n",
128
+ "## Usage\n",
129
+ "\n",
130
+ "### Python\n",
131
+ "\n",
132
+ "```python\n",
133
+ "from inference import CyberForgeInference\n",
134
+ "\n",
135
+ "inference = CyberForgeInference('./models')\n",
136
+ "result = inference.predict('phishing_detection', features)\n",
137
+ "```\n",
138
+ "\n",
139
+ "### API\n",
140
+ "\n",
141
+ "```python\n",
142
+ "import requests\n",
143
+ "\n",
144
+ "response = requests.post(\n",
145
+ " 'https://huggingface.co/spaces/Che237/cyberforge/predict',\n",
146
+ " json={{'model_name': 'phishing_detection', 'features': features}}\n",
147
+ ")\n",
148
+ "```\n",
149
+ "\n",
150
+ "## Model Details\n",
151
+ "\n",
152
+ "- **Framework**: scikit-learn\n",
153
+ "- **Python Version**: 3.11+\n",
154
+ "- **License**: MIT\n",
155
+ "\n",
156
+ "## Citation\n",
157
+ "\n",
158
+ "If you use these models, please cite:\n",
159
+ "\n",
160
+ "```bibtex\n",
161
+ "@software{{cyberforge2024,\n",
162
+ " title = {{CyberForge AI Security Models}},\n",
163
+ " year = {{2024}},\n",
164
+ " url = {{https://huggingface.co/Che237/cyberforge-models}}\n",
165
+ "}}\n",
166
+ "```\n",
167
+ "\"\"\"\n",
168
+ " return card\n",
169
+ " \n",
170
+ " def prepare_upload_package(self, source_dir: Path, output_dir: Path) -> Path:\n",
171
+ " \"\"\"Prepare package for HF upload\"\"\"\n",
172
+ " hf_dir = output_dir / \"huggingface_upload\"\n",
173
+ " hf_dir.mkdir(exist_ok=True)\n",
174
+ " \n",
175
+ " # Copy all files from backend package\n",
176
+ " if source_dir.exists():\n",
177
+ " for item in source_dir.iterdir():\n",
178
+ " if item.is_file():\n",
179
+ " shutil.copy(item, hf_dir / item.name)\n",
180
+ " elif item.is_dir():\n",
181
+ " shutil.copytree(item, hf_dir / item.name, dirs_exist_ok=True)\n",
182
+ " \n",
183
+ " return hf_dir\n",
184
+ "\n",
185
+ "hf_deployer = HuggingFaceDeployer()\n",
186
+ "print(f\"βœ“ HuggingFace Deployer initialized\")\n",
187
+ "print(f\" Authenticated: {hf_deployer.is_authenticated()}\")"
188
+ ]
189
+ },
190
+ {
191
+ "cell_type": "markdown",
192
+ "id": "2ca3a2e2",
193
+ "metadata": {},
194
+ "source": [
195
+ "## 2. Create Deployment Package"
196
+ ]
197
+ },
198
+ {
199
+ "cell_type": "code",
200
+ "execution_count": null,
201
+ "id": "057ab810",
202
+ "metadata": {},
203
+ "outputs": [],
204
+ "source": [
205
+ "# Create HF upload package\n",
206
+ "print(\"Creating deployment package...\\n\")\n",
207
+ "\n",
208
+ "hf_package_dir = hf_deployer.prepare_upload_package(BACKEND_DIR, DEPLOY_DIR)\n",
209
+ "print(f\"βœ“ Package prepared at: {hf_package_dir}\")\n",
210
+ "\n",
211
+ "# Load models info\n",
212
+ "manifest_path = BACKEND_DIR / \"manifest.json\"\n",
213
+ "if manifest_path.exists():\n",
214
+ " with open(manifest_path) as f:\n",
215
+ " manifest = json.load(f)\n",
216
+ " models_info = manifest.get('models', {})\n",
217
+ "else:\n",
218
+ " models_info = {}\n",
219
+ "\n",
220
+ "# Create model card\n",
221
+ "model_card = hf_deployer.create_model_card(models_info)\n",
222
+ "model_card_path = hf_package_dir / \"README.md\"\n",
223
+ "with open(model_card_path, 'w') as f:\n",
224
+ " f.write(model_card)\n",
225
+ "\n",
226
+ "print(f\"βœ“ Model card saved\")"
227
+ ]
228
+ },
229
+ {
230
+ "cell_type": "markdown",
231
+ "id": "5168e806",
232
+ "metadata": {},
233
+ "source": [
234
+ "## 3. Docker Configuration"
235
+ ]
236
+ },
237
+ {
238
+ "cell_type": "code",
239
+ "execution_count": null,
240
+ "id": "c756998d",
241
+ "metadata": {},
242
+ "outputs": [],
243
+ "source": [
244
+ "# Generate Dockerfile for ML services\n",
245
+ "dockerfile_content = '''\n",
246
+ "# CyberForge ML Services Dockerfile\n",
247
+ "FROM python:3.11-slim\n",
248
+ "\n",
249
+ "WORKDIR /app\n",
250
+ "\n",
251
+ "# Install dependencies\n",
252
+ "COPY requirements.txt .\n",
253
+ "RUN pip install --no-cache-dir -r requirements.txt\n",
254
+ "\n",
255
+ "# Copy models and code\n",
256
+ "COPY . .\n",
257
+ "\n",
258
+ "# Expose port\n",
259
+ "EXPOSE 8001\n",
260
+ "\n",
261
+ "# Health check\n",
262
+ "HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \\\n",
263
+ " CMD curl -f http://localhost:8001/health || exit 1\n",
264
+ "\n",
265
+ "# Run server\n",
266
+ "CMD [\"uvicorn\", \"inference:app\", \"--host\", \"0.0.0.0\", \"--port\", \"8001\"]\n",
267
+ "'''\n",
268
+ "\n",
269
+ "dockerfile_path = DEPLOY_DIR / \"Dockerfile\"\n",
270
+ "with open(dockerfile_path, 'w') as f:\n",
271
+ " f.write(dockerfile_content)\n",
272
+ "\n",
273
+ "print(f\"βœ“ Dockerfile saved to: {dockerfile_path}\")"
274
+ ]
275
+ },
276
+ {
277
+ "cell_type": "code",
278
+ "execution_count": null,
279
+ "id": "afea07f6",
280
+ "metadata": {},
281
+ "outputs": [],
282
+ "source": [
283
+ "# Generate requirements.txt for deployment\n",
284
+ "requirements_content = '''\n",
285
+ "# CyberForge ML Requirements\n",
286
+ "fastapi>=0.104.0\n",
287
+ "uvicorn>=0.24.0\n",
288
+ "pydantic>=2.0.0\n",
289
+ "numpy>=1.24.0\n",
290
+ "pandas>=2.0.0\n",
291
+ "scikit-learn>=1.3.0\n",
292
+ "joblib>=1.3.0\n",
293
+ "python-multipart>=0.0.6\n",
294
+ "huggingface-hub>=0.19.0\n",
295
+ "google-generativeai>=0.3.0\n",
296
+ "requests>=2.31.0\n",
297
+ "'''\n",
298
+ "\n",
299
+ "requirements_path = DEPLOY_DIR / \"requirements.txt\"\n",
300
+ "with open(requirements_path, 'w') as f:\n",
301
+ " f.write(requirements_content)\n",
302
+ "\n",
303
+ "print(f\"βœ“ requirements.txt saved\")"
304
+ ]
305
+ },
306
+ {
307
+ "cell_type": "code",
308
+ "execution_count": null,
309
+ "id": "6586c940",
310
+ "metadata": {},
311
+ "outputs": [],
312
+ "source": [
313
+ "# Generate docker-compose.yml\n",
314
+ "docker_compose_content = '''\n",
315
+ "version: \"3.8\"\n",
316
+ "\n",
317
+ "services:\n",
318
+ " ml-services:\n",
319
+ " build:\n",
320
+ " context: .\n",
321
+ " dockerfile: Dockerfile\n",
322
+ " ports:\n",
323
+ " - \"8001:8001\"\n",
324
+ " environment:\n",
325
+ " - GEMINI_API_KEY=${GEMINI_API_KEY}\n",
326
+ " - HF_TOKEN=${HF_TOKEN}\n",
327
+ " volumes:\n",
328
+ " - ./models:/app/models:ro\n",
329
+ " restart: unless-stopped\n",
330
+ " healthcheck:\n",
331
+ " test: [\"CMD\", \"curl\", \"-f\", \"http://localhost:8001/health\"]\n",
332
+ " interval: 30s\n",
333
+ " timeout: 10s\n",
334
+ " retries: 3\n",
335
+ "'''\n",
336
+ "\n",
337
+ "compose_path = DEPLOY_DIR / \"docker-compose.yml\"\n",
338
+ "with open(compose_path, 'w') as f:\n",
339
+ " f.write(docker_compose_content)\n",
340
+ "\n",
341
+ "print(f\"βœ“ docker-compose.yml saved\")"
342
+ ]
343
+ },
344
+ {
345
+ "cell_type": "markdown",
346
+ "id": "665ec48b",
347
+ "metadata": {},
348
+ "source": [
349
+ "## 4. Environment Configuration"
350
+ ]
351
+ },
352
+ {
353
+ "cell_type": "code",
354
+ "execution_count": null,
355
+ "id": "61531f5c",
356
+ "metadata": {},
357
+ "outputs": [],
358
+ "source": [
359
+ "# Generate .env.example\n",
360
+ "env_example = '''\n",
361
+ "# CyberForge ML Services Environment Configuration\n",
362
+ "\n",
363
+ "# Hugging Face\n",
364
+ "HF_TOKEN=your_huggingface_token_here\n",
365
+ "HF_REPO=Che237/cyberforge-models\n",
366
+ "\n",
367
+ "# Gemini API\n",
368
+ "GEMINI_API_KEY=your_gemini_api_key_here\n",
369
+ "\n",
370
+ "# WebScraper API\n",
371
+ "WEBSCRAPER_API_KEY=your_webscraper_api_key_here\n",
372
+ "WEBSCRAPER_API_URL=http://webscrapper.live/api/scrape\n",
373
+ "\n",
374
+ "# Server Configuration\n",
375
+ "ML_SERVICE_PORT=8001\n",
376
+ "ML_SERVICE_HOST=0.0.0.0\n",
377
+ "\n",
378
+ "# Model Configuration\n",
379
+ "MODELS_DIR=./models\n",
380
+ "MAX_INFERENCE_TIME_MS=100\n",
381
+ "CONFIDENCE_THRESHOLD=0.7\n",
382
+ "'''\n",
383
+ "\n",
384
+ "env_path = DEPLOY_DIR / \".env.example\"\n",
385
+ "with open(env_path, 'w') as f:\n",
386
+ " f.write(env_example)\n",
387
+ "\n",
388
+ "print(f\"βœ“ .env.example saved\")"
389
+ ]
390
+ },
391
+ {
392
+ "cell_type": "markdown",
393
+ "id": "86c547c3",
394
+ "metadata": {},
395
+ "source": [
396
+ "## 5. Upload Script"
397
+ ]
398
+ },
399
+ {
400
+ "cell_type": "code",
401
+ "execution_count": null,
402
+ "id": "21703867",
403
+ "metadata": {},
404
+ "outputs": [],
405
+ "source": [
406
+ "# Generate HuggingFace upload script\n",
407
+ "upload_script = '''\n",
408
+ "#!/usr/bin/env python3\n",
409
+ "\"\"\"\n",
410
+ "CyberForge - Upload to Hugging Face Hub\n",
411
+ "\"\"\"\n",
412
+ "\n",
413
+ "import os\n",
414
+ "import sys\n",
415
+ "from pathlib import Path\n",
416
+ "from huggingface_hub import HfApi, login\n",
417
+ "\n",
418
+ "def main():\n",
419
+ " # Get token\n",
420
+ " token = os.environ.get('HF_TOKEN')\n",
421
+ " if not token:\n",
422
+ " print(\"Error: HF_TOKEN environment variable not set\")\n",
423
+ " sys.exit(1)\n",
424
+ " \n",
425
+ " # Login\n",
426
+ " login(token=token)\n",
427
+ " api = HfApi()\n",
428
+ " \n",
429
+ " # Configuration\n",
430
+ " repo_id = os.environ.get('HF_REPO', 'Che237/cyberforge-models')\n",
431
+ " upload_dir = Path('./huggingface_upload')\n",
432
+ " \n",
433
+ " if not upload_dir.exists():\n",
434
+ " print(f\"Error: Upload directory not found: {upload_dir}\")\n",
435
+ " sys.exit(1)\n",
436
+ " \n",
437
+ " print(f\"Uploading to: {repo_id}\")\n",
438
+ " \n",
439
+ " # Upload\n",
440
+ " try:\n",
441
+ " api.upload_folder(\n",
442
+ " folder_path=str(upload_dir),\n",
443
+ " repo_id=repo_id,\n",
444
+ " repo_type=\"model\",\n",
445
+ " commit_message=\"Update CyberForge ML models\"\n",
446
+ " )\n",
447
+ " print(f\"βœ“ Upload complete: https://huggingface.co/{repo_id}\")\n",
448
+ " except Exception as e:\n",
449
+ " print(f\"Error: {e}\")\n",
450
+ " sys.exit(1)\n",
451
+ "\n",
452
+ "if __name__ == \"__main__\":\n",
453
+ " main()\n",
454
+ "'''\n",
455
+ "\n",
456
+ "upload_script_path = DEPLOY_DIR / \"upload_to_hf.py\"\n",
457
+ "with open(upload_script_path, 'w') as f:\n",
458
+ " f.write(upload_script)\n",
459
+ "\n",
460
+ "os.chmod(upload_script_path, 0o755)\n",
461
+ "print(f\"βœ“ Upload script saved\")"
462
+ ]
463
+ },
464
+ {
465
+ "cell_type": "markdown",
466
+ "id": "289852a7",
467
+ "metadata": {},
468
+ "source": [
469
+ "## 6. Deployment Documentation"
470
+ ]
471
+ },
472
+ {
473
+ "cell_type": "code",
474
+ "execution_count": null,
475
+ "id": "4b823e84",
476
+ "metadata": {},
477
+ "outputs": [],
478
+ "source": [
479
+ "# Generate deployment guide\n",
480
+ "deployment_guide = f'''\n",
481
+ "# CyberForge ML Deployment Guide\n",
482
+ "\n",
483
+ "## Quick Start\n",
484
+ "\n",
485
+ "### 1. Local Deployment\n",
486
+ "\n",
487
+ "```bash\n",
488
+ "# Install dependencies\n",
489
+ "pip install -r requirements.txt\n",
490
+ "\n",
491
+ "# Start server\n",
492
+ "uvicorn inference:app --host 0.0.0.0 --port 8001\n",
493
+ "```\n",
494
+ "\n",
495
+ "### 2. Docker Deployment\n",
496
+ "\n",
497
+ "```bash\n",
498
+ "# Build and run\n",
499
+ "docker-compose up -d\n",
500
+ "\n",
501
+ "# Check logs\n",
502
+ "docker-compose logs -f\n",
503
+ "```\n",
504
+ "\n",
505
+ "### 3. Hugging Face Deployment\n",
506
+ "\n",
507
+ "```bash\n",
508
+ "# Set token\n",
509
+ "export HF_TOKEN=your_token_here\n",
510
+ "\n",
511
+ "# Upload\n",
512
+ "python upload_to_hf.py\n",
513
+ "```\n",
514
+ "\n",
515
+ "## API Endpoints\n",
516
+ "\n",
517
+ "### Prediction\n",
518
+ "\n",
519
+ "```\n",
520
+ "POST /predict\n",
521
+ "Content-Type: application/json\n",
522
+ "\n",
523
+ "{{\n",
524
+ " \"model_name\": \"phishing_detection\",\n",
525
+ " \"features\": {{\n",
526
+ " \"url_length\": 50,\n",
527
+ " \"has_https\": true,\n",
528
+ " ...\n",
529
+ " }}\n",
530
+ "}}\n",
531
+ "```\n",
532
+ "\n",
533
+ "### List Models\n",
534
+ "\n",
535
+ "```\n",
536
+ "GET /models\n",
537
+ "```\n",
538
+ "\n",
539
+ "### Health Check\n",
540
+ "\n",
541
+ "```\n",
542
+ "GET /health\n",
543
+ "```\n",
544
+ "\n",
545
+ "## Environment Variables\n",
546
+ "\n",
547
+ "| Variable | Description | Required |\n",
548
+ "|----------|-------------|----------|\n",
549
+ "| HF_TOKEN | Hugging Face API token | For HF upload |\n",
550
+ "| GEMINI_API_KEY | Gemini API key | For AI reasoning |\n",
551
+ "| WEBSCRAPER_API_KEY | WebScraper API key | For data collection |\n",
552
+ "\n",
553
+ "## Monitoring\n",
554
+ "\n",
555
+ "- Health endpoint: `/health`\n",
556
+ "- Metrics endpoint: `/metrics`\n",
557
+ "- Logs: `docker-compose logs -f ml-services`\n",
558
+ "\n",
559
+ "## Troubleshooting\n",
560
+ "\n",
561
+ "### Model not found\n",
562
+ "- Check model files exist in `models/` directory\n",
563
+ "- Verify manifest.json includes the model\n",
564
+ "\n",
565
+ "### Slow inference\n",
566
+ "- Check model size\n",
567
+ "- Consider using lighter model variant\n",
568
+ "- Verify no resource contention\n",
569
+ "\n",
570
+ "### API errors\n",
571
+ "- Check logs for stack traces\n",
572
+ "- Verify input format matches expected schema\n",
573
+ "- Ensure all dependencies installed\n",
574
+ "\n",
575
+ "## Support\n",
576
+ "\n",
577
+ "For issues and feature requests, please open an issue on GitHub.\n",
578
+ "'''\n",
579
+ "\n",
580
+ "guide_path = DEPLOY_DIR / \"DEPLOYMENT.md\"\n",
581
+ "with open(guide_path, 'w') as f:\n",
582
+ " f.write(deployment_guide)\n",
583
+ "\n",
584
+ "print(f\"βœ“ Deployment guide saved\")"
585
+ ]
586
+ },
587
+ {
588
+ "cell_type": "markdown",
589
+ "id": "54cad563",
590
+ "metadata": {},
591
+ "source": [
592
+ "## 7. Final Package Verification"
593
+ ]
594
+ },
595
+ {
596
+ "cell_type": "code",
597
+ "execution_count": null,
598
+ "id": "502eb389",
599
+ "metadata": {},
600
+ "outputs": [],
601
+ "source": [
602
+ "# Verify deployment package\n",
603
+ "required_files = [\n",
604
+ " 'Dockerfile',\n",
605
+ " 'docker-compose.yml',\n",
606
+ " 'requirements.txt',\n",
607
+ " '.env.example',\n",
608
+ " 'upload_to_hf.py',\n",
609
+ " 'DEPLOYMENT.md',\n",
610
+ " 'huggingface_upload/README.md'\n",
611
+ "]\n",
612
+ "\n",
613
+ "print(\"Verifying deployment package...\\n\")\n",
614
+ "\n",
615
+ "all_present = True\n",
616
+ "for file in required_files:\n",
617
+ " path = DEPLOY_DIR / file\n",
618
+ " exists = path.exists()\n",
619
+ " status = \"βœ“\" if exists else \"βœ—\"\n",
620
+ " print(f\" {status} {file}\")\n",
621
+ " if not exists:\n",
622
+ " all_present = False\n",
623
+ "\n",
624
+ "print(f\"\\n{'βœ“ All files present' if all_present else 'βœ— Some files missing'}\")"
625
+ ]
626
+ },
627
+ {
628
+ "cell_type": "markdown",
629
+ "id": "4eea48b3",
630
+ "metadata": {},
631
+ "source": [
632
+ "## 8. Summary"
633
+ ]
634
+ },
635
+ {
636
+ "cell_type": "code",
637
+ "execution_count": null,
638
+ "id": "0371e31b",
639
+ "metadata": {},
640
+ "outputs": [],
641
+ "source": [
642
+ "# Calculate package stats\n",
643
+ "total_files = len(list(DEPLOY_DIR.rglob('*')))\n",
644
+ "total_size = sum(f.stat().st_size for f in DEPLOY_DIR.rglob('*') if f.is_file())\n",
645
+ "\n",
646
+ "print(\"\\n\" + \"=\" * 60)\n",
647
+ "print(\"DEPLOYMENT ARTIFACTS COMPLETE\")\n",
648
+ "print(\"=\" * 60)\n",
649
+ "\n",
650
+ "print(f\"\"\"\n",
651
+ "πŸš€ Deployment Package Ready:\n",
652
+ " - Location: {DEPLOY_DIR}\n",
653
+ " - Files: {total_files}\n",
654
+ " - Total size: {total_size / (1024*1024):.2f} MB\n",
655
+ "\n",
656
+ "πŸ“¦ Package Contents:\n",
657
+ " - Dockerfile: Container configuration\n",
658
+ " - docker-compose.yml: Multi-service orchestration\n",
659
+ " - requirements.txt: Python dependencies\n",
660
+ " - .env.example: Environment template\n",
661
+ " - upload_to_hf.py: HuggingFace upload script\n",
662
+ " - DEPLOYMENT.md: Deployment guide\n",
663
+ " - huggingface_upload/: HF Hub package\n",
664
+ "\n",
665
+ "πŸ”§ Deployment Options:\n",
666
+ " 1. Local: uvicorn inference:app --port 8001\n",
667
+ " 2. Docker: docker-compose up -d\n",
668
+ " 3. HuggingFace: python upload_to_hf.py\n",
669
+ "\n",
670
+ "πŸ“‹ Next Steps:\n",
671
+ " 1. Set environment variables (copy .env.example to .env)\n",
672
+ " 2. Choose deployment method\n",
673
+ " 3. Verify health endpoint\n",
674
+ " 4. Integrate with backend services\n",
675
+ "\n",
676
+ "πŸŽ‰ CyberForge ML Pipeline Complete!\n",
677
+ "\"\"\")\n",
678
+ "print(\"=\" * 60)"
679
+ ]
680
+ },
681
+ {
682
+ "cell_type": "markdown",
683
+ "id": "2ee1fdf8",
684
+ "metadata": {},
685
+ "source": [
686
+ "## 9. Optional: Upload to Hugging Face"
687
+ ]
688
+ },
689
+ {
690
+ "cell_type": "code",
691
+ "execution_count": null,
692
+ "id": "7ce9155a",
693
+ "metadata": {},
694
+ "outputs": [],
695
+ "source": [
696
+ "# Uncomment to upload to Hugging Face\n",
697
+ "# \n",
698
+ "# if hf_deployer.is_authenticated():\n",
699
+ "# print(\"Uploading to Hugging Face Hub...\")\n",
700
+ "# try:\n",
701
+ "# hf_deployer.api.upload_folder(\n",
702
+ "# folder_path=str(hf_package_dir),\n",
703
+ "# repo_id=hf_deployer.repo_id,\n",
704
+ "# repo_type=\"model\",\n",
705
+ "# commit_message=\"Update CyberForge ML models\"\n",
706
+ "# )\n",
707
+ "# print(f\"βœ“ Uploaded to: https://huggingface.co/{hf_deployer.repo_id}\")\n",
708
+ "# except Exception as e:\n",
709
+ "# print(f\"Upload error: {e}\")\n",
710
+ "# else:\n",
711
+ "# print(\"Not authenticated. Set HF_TOKEN to upload.\")\n",
712
+ "\n",
713
+ "print(\"To upload, set HF_TOKEN and run: python deployment/upload_to_hf.py\")"
714
+ ]
715
+ }
716
+ ],
717
+ "metadata": {
718
+ "language_info": {
719
+ "name": "python"
720
+ }
721
+ },
722
+ "nbformat": 4,
723
+ "nbformat_minor": 5
724
+ }