Fix cells - make each self-contained with imports

notebooks/00_environment_setup.ipynb

@@ -51,23 +51,23 @@
     "\n",
     "# Python version check\n",
     "python_version = sys.version_info\n",
-    "print(f\"\\n✓ Python Version: {python_version.major}.{python_version.minor}.{python_version.micro}\")\n",
+    "print(f\"\\n\u2713 Python Version: {python_version.major}.{python_version.minor}.{python_version.micro}\")\n",
     "\n",
     "if python_version.major < 3 or (python_version.major == 3 and python_version.minor < 10):\n",
     "    raise EnvironmentError(\"Python 3.10+ is required. Please upgrade your Python installation.\")\n",
     "\n",
     "# System info\n",
-    "print(f\"✓ Platform: {platform.system()} {platform.release()}\")\n",
-    "print(f\"✓ Architecture: {platform.machine()}\")\n",
-    "print(f\"✓ Processor: {platform.processor() or 'Unknown'}\")\n",
+    "print(f\"\u2713 Platform: {platform.system()} {platform.release()}\")\n",
+    "print(f\"\u2713 Architecture: {platform.machine()}\")\n",
+    "print(f\"\u2713 Processor: {platform.processor() or 'Unknown'}\")\n",
     "\n",
     "# Memory info\n",
     "try:\n",
     "    import psutil\n",
     "    memory = psutil.virtual_memory()\n",
-    "    print(f\"✓ Available Memory: {memory.available / (1024**3):.2f} GB / {memory.total / (1024**3):.2f} GB\")\n",
+    "    print(f\"\u2713 Available Memory: {memory.available / (1024**3):.2f} GB / {memory.total / (1024**3):.2f} GB\")\n",
     "except ImportError:\n",
-    "    print(\"⚠ psutil not installed - memory check skipped\")\n",
+    "    print(\"\u26a0 psutil not installed - memory check skipped\")\n",
     "\n",
     "print(\"\\n\" + \"=\" * 60)"
    ]
@@ -86,50 +86,7 @@
    "id": "faa9b079",
    "metadata": {},
    "outputs": [],
-   "source": [
-    "# Core dependencies with pinned versions for reproducibility\n",
-    "DEPENDENCIES = \"\"\"\n",
-    "# Core ML/AI\n",
-    "numpy>=1.24.0,<2.0.0\n",
-    "pandas>=2.0.0\n",
-    "scikit-learn>=1.3.0\n",
-    "scipy>=1.11.0\n",
-    "\n",
-    "# Deep Learning\n",
-    "torch>=2.0.0\n",
-    "transformers>=4.30.0\n",
-    "\n",
-    "# Gemini API\n",
-    "google-generativeai>=0.3.0\n",
-    "\n",
-    "# Data Processing\n",
-    "joblib>=1.3.0\n",
-    "tqdm>=4.65.0\n",
-    "\n",
-    "# Feature Engineering\n",
-    "tldextract>=5.0.0\n",
-    "validators>=0.22.0\n",
-    "ipaddress>=1.0.23\n",
-    "\n",
-    "# Web/API\n",
-    "httpx>=0.25.0\n",
-    "aiohttp>=3.8.0\n",
-    "requests>=2.31.0\n",
-    "\n",
-    "# Hugging Face\n",
-    "huggingface_hub>=0.19.0\n",
-    "\n",
-    "# Utilities\n",
-    "python-dotenv>=1.0.0\n",
-    "pyyaml>=6.0.0\n",
-    "psutil>=5.9.0\n",
-    "\"\"\"\n",
-    "\n",
-    "# Write requirements file\n",
-    "requirements_path = Path(\"../requirements_notebooks.txt\")\n",
-    "requirements_path.write_text(DEPENDENCIES.strip())\n",
-    "print(f\"✓ Requirements written to: {requirements_path.absolute()}\")"
-   ]
+   "source": "from pathlib import Path\n\n# Core dependencies with pinned versions for reproducibility\nDEPENDENCIES = \"\"\"\n# Core ML/AI\nnumpy>=1.24.0,<2.0.0\npandas>=2.0.0\nscikit-learn>=1.3.0\nscipy>=1.11.0\n\n# Deep Learning  \ntorch>=2.0.0\ntransformers>=4.30.0\n\n# Gemini API\ngoogle-generativeai>=0.3.0\n\n# Data Processing\njoblib>=1.3.0\ntqdm>=4.65.0\n\n# Feature Engineering\ntldextract>=5.0.0\nvalidators>=0.22.0\n\n# Web/API\nhttpx>=0.25.0\nrequests>=2.31.0\n\n# Hugging Face\nhuggingface_hub>=0.19.0\n\n# Utilities\npython-dotenv>=1.0.0\npyyaml>=6.0.0\npsutil>=5.9.0\n\"\"\"\n\n# Write requirements file\nrequirements_path = Path(\"../requirements_notebooks.txt\")\nrequirements_path.write_text(DEPENDENCIES.strip())\nprint(f\"\u2713 Requirements written to: {requirements_path.absolute()}\")\n"
   },
   {
    "cell_type": "code",
@@ -137,22 +94,7 @@
    "id": "7dc8c6ca",
    "metadata": {},
    "outputs": [],
-   "source": [
-    "# Install dependencies\n",
-    "import subprocess\n",
-    "\n",
-    "print(\"Installing dependencies... This may take a few minutes.\")\n",
-    "result = subprocess.run(\n",
-    "    [sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"-r\", str(requirements_path)],\n",
-    "    capture_output=True,\n",
-    "    text=True\n",
-    ")\n",
-    "\n",
-    "if result.returncode == 0:\n",
-    "    print(\"✓ All dependencies installed successfully!\")\n",
-    "else:\n",
-    "    print(f\"⚠ Installation warnings: {result.stderr[:500] if result.stderr else 'None'}\")"
-   ]
+   "source": "import subprocess\nimport sys\nfrom pathlib import Path\n\n# Install dependencies\nrequirements_path = Path(\"../requirements_notebooks.txt\")\n\nif requirements_path.exists():\n    print(\"Installing dependencies... This may take a few minutes.\")\n    result = subprocess.run(\n        [sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"-r\", str(requirements_path)],\n        capture_output=True,\n        text=True\n    )\n\n    if result.returncode == 0:\n        print(\"\u2713 All dependencies installed successfully!\")\n    else:\n        print(f\"\u26a0 Installation warnings: {result.stderr[:500] if result.stderr else 'None'}\")\nelse:\n    print(\"\u26a0 Requirements file not found. Run previous cell first or skip if deps installed.\")\n"
   },
   {
    "cell_type": "markdown",
@@ -177,26 +119,26 @@
     "\n",
     "# Check CUDA availability\n",
     "cuda_available = torch.cuda.is_available()\n",
-    "print(f\"\\n✓ PyTorch Version: {torch.__version__}\")\n",
-    "print(f\"✓ CUDA Available: {cuda_available}\")\n",
+    "print(f\"\\n\u2713 PyTorch Version: {torch.__version__}\")\n",
+    "print(f\"\u2713 CUDA Available: {cuda_available}\")\n",
     "\n",
     "if cuda_available:\n",
-    "    print(f\"✓ CUDA Version: {torch.version.cuda}\")\n",
-    "    print(f\"✓ GPU Count: {torch.cuda.device_count()}\")\n",
+    "    print(f\"\u2713 CUDA Version: {torch.version.cuda}\")\n",
+    "    print(f\"\u2713 GPU Count: {torch.cuda.device_count()}\")\n",
     "    for i in range(torch.cuda.device_count()):\n",
     "        props = torch.cuda.get_device_properties(i)\n",
     "        print(f\"  - GPU {i}: {props.name} ({props.total_memory / (1024**3):.2f} GB)\")\n",
     "    DEVICE = torch.device(\"cuda\")\n",
     "else:\n",
-    "    print(\"⚠ No GPU detected - using CPU for training\")\n",
+    "    print(\"\u26a0 No GPU detected - using CPU for training\")\n",
     "    DEVICE = torch.device(\"cpu\")\n",
     "\n",
     "# Check MPS (Apple Silicon)\n",
     "if hasattr(torch.backends, 'mps') and torch.backends.mps.is_available():\n",
-    "    print(\"✓ Apple MPS (Metal) available\")\n",
+    "    print(\"\u2713 Apple MPS (Metal) available\")\n",
     "    DEVICE = torch.device(\"mps\")\n",
     "\n",
-    "print(f\"\\n✓ Selected Device: {DEVICE}\")\n",
+    "print(f\"\\n\u2713 Selected Device: {DEVICE}\")\n",
     "print(\"=\" * 60)"
    ]
   },
@@ -214,67 +156,7 @@
    "id": "0f63a5ce",
    "metadata": {},
    "outputs": [],
-   "source": [
-    "import json\n",
-    "import os\n",
-    "from pathlib import Path\n",
-    "\n",
-    "# Load configuration from notebook_config.json first (for HF Spaces)\n",
-    "config_json_path = Path(\"notebook_config.json\")\n",
-    "if config_json_path.exists():\n",
-    "    with open(config_json_path, \"r\") as f:\n",
-    "        loaded_config = json.load(f)\n",
-    "    print(f\"✓ Loaded configuration from: {config_json_path.absolute()}\")\n",
-    "else:\n",
-    "    loaded_config = {}\n",
-    "    print(f\"⚠ No notebook_config.json found, using defaults\")\n",
-    "\n",
-    "# Try loading .env file as fallback (for local dev)\n",
-    "try:\n",
-    "    from dotenv import load_dotenv\n",
-    "    env_path = Path(\"../.env\")\n",
-    "    if env_path.exists():\n",
-    "        load_dotenv(env_path)\n",
-    "        print(f\"✓ Loaded environment from: {env_path.absolute()}\")\n",
-    "except ImportError:\n",
-    "    pass\n",
-    "\n",
-    "# Configuration class\n",
-    "class Config:\n",
-    "    # API Keys - priority: config.json > env vars > HF secrets\n",
-    "    GEMINI_API_KEY = loaded_config.get(\"gemini_api_key\") or os.getenv(\"GEMINI_API_KEY\", \"AIzaSyA3HdWTLk_zJQ5P9G8Z8a8BEYSTPvLglhs\")\n",
-    "    HUGGINGFACE_TOKEN = os.getenv(\"HUGGINGFACE_API_TOKEN\", os.getenv(\"HF_TOKEN\", \"\"))\n",
-    "    WEBSCRAPER_API_KEY = loaded_config.get(\"webscraper_api_key\", \"sk-fd14eaa7bceb478db7afc7256e514d2b\")\n",
-    "    WEBSCRAPER_API_URL = loaded_config.get(\"webscraper_api_url\", \"http://webscrapper.live/api/scrape\")\n",
-    "    \n",
-    "    # Gemini model\n",
-    "    GEMINI_MODEL = loaded_config.get(\"gemini_model\", \"gemini-2.5-flash\")\n",
-    "    \n",
-    "    # Paths\n",
-    "    BASE_DIR = Path(\"..\").resolve()\n",
-    "    DATASETS_DIR = BASE_DIR / \"datasets\"\n",
-    "    MODELS_DIR = BASE_DIR / \"models\"\n",
-    "    ARTIFACTS_DIR = BASE_DIR / \"artifacts\"\n",
-    "    \n",
-    "    # ML Settings\n",
-    "    RANDOM_STATE = loaded_config.get(\"random_state\", 42)\n",
-    "    TEST_SIZE = loaded_config.get(\"test_size\", 0.2)\n",
-    "    CV_FOLDS = loaded_config.get(\"cv_folds\", 5)\n",
-    "    \n",
-    "    # Device\n",
-    "    DEVICE = DEVICE\n",
-    "\n",
-    "config = Config()\n",
-    "\n",
-    "# Validate required API keys\n",
-    "print(\"\\n\" + \"=\" * 60)\n",
-    "print(\"API CONFIGURATION STATUS\")\n",
-    "print(\"=\" * 60)\n",
-    "print(f\"✓ Gemini API Key: {'Configured (' + config.GEMINI_API_KEY[:10] + '...)' if config.GEMINI_API_KEY else '⚠ NOT SET'}\")\n",
-    "print(f\"✓ Gemini Model: {config.GEMINI_MODEL}\")\n",
-    "print(f\"✓ HuggingFace Token: {'Configured' if config.HUGGINGFACE_TOKEN else '⚠ NOT SET (optional)'}\")\n",
-    "print(f\"✓ WebScraper API: Configured\")"
-   ]
+   "source": "import json\nimport os\nfrom pathlib import Path\n\n# Load configuration from notebook_config.json first (for HF Spaces)\nconfig_json_path = Path(\"notebook_config.json\")\nif config_json_path.exists():\n    with open(config_json_path, \"r\") as f:\n        loaded_config = json.load(f)\n    print(f\"\u2713 Loaded configuration from: {config_json_path.absolute()}\")\nelse:\n    loaded_config = {}\n    print(f\"\u26a0 No notebook_config.json found, using defaults\")\n\n# Try loading .env file as fallback (for local dev)\ntry:\n    from dotenv import load_dotenv\n    env_path = Path(\"../.env\")\n    if env_path.exists():\n        load_dotenv(env_path)\n        print(f\"\u2713 Loaded environment from: {env_path.absolute()}\")\nexcept ImportError:\n    pass\n\n# Detect device\ntry:\n    import torch\n    if torch.cuda.is_available():\n        DEVICE = torch.device(\"cuda\")\n    elif hasattr(torch.backends, 'mps') and torch.backends.mps.is_available():\n        DEVICE = torch.device(\"mps\")\n    else:\n        DEVICE = torch.device(\"cpu\")\nexcept ImportError:\n    DEVICE = \"cpu\"\n\n# Configuration class\nclass Config:\n    # API Keys - priority: config.json > env vars > defaults\n    GEMINI_API_KEY = loaded_config.get(\"gemini_api_key\") or os.getenv(\"GEMINI_API_KEY\", \"AIzaSyA3HdWTLk_zJQ5P9G8Z8a8BEYSTPvLglhs\")\n    HUGGINGFACE_TOKEN = os.getenv(\"HUGGINGFACE_API_TOKEN\", os.getenv(\"HF_TOKEN\", \"\"))\n    WEBSCRAPER_API_KEY = loaded_config.get(\"webscraper_api_key\", \"sk-fd14eaa7bceb478db7afc7256e514d2b\")\n    WEBSCRAPER_API_URL = loaded_config.get(\"webscraper_api_url\", \"http://webscrapper.live/api/scrape\")\n    \n    # Gemini model\n    GEMINI_MODEL = loaded_config.get(\"gemini_model\", \"gemini-2.5-flash\")\n    \n    # Paths\n    BASE_DIR = Path(\"..\").resolve()\n    DATASETS_DIR = BASE_DIR / \"datasets\"\n    MODELS_DIR = BASE_DIR / \"models\"\n    ARTIFACTS_DIR = BASE_DIR / \"artifacts\"\n    \n    # ML Settings\n    RANDOM_STATE = loaded_config.get(\"random_state\", 42)\n    TEST_SIZE = loaded_config.get(\"test_size\", 0.2)\n    CV_FOLDS = loaded_config.get(\"cv_folds\", 5)\n    \n    # Device\n    DEVICE = DEVICE\n\nconfig = Config()\n\n# Validate required API keys\nprint(\"\\n\" + \"=\" * 60)\nprint(\"API CONFIGURATION STATUS\")\nprint(\"=\" * 60)\n"
   },
   {
    "cell_type": "markdown",
@@ -317,9 +199,9 @@
     "success, message = test_gemini_connection()\n",
     "\n",
     "if success:\n",
-    "    print(f\"✓ Gemini API: {message}\")\n",
+    "    print(f\"\u2713 Gemini API: {message}\")\n",
     "else:\n",
-    "    print(f\"⚠ Gemini API: Connection failed - {message}\")"
+    "    print(f\"\u26a0 Gemini API: Connection failed - {message}\")"
    ]
   },
   {
@@ -336,44 +218,7 @@
    "id": "beb1b036",
    "metadata": {},
    "outputs": [],
-   "source": [
-    "import httpx\n",
-    "import asyncio\n",
-    "\n",
-    "# Install nest_asyncio for Jupyter compatibility\n",
-    "try:\n",
-    "    import nest_asyncio\n",
-    "    nest_asyncio.apply()\n",
-    "except ImportError:\n",
-    "    pass  # Will use synchronous fallback\n",
-    "\n",
-    "def test_webscraper_connection_sync():\n",
-    "    \"\"\"Test WebScrapper.live API connectivity (sync version)\"\"\"\n",
-    "    try:\n",
-    "        with httpx.Client(timeout=30.0) as client:\n",
-    "            response = client.post(\n",
-    "                config.WEBSCRAPER_API_URL,\n",
-    "                json={\"url\": \"https://example.com\"},\n",
-    "                headers={\n",
-    "                    \"Content-Type\": \"application/json\",\n",
-    "                    \"X-API-Key\": config.WEBSCRAPER_API_KEY\n",
-    "                }\n",
-    "            )\n",
-    "            if response.status_code == 200:\n",
-    "                return True, \"Connected\"\n",
-    "            else:\n",
-    "                return False, f\"Status {response.status_code}: {response.text[:100]}\"\n",
-    "    except Exception as e:\n",
-    "        return False, str(e)\n",
-    "\n",
-    "print(\"Testing Web Scraper API connection...\")\n",
-    "success, message = test_webscraper_connection_sync()\n",
-    "\n",
-    "if success:\n",
-    "    print(f\"✓ WebScraper API: Connected successfully\")\n",
-    "else:\n",
-    "    print(f\"⚠ WebScraper API: {message}\")"
-   ]
+   "source": "import httpx\n\ndef test_webscraper_connection_sync():\n    \"\"\"Test WebScrapper.live API connectivity (sync version)\"\"\"\n    try:\n        with httpx.Client(timeout=30.0) as client:\n            response = client.post(\n                config.WEBSCRAPER_API_URL,\n                json={\"url\": \"https://example.com\"},\n                headers={\n                    \"Content-Type\": \"application/json\",\n                    \"X-API-Key\": config.WEBSCRAPER_API_KEY\n                }\n            )\n            if response.status_code == 200:\n                return True, \"Connected\"\n            else:\n                return False, f\"Status {response.status_code}: {response.text[:100]}\"\n    except Exception as e:\n        return False, str(e)\n\nprint(\"Testing Web Scraper API connection...\")\nsuccess, message = test_webscraper_connection_sync()\n\nif success:\n    print(f\"\u2713 WebScraper API: Connected successfully\")\nelse:\n    print(f\"\u26a0 WebScraper API: {message}\")\n"
   },
   {
    "cell_type": "markdown",
@@ -402,9 +247,9 @@
     "print(\"Creating directory structure...\")\n",
     "for directory in directories:\n",
     "    directory.mkdir(parents=True, exist_ok=True)\n",
-    "    print(f\"  ✓ {directory}\")\n",
+    "    print(f\"  \u2713 {directory}\")\n",
     "\n",
-    "print(\"\\n✓ Directory structure ready!\")"
+    "print(\"\\n\u2713 Directory structure ready!\")"
    ]
   },
   {
@@ -446,7 +291,7 @@
     "with open(config_path, \"w\") as f:\n",
     "    json.dump(notebook_config, f, indent=2)\n",
     "\n",
-    "print(f\"✓ Configuration saved to: {config_path}\")\n",
+    "print(f\"\u2713 Configuration saved to: {config_path}\")\n",
     "print(\"\\n\" + json.dumps(notebook_config, indent=2))"
    ]
   },
@@ -469,16 +314,16 @@
     "print(\"ENVIRONMENT SETUP COMPLETE\")\n",
     "print(\"=\" * 60)\n",
     "print(f\"\"\"\n",
-    "✅ Python: {python_version.major}.{python_version.minor}.{python_version.micro}\n",
-    "✅ Device: {DEVICE}\n",
-    "✅ PyTorch: {torch.__version__}\n",
-    "✅ Gemini API: {'Ready' if config.GEMINI_API_KEY else 'Not configured'}\n",
-    "✅ HuggingFace: {'Ready' if config.HUGGINGFACE_TOKEN else 'Not configured'}\n",
-    "✅ WebScraper API: Ready\n",
-    "✅ Directories: Created\n",
+    "\u2705 Python: {python_version.major}.{python_version.minor}.{python_version.micro}\n",
+    "\u2705 Device: {DEVICE}\n",
+    "\u2705 PyTorch: {torch.__version__}\n",
+    "\u2705 Gemini API: {'Ready' if config.GEMINI_API_KEY else 'Not configured'}\n",
+    "\u2705 HuggingFace: {'Ready' if config.HUGGINGFACE_TOKEN else 'Not configured'}\n",
+    "\u2705 WebScraper API: Ready\n",
+    "\u2705 Directories: Created\n",
     "\n",
     "You can now proceed to the next notebook:\n",
-    "  → 01_data_acquisition.ipynb\n",
+    "  \u2192 01_data_acquisition.ipynb\n",
     "\"\"\")\n",
     "print(\"=\" * 60)"
    ]
@@ -491,4 +336,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 5
-}
+}