Add command-line argument parsing and scheduled data pipeline for batch URL scanning

.github/workflows/scheduled_data_pipeline.yml

@@ -0,0 +1,99 @@
+name: Scheduled Data Pipeline
+
+on:
+  schedule:
+    # Load phishing URLs every 24 hours (at midnight UTC)
+    - cron: '0 0 * * *'
+    # Run batch scanner every 2 hours
+    - cron: '0 */2 * * *'
+  workflow_dispatch:  # Allow manual triggering
+    inputs:
+      job_to_run:
+        description: 'Which job to run'
+        required: true
+        type: choice
+        options:
+          - load_phishing_urls
+          - batch_scanner
+          - both
+
+jobs:
+  load-phishing-urls:
+    name: Load Phishing URLs
+    runs-on: ubuntu-latest
+    # Only run on midnight schedule or manual trigger
+    if: |
+      github.event.schedule == '0 0 * * *' ||
+      github.event_name == 'workflow_dispatch' &&
+      (github.event.inputs.job_to_run == 'load_phishing_urls' || github.event.inputs.job_to_run == 'both')
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          lfs: true  # Pull Git LFS files
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+        with:
+          version: "latest"
+
+      - name: Install dependencies
+        run: uv sync --group dev
+
+      - name: Load phishing URLs to Hopsworks
+        env:
+          HOPSWORKS_API_KEY: ${{ secrets.HOPSWORKS_API_KEY }}
+          HOPSWORKS_PROJECT_NAME: ${{ secrets.HOPSWORKS_PROJECT_NAME }}
+        run: |
+          uv run python src/phising_detection/data/load_phishing_urls.py
+
+  batch-url-scanner:
+    name: Batch URL Scanner
+    runs-on: ubuntu-latest
+    # Run on 2-hour schedule or manual trigger
+    if: |
+      github.event.schedule == '0 */2 * * *' ||
+      github.event_name == 'workflow_dispatch' &&
+      (github.event.inputs.job_to_run == 'batch_scanner' || github.event.inputs.job_to_run == 'both')
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          lfs: true
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+        with:
+          version: "latest"
+
+      - name: Install dependencies
+        run: uv sync --group dev
+
+      - name: Run batch URL scanner
+        env:
+          URLSCAN_API_KEY: ${{ secrets.URLSCAN_API_KEY }}
+          HOPSWORKS_API_KEY: ${{ secrets.HOPSWORKS_API_KEY }}
+          HOPSWORKS_PROJECT_NAME: ${{ secrets.HOPSWORKS_PROJECT_NAME }}
+        run: |
+          uv run python src/phising_detection/features/batch_url_scanner.py \
+            --fg1-name phishing_urls \
+            --fg1-version 2 \
+            --fg2-name legit_urls_before_scan \
+            --fg2-version 1 \
+            --output-fg-name urlscan_features \
+            --output-version 1 \
+            --batch-size 500 \
+            --sample-size 500
+        timeout-minutes: 30  # Prevent job from running too long

src/phising_detection/features/batch_url_scanner.py

@@ -12,6 +12,7 @@ import sys
 import os
 import logging
 import time
+import argparse
 from typing import List, Dict, Any
 import pandas as pd
 
@@ -612,15 +613,74 @@ def main(
     logger.info("=" * 80)
 
 
+def parse_args():
+    """Parse command-line arguments."""
+    parser = argparse.ArgumentParser(
+        description="Batch URL scanning pipeline for phishing detection"
+    )
+
+    parser.add_argument(
+        "--fg1-name",
+        type=str,
+        default="phishing_urls",
+        help="Name of first feature group (default: phishing_urls)"
+    )
+    parser.add_argument(
+        "--fg1-version",
+        type=int,
+        default=2,
+        help="Version of first feature group (default: 2)"
+    )
+    parser.add_argument(
+        "--fg2-name",
+        type=str,
+        default="legitimate_urls",
+        help="Name of second feature group (default: legitimate_urls)"
+    )
+    parser.add_argument(
+        "--fg2-version",
+        type=int,
+        default=1,
+        help="Version of second feature group (default: 1)"
+    )
+    parser.add_argument(
+        "--output-fg-name",
+        type=str,
+        default="urlscan_features",
+        help="Name of output feature group (default: urlscan_features)"
+    )
+    parser.add_argument(
+        "--output-version",
+        type=int,
+        default=1,
+        help="Version of output feature group (default: 1)"
+    )
+    parser.add_argument(
+        "--batch-size",
+        type=int,
+        default=200,
+        help="Number of URLs to scan per batch (default: 200)"
+    )
+    parser.add_argument(
+        "--sample-size",
+        type=int,
+        default=None,
+        help="Number of samples from each input group (default: None = use all)"
+    )
+
+    return parser.parse_args()
+
+
 if __name__ == "__main__":
-    # Example usage - adjust parameters as needed
+    args = parse_args()
+
     main(
-        fg1_name="phishing_urls",
-        fg1_version=2,
-        fg2_name="legit_urls_before_scan",
-        fg2_version=1,
-        output_fg_name="urlscan_features",
-        output_version=1,
-        batch_size=300,
-        sample_size=None  # Set to None to use all available data
+        fg1_name=args.fg1_name,
+        fg1_version=args.fg1_version,
+        fg2_name=args.fg2_name,
+        fg2_version=args.fg2_version,
+        output_fg_name=args.output_fg_name,
+        output_version=args.output_version,
+        batch_size=args.batch_size,
+        sample_size=args.sample_size
     )