Add unit tests for sitemap parser and URL extraction functions (#8)

examples/upload_to_hopsworks.py

@@ -0,0 +1,47 @@
+import sys
+from pathlib import Path
+
+import hopsworks
+import pandas as pd
+
+project_root = Path(__file__).parent.parent
+sys.path.insert(0, str(project_root / "src"))
+
+from phising_detection.utils.hopsworks_utils import (
+    connect_to_hopsworks,
+    get_or_create_feature_group,
+    upload_dataframe_to_feature_group
+)
+
+
+def upload_legit_urlcsv_to_hopsworks():
+
+
+    # Load the legitimate URLs dataset
+    legit_urls_df = pd.read_csv("../src/phising_detection/data/data_files/legitimate-urls-extracted.csv",)
+
+    # Connect to Hopsworks project
+    project = connect_to_hopsworks()
+
+    # Define feature group parameters
+    feature_group_name = "scan_progress_legit_urls"
+    version = 1
+    description = "Feature group for legitimate URLs dataset"
+    primary_key = ["domain"]
+    event_time = None
+    online_enabled = False
+
+    # Upload the DataFrame to Hopsworks feature group
+    upload_dataframe_to_feature_group(
+        project=project,
+        df=legit_urls_df,
+        feature_group_name=feature_group_name,
+        version=version,
+        description=description,
+        primary_key=primary_key,
+        event_time=event_time,
+        online_enabled=online_enabled
+    )
+
+if __name__ == "__main__":
+    upload_legit_urlcsv_to_hopsworks()

pyproject.toml

@@ -4,9 +4,11 @@ version = "0.1.0"
 description = "Add your description here"
 requires-python = ">=3.12"
 dependencies = [
-    "pandas>=2.3.3",
-    "pandas-stubs==2.3.3.251219",
+    "confluent-kafka>=2.3.0",
+    "hopsworks>=4.2.0",
+    "pandas>=2.1.0,<2.2.0",
     "python-dotenv>=1.0.0",
+    "pyarrow>=14.0.0",
     "requests>=2.32.5",
 ]
 

src/phising_detection/data/load_data.py

@@ -22,7 +22,7 @@ def load_phishing_urls(
     """
     if file_path is None:
         # Default to the file in the same directory as this module
-        file_path = Path(__file__).parent / "phishing-links-ACTIVE.txt"
+        file_path = Path(__file__).parent / "data_files" /"phishing-links-ACTIVE.txt"
     else:
         file_path = Path(file_path)
 

src/phising_detection/utils/hopsworks_utils.py

@@ -0,0 +1,224 @@
+"""Utilities for connecting to and interacting with Hopsworks Feature Store."""
+
+import os
+import logging
+from typing import Optional
+import pandas as pd
+import hopsworks
+from dotenv import load_dotenv
+
+logger = logging.getLogger(__name__)
+
+# Load environment variables
+load_dotenv()
+
+
+def connect_to_hopsworks(api_key: Optional[str] = None, project_name: Optional[str] = None):
+    """
+    Connect to Hopsworks.
+
+    Args:
+        api_key: Hopsworks API key. If None, reads from HOPSWORKS_API_KEY env variable.
+        project_name: Hopsworks project name. If None, reads from HOPSWORKS_PROJECT_NAME env variable.
+
+    Returns:
+        Hopsworks project object
+
+    Raises:
+        ValueError: If API key or project name is not provided
+    """
+    # Get API key from parameter or environment
+    api_key = api_key or os.getenv("HOPSWORKS_API_KEY")
+    if not api_key:
+        raise ValueError(
+            "Hopsworks API key not provided. "
+            "Set HOPSWORKS_API_KEY environment variable or pass api_key parameter."
+        )
+
+    # Get project name from parameter or environment
+    project_name = project_name or os.getenv("HOPSWORKS_PROJECT")
+
+    logger.info(f"Connecting to Hopsworks project: {project_name or 'default'}")
+
+    # Login to Hopsworks
+    try:
+        if project_name:
+            project = hopsworks.login(
+                api_key_value=api_key,
+                project=project_name,
+                engine="python"  # Use Python engine (serverless, no cert download)
+            )
+        else:
+            project = hopsworks.login(
+                api_key_value=api_key,
+                engine="python"
+            )
+    except Exception as e:
+        logger.error(f"Failed to connect to Hopsworks: {e}")
+        logger.info("Trying to connect without specifying project...")
+        project = hopsworks.login(
+            api_key_value=api_key,
+            engine="python"
+        )
+
+    logger.info(f"Successfully connected to Hopsworks project: {project.name}")
+    return project
+
+
+def get_or_create_feature_group(
+    project,
+    name: str,
+    version: int = 1,
+    description: str = "",
+    primary_key: list = None,
+    event_time: Optional[str] = None,
+    online_enabled: bool = False
+):
+    """
+    Get existing feature group or create new one if it doesn't exist.
+
+    Args:
+        project: Hopsworks project object
+        name: Feature group name
+        version: Feature group version
+        description: Description of the feature group
+        primary_key: List of column names to use as primary key
+        event_time: Column name to use as event time
+        online_enabled: Whether to enable online feature serving
+
+    Returns:
+        Feature group object
+    """
+    fs = project.get_feature_store()
+
+    try:
+        # Try to get existing feature group
+        fg = fs.get_feature_group(name=name, version=version)
+        logger.info(f"Retrieved existing feature group: {name} (version {version})")
+        return fg
+    except Exception:
+        # Feature group doesn't exist, will need to create it
+        logger.info(f"Feature group {name} (version {version}) not found, will create on first insert")
+        return None
+
+
+def upload_dataframe_to_feature_group(
+    project,
+    df: pd.DataFrame,
+    feature_group_name: str,
+    version: int = 1,
+    description: str = "",
+    primary_key: list = None,
+    event_time: Optional[str] = None,
+    online_enabled: bool = False,
+    write_options: dict = None
+):
+    """
+    Upload a DataFrame to a Hopsworks feature group.
+
+    Args:
+        project: Hopsworks project object
+        df: Pandas DataFrame to upload
+        feature_group_name: Name of the feature group
+        version: Feature group version
+        description: Description of the feature group
+        primary_key: List of column names to use as primary key
+        event_time: Column name to use as event time
+        online_enabled: Whether to enable online feature serving
+        write_options: Additional write options (e.g., {"wait_for_job": False})
+
+    Returns:
+        Feature group object
+    """
+    fs = project.get_feature_store()
+
+    logger.info(f"Uploading DataFrame to feature group: {feature_group_name} (version {version})")
+    logger.info(f"DataFrame shape: {df.shape}")
+
+    # Create or get feature group
+    fg = fs.get_or_create_feature_group(
+        name=feature_group_name,
+        version=version,
+        description=description,
+        primary_key=primary_key or [],
+        event_time=event_time,
+        online_enabled=online_enabled
+    )
+
+    # Insert data
+    write_options = write_options or {"wait_for_job": True}
+    fg.insert(df, write_options=write_options)
+
+    logger.info(f"Successfully uploaded {len(df)} rows to {feature_group_name}")
+    return fg
+
+
+def read_feature_group(
+    project,
+    feature_group_name: str,
+    version: int = 1,
+    online: bool = False
+) -> pd.DataFrame:
+    """
+    Read data from a Hopsworks feature group.
+
+    Args:
+        project: Hopsworks project object
+        feature_group_name: Name of the feature group
+        version: Feature group version
+        online: Whether to read from online feature store
+
+    Returns:
+        Pandas DataFrame with feature group data
+    """
+    fs = project.get_feature_store()
+
+    logger.info(f"Reading feature group: {feature_group_name} (version {version})")
+
+    fg = fs.get_feature_group(name=feature_group_name, version=version)
+
+    if online:
+        df = fg.read(online=True)
+    else:
+        df = fg.read()
+
+    logger.info(f"Read {len(df)} rows from {feature_group_name}")
+    return df
+
+
+def create_feature_view(
+    project,
+    name: str,
+    version: int = 1,
+    description: str = "",
+    query=None,
+    labels: list = None
+):
+    """
+    Create a feature view for training datasets.
+
+    Args:
+        project: Hopsworks project object
+        name: Feature view name
+        version: Feature view version
+        description: Description of the feature view
+        query: Query object to define feature selection
+        labels: List of label column names
+
+    Returns:
+        Feature view object
+    """
+    fs = project.get_feature_store()
+
+    logger.info(f"Creating feature view: {name} (version {version})")
+
+    fv = fs.create_feature_view(
+        name=name,
+        version=version,
+        description=description,
+        query=query,
+        labels=labels or []
+    )
+
+    logger.info(f"Successfully created feature view: {name}")
+    return fv