Add functionality to load and process legitimate URLs from CSV files (#3)

.gitattributes

@@ -1 +1,3 @@
 src/phising_detection/data/phishing-links-ACTIVE.txt filter=lfs diff=lfs merge=lfs -text
+src/phising_detection/data/legitimate-urls.txt filter=lfs diff=lfs merge=lfs -text
+src/phising_detection/data/top-1m.csv filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -102,4 +102,5 @@ dmypy.json
 .pytype/
 
 # Ruff
-.ruff_cache/
+.ruff_cache/
+/CLAUDE.md

examples/load_legitimate_urls_example.py

@@ -0,0 +1,28 @@
+"""Example script showing how to load legitimate URLs into a DataFrame."""
+
+import sys
+from pathlib import Path
+
+# Add src directory to Python path
+project_root = Path(__file__).parent.parent
+sys.path.insert(0, str(project_root / "src"))
+
+from phising_detection.data import load_phishing_urls
+
+# Load legitimate URLs from the file (using is_phishing=False)
+df = load_phishing_urls("../src/phising_detection/data/legitimate-urls.txt", is_phishing=False)
+
+# Display basic information
+print(f"Loaded {len(df)} legitimate URLs")
+print(f"\nDataFrame shape: {df.shape}")
+print(f"\nColumn names: {df.columns.tolist()}")
+print("\nFirst 5 rows:")
+print(df.head())
+
+# Display summary statistics
+print("\nDataset info:")
+print(df.info())
+
+# Check for duplicates
+duplicates = df['url'].duplicated().sum()
+print(f"\nNumber of duplicate URLs: {duplicates}")

src/phising_detection/data/legitimate-urls.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:910be3bd462bb93fb74c2538037e41830129c03f80e8ed3f85ed09b8eb7201ac
+size 14433775

src/phising_detection/data/load_data.py

@@ -42,4 +42,24 @@ def load_phishing_urls(
 
     return df
 
+def convert_csv_to_urls(csv_path: Union[str, Path], output_path: Union[str, Path]) -> None:
+    """
+    Convert a CSV file containing URLs to a text file with one URL per line.
+
+    Args:
+        csv_path: Path to the input CSV file.
+        output_path: Path to the output text file.
+    """
+    csv_path = Path(csv_path)
+    output_path = Path(output_path)
+
+    if not csv_path.exists():
+        raise FileNotFoundError(f"CSV file not found: {csv_path}")
+
+    # Read CSV and extract domains
+    df = pd.read_csv(csv_path, header=None, names=['rank', 'domain'])
+
+    # Write domains to text file
+    output_path.write_text('\n'.join(df['domain']))
+
 

src/phising_detection/data/top-1m.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:515899ed08b354920bac5aa60b351d25b8374f79a3f1bd7716a4f26b1c570ae2
+size 22322672

tests/test_load_data.py

@@ -106,3 +106,43 @@ class TestLoadPhishingUrls:
 
         assert df['url_id'].is_unique
         assert len(df['url_id'].unique()) == len(df)
+
+
+class TestLoadLegitimateUrls:
+    """Tests for loading legitimate URLs using load_phishing_urls with is_phishing=False."""
+
+    def test_load_legitimate_urls_basic(self, temp_legitimate_file, sample_legitimate_urls):
+        """Test basic loading of legitimate URLs."""
+        df = load_phishing_urls(temp_legitimate_file, is_phishing=False)
+
+        # Check DataFrame shape
+        assert len(df) == len(sample_legitimate_urls)
+        assert df.shape[1] == 3  # url_id, url, is_phishing
+
+        # Check all are labeled as non-phishing
+        assert all(df['is_phishing'] == 0)
+
+    def test_load_legitimate_urls_content(self, temp_legitimate_file, sample_legitimate_urls):
+        """Test that legitimate URLs are loaded correctly."""
+        df = load_phishing_urls(temp_legitimate_file, is_phishing=False)
+
+        # Check URLs match
+        assert df['url'].tolist() == sample_legitimate_urls
+
+        # Check url_ids are sequential
+        assert df['url_id'].tolist() == list(range(len(sample_legitimate_urls)))
+
+    def test_combined_phishing_and_legitimate(
+        self, temp_phishing_file, temp_legitimate_file
+    ):
+        """Test loading and combining phishing and legitimate URLs."""
+        phishing_df = load_phishing_urls(temp_phishing_file, is_phishing=True)
+        legitimate_df = load_phishing_urls(temp_legitimate_file, is_phishing=False)
+
+        # Combine datasets
+        combined_df = pd.concat([phishing_df, legitimate_df], ignore_index=True)
+
+        # Check that we have both types
+        assert (combined_df['is_phishing'] == 1).sum() == len(phishing_df)
+        assert (combined_df['is_phishing'] == 0).sum() == len(legitimate_df)
+        assert len(combined_df) == len(phishing_df) + len(legitimate_df)