Spaces:
Sleeping
Sleeping
File size: 3,480 Bytes
270b534 b8043cd eaa9a59 b8043cd 270b534 b8043cd 270b534 b8043cd 270b534 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 | """
Collecting Phishing URLs from the Phishing database on Github and loading them into Hopswork
There are multiple pages on github the .txt files with active pshing urls.
1. Gather the urls to all the .txt files witht the Phishing urls
2. Gather the data from the .txt files
3. Load the data into Hopswork
"""
# Imports
import requests
import pandas as pd
import sys
import os
# Add the src folder to sys.path so Python can see utils
src_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) # go up one level from data/
sys.path.append(src_folder)
# Now import
from utils import hopsworks_utils as hw
# Global variables
MANIFEST_LINK = "https://raw.githubusercontent.com/Phishing-Database/Phishing.Database/refs/heads/master/phishing-links-ACTIVE/phishing-links-ACTIVE.manifest.txt"
LINK_BASE = "https://raw.githubusercontent.com/Phishing-Database/Phishing.Database/refs/heads/master/phishing-links-ACTIVE/"
def request_manifest_link():
"""
Requests and collects data from the manifest page.
The manifest pages that holds what pages that contain the active Phishing urls
"""
# Gather all links to the .txt files with the active Phising urls
resp = requests.get(MANIFEST_LINK, timeout=10)
resp.raise_for_status()
content = resp.text
txt_links = list(dict.fromkeys(line for line in content.split("\n") if line.strip()))
return txt_links
def request_phishing_urls(links):
"""
Requesting data from links to .txt files with phishing urls
:param links: a list with the suffix of the URLs for pages with active phishing links
:return all_dataframe: a dataframe containing the phishing URLs
"""
all_urls = []
for link_suffix in links:
# Request the data
resp = requests.get(LINK_BASE + link_suffix, timeout=10)
resp.raise_for_status()
content = resp.text
current_urls = list(dict.fromkeys(line for line in content.split("\n") if line.strip()))
# Keep only http/https URLs
current_urls = [url for url in current_urls if url.startswith(("http://", "https://"))]
#print(f"amount of collected urls after filtering:{len(current_urls)}")
# Append to all_urls
all_urls += current_urls
# remove duplicates across files
all_urls = list(dict.fromkeys(all_urls))
# Create DataFrame
df = pd.DataFrame({
'url_id': range(len(all_urls)),
'url': all_urls,
'is_phishing': 1 # 0 for legitimate URLs
})
return df
def load_into_hopswork(dataframe):
"""
Loading the Phishing urls into Hopswork
:param dataframes: A list of dataframes containing the Phishing URLs
"""
# connecting to hopswork
project = hw.connect_to_hopsworks()
# load dataframe into a feature group
fg = hw.upload_dataframe_to_feature_group(
project=project,
df=dataframe,
feature_group_name="phishing_urls",
version=2,
description="A dataframe with all the active Phishing URLs collected from the updated Phishing database",
primary_key= ["url_id"],
event_time= None,
online_enabled=True
)
def main():
# Get the required link suffixs
all_links = request_manifest_link()
# Get the data from the links
dataframes = request_phishing_urls(all_links)
# load the data into Hopswork
load_into_hopswork(dataframes)
if __name__ == "__main__":
main() |