feature_extraction.py

# import pandas as pd
import re
from urllib.parse import urlparse, parse_qs
import tldextract
from  tldextract import  extract
import pandas as pd
import ipaddress
import math
from fuzzywuzzy import fuzz
from collections import Counter
import socket
import time




# Entropi Hesaplama Fonksiyonu
def calculate_entropy(data):
    if not data:
        return 0
    probabilities = [data.count(c) / len(data) for c in set(data)]
    return round(-sum(p * math.log2(p) for p in probabilities), 6)


legitimate_domains = [
    'google.com', 'facebook.com', 'amazon.com', 'twitter.com', 'microsoft.com', 'apple.com',
    'youtube.com', 'instagram.com', 'linkedin.com', 'wikipedia.org', 'reddit.com', 'pinterest.com',
    'tumblr.com', 'ebay.com', 'paypal.com', 'net1flix.com', 'yahoo.com', 'spotify.com', 'github.com',
    'whatsapp.com', 'dropbox.com', 'slack.com', 'airbnb.com', 'zoom.us', 'merriam-webster.com',
    'cnn.com', 'bbc.com', 'bbc.co.uk', 'nytimes.com', 'forbes.com', 'businessinsider.com', 'dell.com',
    'adobe.com', 'etsy.com', 'paypal.com', 'etsy.com', 'bloomberg.com', 'chase.com', 'bankofamerica.com',
    'hulu.com', 'salesforce.com', 'twitch.tv', 'square.com', 'vimeo.com', 'messenger.com', 'snapchat.com',
    'cnn.com', 'theguardian.com', 'theverge.com', 'techcrunch.com', 'wired.com', 'nytimes.com', 'bbc.co.uk',
    'cnbc.com', 'zoom.com', 'samsung.com', 'intel.com', 'oracle.com', 'samsung.com', 't-mobile.com',
    'att.com', 'verizon.com', 'airbnb.com', 'paypal.com', 'spotify.com', 'yelp.com', 'google.co.uk',
    'bbc.co.uk', 'sainsburys.co.uk', 'guardian.co.uk', 'walmart.com', 'target.com', 'bestbuy.com', 'costco.com',
    'alibaba.com', 'wish.com', 'etsy.com', 'snapdeal.com', 'craigslist.org', 'flickr.com', 'tripadvisor.com',
    'expedia.com', 'hotels.com', 'booking.com', 'trivago.com', 'skyscanner.com', 'airasia.com', 'ryanair.com'
]
legitimate_brands = {
    'apple', 'google', 'microsoft', 'amazon', 'facebook', 'twitter', 'paypal',
    'samsung', 'sony', 'instagram', 'linkedin', 'netflix', 'youtube', 'whatsapp',
    'tesla', 'spotify', 'airbnb', 'pinterest', 'dropbox', 'uber', 'aircanada',
    'walmart', 'bestbuy', 'target', 'mcdonalds', 'nike', 'adidas', 'wellsfargo',
    'chase', 'bankofamerica', 'hsbc', 'citibank', 'barclays', 'goldman', 'americanexpress',
    'visa', 'mastercard', 'paypal', 'square', 'shopify', 'etoro', 'robinhood', 'deutschebank',
    'goldman', 'coca-cola', 'pepsi', 'loreal', 'procterandgamble', 'colgate', 'nestle',
    'unilever', 'ford', 'chevrolet', 'toyota', 'honda', 'bmw', 'mercedes', 'audi',
    'volkswagen', 'hyundai', 'kia', 'bose', 'boeing', 'nike', 'newbalance', 'reebok',
    'underarmour', 'puma', 'gucci', 'prada', 'louisvuitton', 'chanel', 'michaelkors',
    'tiffany', 'cartier', 'burberry', 'ralphlauren', 'loreal', 'nike', 'puma', 'lacoste',
    'asics', 'skype', 'viber', 'slack', 'zoom', 'discord', 'snapchat', 'telegram', 'wechat',
    'vimeo', 'yahoo', 'reddit', 'ebay', 'etsy', 'alibaba', 'taobao', 'mercadolibre',
    'shopify', 'zara', 'h&m', 'uniqlo', 'gap', 'mango', 'asos', 'nike', 'adidas', 'puma',
    'reebok', 'newbalance', 'vans', 'converse', 'skechers', 'keds', 'ugg', 'louisvuitton',
    'chanel', 'prada', 'gucci', 'fendi', 'balenciaga', 'versace', 'hermes', 'louisvuitton',
    'bally', 'salvatoreferragamo', 'tiffany', 'cartier', 'jacobcohen', 'montblanc', 'omega',
    'tagheuer', 'calvinklein', 'tommyhilfiger', 'ralphlauren', 'lindor', 'nutella', 'dove',
    'sainsburys', 'tesco', 'costco', 'aldi', 'lidl', 'bmw', 'mercedes', 'audi', 'toyota',
    'ford', 'volvo', 'mazda', 'peugeot', 'renault', 'citroen', 'kia', 'hyundai', 'honda',
    'mitsubishi', 'nissan', 'subaru', 'chrysler', 'gmc', 'chevrolet', 'buick', 'cadillac',
    'jeep', 'ram', 'dodge', 'ferrari', 'lamborghini', 'porsche', 'rollsroyce', 'bentley',
    'astonmartin', 'mclaren', 'bugatti', 'koenigsegg', 'pagani', 'chanel', 'gucci', 'hermes'
}

with open("tldlist.txt", "r", encoding="utf-8") as file:
    known_tlds = {line.strip().lower() for line in file if line.strip()}

def contains_www(url):
    # Protokol kısmını (http:// veya https://) ayıklayalım
    # Eğer varsa, https:// veya http:// kısmını atlayacağız
    url = re.sub(r'^https?://', '', url)  # https:// veya http://'yi kaldır

    # URL'nin başında tam olarak 'www.' olup olmadığını kontrol et
    if url.startswith("www.") and url.count("www.") == 1:
        return True
    return False


def count_www(url):
    # Protokol kısmını (http:// veya https://) ayıklayalım
    url = re.sub(r'^https?://', '', url)  # https:// veya http://'yi kaldır

    # 'www.' alt dizisinin kaç kez geçtiğini kontrol et
    return url.count("www.")


def check_known_tld(url, known_tlds):
    match = re.search(r"\.([a-zA-Z]{2,})/?$", url)

    if match:
        tld = match.group(1).lower()
        return 1 if tld in known_tlds else 0  # True → 1, False → 0

    return 0  # TLD bulunamadıysa


# Typosquatting kontrol fonksiyonu (benzerlik)
def check_typosquatting(url, domain_list=legitimate_domains):
    url_domain = re.sub(r'http[s]?://|www\.', '', url).split('/')[0]  # URL'yi temizle

    # Eğer URL geçerli bir domain listesinde varsa, aldatmaca değildir
    if url_domain in domain_list:
        return 0  # Domain geçerli, aldatmaca yok

    # Potansiyel aldatmaca domainlerini kontrol et
    for domain in domain_list:
        similarity_score = fuzz.ratio(url_domain, domain)

        # Eğer benzerlik skoru 50'nin üzerinde ise aldatmaca olarak kabul et
        if similarity_score > 70:
            return 1  # Potansiyel aldatmaca

    return 0  # Benzerlik düşükse aldatmaca değil


# 2. Özellik Çıkarma Fonksiyonları
def extract_url_features(url):
    features = {}
    feature_times = {}

    try:
        if not isinstance(url, str) or not url:
            return None
        # URL'nin farklı kısımlarını ayırmak
        url = url.replace("[", "").replace("]", "")  # [ ve ] karakterlerini kaldır
        parsed_url = urlparse(url)


        extracted = tldextract.extract(url)
        domain_info = f"{extracted.domain}.{extracted.suffix}"
        domain_info_netloc = extract(parsed_url.netloc)
        hostname = parsed_url.hostname if parsed_url.hostname else ""
        path = parsed_url.path if parsed_url.path else ""
        query_params = parse_qs(parsed_url.query)
        subdomain = extracted.subdomain if extracted.subdomain else ""  # Subdomain varsa al, yoksa boş string
        tld = extracted.suffix if extracted.suffix else ""  # TLD varsa al, yoksa boş string



        #1 LENGTH
        start = time.time()
        features['url_length'] = len(url)
        feature_times['url_length'] = time.time() - start

        start = time.time()
        features['domain_length'] = len(domain_info) if domain_info else 0
        feature_times['domain_length'] = time.time() - start

        start = time.time()
        features['hostname_length'] = len(parsed_url.hostname) if parsed_url.hostname else 0
        feature_times['hostname_length'] = time.time() - start

        start = time.time()
        features['path_length'] = len(parsed_url.path) if parsed_url.path else 0
        feature_times['path_length'] = time.time() - start

        start = time.time()
        features['query_length'] = len(parsed_url.query) if parsed_url.query else 0
        feature_times['query_length'] = time.time() - start

        start = time.time()
        features['fragment_length'] = len(parsed_url.fragment) if parsed_url.fragment else 0
        feature_times['fragment_length'] = time.time() - start

        start = time.time()
        cleaned_url1 = url.rstrip('/')
        words_url = re.findall(r'\w+', cleaned_url1)
        features['length_words_raw'] = sum(len(word) for word in words_url)
        feature_times['length_words_raw'] = time.time() - start

        start = time.time()
        words_url2 = re.split(r'[./\-?&=_:,;!~@#\$%\^\*\(\)\[\]\{\}<>]', cleaned_url1)
        features['longest_words_raw'] = max(len(word) for word in words_url2) if words_url else 0
        feature_times['longest_words_raw'] = time.time() - start

        start = time.time()
        words_host = re.split(r'[./\-?&=_:,;!~@#\$%\^\*\(\)\[\]\{\}<>]', hostname)
        features['longest_word_host'] = max(len(word) for word in words_host) if words_host else 0
        feature_times['longest_word_host'] = time.time() - start

        start = time.time()
        words_path = re.split(r'[./\-?&=_:,;!~@#\$%\^\*\(\)\[\]\{\}<>]', path)
        features['longest_word_path'] = max(len(word) for word in words_path) if words_path else 0
        feature_times['longest_word_path'] = time.time() - start

        start = time.time()
        path_parts = [part for part in path.split('/') if part]
        features['num_words_path'] = len(path_parts)
        feature_times['num_words_path'] = time.time() - start

        start = time.time()
        words_host = [word for word in words_host if word.isalpha()]
        features['num_words_host'] = len(words_host)
        feature_times['num_words_host'] = time.time() - start

        start = time.time()
        features['num_uppercase'] = sum(1 for char in url if char.isupper())
        feature_times['num_uppercase'] = time.time() - start

        start = time.time()
        features['num_lowercase'] = sum(1 for char in url if char.islower())
        feature_times['num_lowercase'] = time.time() - start

        start = time.time()
        features['num_queries'] = len(query_params)
        feature_times['num_queries'] = time.time() - start

        start = time.time()
        features['max_query_length'] = max(
    (len(k) + sum(len(vv) for vv in v) + 1) for k, v in query_params.items()
) if query_params else 0
        feature_times['max_query_length'] = time.time() - start

        start = time.time()
        features['path_depth'] = parsed_url.path.count('/')
        feature_times['path_depth'] = time.time() - start


        #2 TOTAL - COUNT

        start = time.time()
        features["total_tld_count"] = len(extracted.suffix.split(".")) if extracted.suffix else 0
        feature_times["total_tld_count"] = time.time() - start

        start = time.time()
        features['total_of_dot'] = url.count('.')
        feature_times['total_of_dot'] = time.time() - start

        start = time.time()
        features['total_of_hypens'] = url.count('-')
        feature_times['total_of_hypens'] = time.time() - start

        start = time.time()
        features['total_of_at'] = url.count('@')
        feature_times['total_of_at'] = time.time() - start

        start = time.time()
        features['total_of_question_mark'] = url.count('?')
        feature_times['total_of_question_mark'] = time.time() - start

        start = time.time()
        features['total_of_slash'] = url.count('/')
        feature_times['total_of_slash'] = time.time() - start

        start = time.time()
        features['total_of_percent'] = url.count('%')
        feature_times['total_of_percent'] = time.time() - start

        start = time.time()
        features['total_of_underscore'] = url.count('_')
        feature_times['total_of_underscore'] = time.time() - start

        start = time.time()
        features['total_of_https'] = len(re.findall(r'\bhttps\b', url))
        feature_times['total_of_https'] = time.time() - start

        start = time.time()
        features['total_of_http'] = len(re.findall(r'\bhttp\b', url))
        feature_times['total_of_http'] = time.time() - start

        start = time.time()
        features['path_https_count'] = len(re.findall(r'\bhttps\b', path))
        feature_times['path_https_count'] = time.time() - start

        start = time.time()
        features['path_http_count'] = len(re.findall(r'\bhttp\b', path))
        feature_times['path_http_count'] = time.time() - start

        start = time.time()
        features['host_https_count'] = len(re.findall(r'\bhttps\b', hostname))
        feature_times['host_https_count'] = time.time() - start

        start = time.time()
        features['host_http_count'] = len(re.findall(r'\bhttp\b', hostname))
        feature_times['host_http_count'] = time.time() - start

        start = time.time()
        features['percent_20_count'] = url.count('%20')
        feature_times['percent_20_count'] = time.time() - start

        start = time.time()
        special_chars = re.findall(r"[^a-zA-Z0-9]", url)
        features['total_special_chars'] = len(special_chars)
        feature_times['total_special_chars'] = time.time() - start

        start = time.time()
        specialch_count_path = len(re.findall(r"[^a-zA-Z0-9/]", path))
        features['specialch_count_path'] = specialch_count_path
        feature_times['specialch_count_path'] = time.time() - start

        start = time.time()
        features['www_count'] = count_www(url)
        feature_times['www_count'] = time.time() - start

        start = time.time()
        features['num_subdomain_hyphens'] = sum(
            sub.count('-') for sub in extracted.subdomain.split('.')) if extracted.subdomain else 0
        feature_times['num_subdomain_hyphens'] = time.time() - start

        start = time.time()
        features['path_digits_count'] = len(re.findall(r'\d', path))
        feature_times['path_digits_count'] = time.time() - start

        start = time.time()
        features['host_digits_count'] = len(re.findall(r'\d', hostname))
        feature_times['host_digits_count'] = time.time() - start


        #3 RATİO (ORT)
        start = time.time()
        features['ratio_special_chars'] = round(features['total_special_chars'] / features['url_length'] if features[
                                                                                                          'url_length'] > 0 else 0, 6)
        feature_times['ratio_special_chars'] = time.time() - start

        start = time.time()
        numeric_chars = re.findall(r'\d', url)
        features['ratio_special_to_numeric'] = round(features['total_special_chars'] / len(numeric_chars) if len(
            numeric_chars) > 0 else 0, 6)
        feature_times['ratio_special_to_numeric'] = time.time() - start

        start = time.time()
        digits_in_url = len(re.findall(r'\d', url))
        features['ratio_digits_url'] = round(digits_in_url / len(url) if len(url) > 0 else 0, 6)
        feature_times['ratio_digits_url'] = time.time() - start

        start = time.time()
        digits_in_domain = len(re.findall(r'\d', domain_info))
        features['ratio_digits_host'] = digits_in_domain / len(domain_info) if len(
            domain_info) > 0 else 0
        feature_times['ratio_digits_host'] = time.time() - start

        start = time.time()
        features['avg_query_length'] = round(features['query_length'] / features['num_queries'] if features[
                                                                                                      'num_queries'] > 0 else 0, 6)
        feature_times['avg_query_length'] = time.time() - start


        start = time.time()
        features['entropy_of_url'] = calculate_entropy(url)
        feature_times['entropy_of_url'] = time.time() - start

        # Ortalama Kelime Uzunluğu
        start = time.time()
        words2 = re.split(r'[./\-?&=_:,;!~@#\$%\^\*\(\)\[\]\{\}<>]', url)
        words2 = [word for word in words2 if word]  # Boş stringleri temizle
        features["avg_word_length"] = round(sum(len(word) for word in words2) / len(words2) if words2 else 0, 6)
        feature_times["avg_word_length"] = time.time() - start

        #4 HAS (1 veya 0)

        start = time.time()
        features['typosquatting'] = check_typosquatting(url)  # Potansiyel aldatmaca durumu
        feature_times['typosquatting'] = time.time() - start

        start = time.time()
        features['contains_fragments'] = 1 if parsed_url.fragment else 0
        feature_times['contains_fragments'] = time.time() - start

        start = time.time()
        try:
            ipaddress.ip_address(domain_info)
            features['contains_ip'] = 1
        except ValueError:
            features['contains_ip'] = 0
        feature_times['contains_ip'] = time.time() - start

        start = time.time()
        features['contains_https_token'] = 1 if url.startswith("https://") else 0
        feature_times['contains_https_token'] = time.time() - start

        start = time.time()
        features['contains_http_token'] = 1 if url.startswith("http://") else 0
        feature_times['contains_http_token'] = time.time() - start

        start = time.time()
        features['contains_legitimate_brand'] = 1 if any(brand in url.lower() for brand in legitimate_brands) else 0
        feature_times['contains_legitimate_brand'] = time.time() - start

        start = time.time()
        features["is_known_tld"] = 1 if check_known_tld(url, known_tlds) else 0
        feature_times["is_known_tld"] = time.time() - start

        start = time.time()
        features['punycode'] = 1 if re.search(r'xn--', domain_info) else 0
        feature_times['punycode'] = time.time() - start

        start = time.time()
        features['prefix_suffix'] = 1 if '-' in domain_info else 0
        feature_times['prefix_suffix'] = time.time() - start

        start = time.time()
        if re.match(r'^\d{1,3}(\.\d{1,3}){3}$', domain_info):
            tld_in_subdomain = 0  # IP adresi durumunda TLD subdomain'de olamaz
        else:
            tld_in_subdomain = 1 if tld in subdomain.split('.') else 0  # TLD, subdomain'de varsa 1, yoksa 0
        features['tld_in_subdomain'] = tld_in_subdomain
        feature_times['tld_in_subdomain'] = time.time() - start

        start = time.time()
        features['tld_in_path'] = 1 if extracted.suffix in parsed_url.path else 0
        feature_times['tld_in_path'] = time.time() - start

        start = time.time()
        features['contains_www'] = 1 if contains_www(url) else 0
        feature_times['contains_www'] = time.time() - start

        start = time.time()
        try:
            features["contains_port"] = 1 if parsed_url.port and isinstance(parsed_url.port, int) else 0
        except ValueError:
            features["contains_port"] = 0  # Hatalı port formatı varsa yokmuş gibi kabul et
        feature_times["contains_port"] = time.time() - start

        start = time.time()
        features['contains_login'] = 1 if 'login' in url.lower() else 0
        feature_times['contains_login'] = time.time() - start

        start = time.time()
        features['contains_bank'] = 1 if 'bank' in url.lower() else 0
        feature_times['contains_bank'] = time.time() - start

        start = time.time()
        features['contains_secure'] = 1 if 'secure' in url.lower() else 0
        feature_times['contains_secure'] = time.time() - start

        start = time.time()
        features['contains_account'] = 1 if 'account' in url.lower() else 0
        feature_times['contains_account'] = time.time() - start

        start = time.time()
        features['contains_confirm'] = 1 if 'confirm' in url.lower() else 0
        feature_times['contains_confirm'] = time.time() - start

        start = time.time()
        features['contains_token'] = 1 if 'token' in url.lower() else 0
        feature_times['contains_token'] = time.time() - start

        start = time.time()
        features['contains_free'] = 1 if 'free' in url.lower() else 0
        feature_times['contains_free'] = time.time() - start

        start = time.time()
        features['contains_win'] = 1 if 'win' in url.lower() else 0
        feature_times['contains_win'] = time.time() - start

        start = time.time()
        features['is_repeated_word'] = 1 if any(count > 1 for count in Counter(words_url).values()) else 0
        feature_times['is_repeated_word'] = time.time() - start

        start = time.time()
        features['contains_redirect'] = 1 if "//" in parsed_url.path[1:] else 0
        feature_times['contains_redirect'] = time.time() - start

        start = time.time()
        features['contains_js_code'] = 1 if 'javascript:' in parsed_url.path.lower() else 0
        feature_times['contains_js_code'] = time.time() - start

        return features, feature_times
    except Exception as e:
        print(f"Error processing URL {url}: {e}")
        return None  # Hatalı URL'leri atla