Upload 18 files

Version_4/New Text Document.txt

@@ -0,0 +1,105 @@
+import csv
+import json
+import socket
+import random
+from urllib.parse import urlparse
+
+# ==========================================
+# DEFENDER V5 - BALANCED CSV TO JSON PIPELINE
+# ==========================================
+
+def extract_domain(raw_url):
+    """Safely extracts the domain name from a raw URL string."""
+    url = raw_url.strip()
+    if not url.startswith('http'):
+        url = 'http://' + url
+    try:
+        parsed = urlparse(url)
+        domain = parsed.netloc.split(':')[0]
+        return domain if domain else None
+    except:
+        return None
+
+def resolve_domain(domain):
+    """Attempts to resolve domain to an active IP address."""
+    try:
+        return socket.gethostbyname(domain)
+    except socket.gaierror:
+        return None
+
+def build_dataset_from_csv():
+    csv_filename = "malicious_urls.csv"  # <-- Change this if your file name is different
+    output_file = "network_telemetry.json"
+    
+    print(f"[*] Scanning {csv_filename} to extract 2500 'good' and 2500 'bad' URLs...")
+    
+    telemetry_data = []
+    bad_count = 0
+    good_count = 0
+    
+    try:
+        with open(csv_filename, mode='r', encoding='utf-8') as file:
+            reader = csv.DictReader(file)
+            
+            for row in reader:
+                # Stop parsing immediately if both limits are reached
+                if bad_count >= 2500 and good_count >= 2500:
+                    break
+                    
+                raw_url = row.get('URL', '')
+                label_str = row.get('Label', '').strip().lower()
+                
+                if not raw_url:
+                    continue
+                
+                # Check label and verify if we still need more samples for that category
+                if label_str == 'bad' and bad_count < 2500:
+                    is_malicious = 1.0
+                elif label_str == 'good' and good_count < 2500:
+                    is_malicious = 0.0
+                else:
+                    continue  # Skip row if label doesn't match or target is already full
+
+                domain = extract_domain(raw_url)
+                if not domain:
+                    continue
+                    
+                # DNS Lookup (ensures high quality topological edges for the Graph)
+                ip = resolve_domain(domain)
+                
+                if ip:
+                    telemetry_data.append({
+                        "ip": ip,
+                        "domain": domain,
+                        "asn": random.randint(40000, 60000) if is_malicious == 1.0 else 15169, # Placeholder ASNs
+                        "is_malicious": is_malicious
+                    })
+                    
+                    if is_malicious == 1.0:
+                        bad_count += 1
+                    else:
+                        good_count += 1
+                        
+                    # Progress counter (DNS resolution takes a moment)
+                    total_processed = bad_count + good_count
+                    if total_processed % 100 == 0:
+                        print(f" -> Processed {total_processed} active rows... (Good: {good_count}/2500 | Bad: {bad_count}/2500)")
+                        
+        print(f"\n[+] Extraction Complete!")
+        print(f"    - Legitimate (Good) URLs mapped: {good_count}")
+        print(f"    - Malicious (Bad) URLs mapped: {bad_count}")
+
+        # Shuffle the list so the model doesn't process all bad URLs first, then all good URLs
+        random.shuffle(telemetry_data)
+        
+        with open(output_file, "w") as f:
+            json.dump(telemetry_data, f, indent=4)
+            
+        print(f"[+] Balanced dataset successfully saved to: {output_file}")
+        print("[*] Ready to train! Run: python train.py")
+        
+    except FileNotFoundError:
+        print(f"[-] Error: Could not find '{csv_filename}' in this directory.")
+
+if __name__ == "__main__":
+    build_dataset_from_csv()

Version_4/app.py

@@ -0,0 +1,88 @@
+from fastapi import FastAPI, HTTPException
+from pydantic import BaseModel
+import torch
+import os
+import socket
+from urllib.parse import urlparse
+from models.gnn import PhishingGNN_Model
+from pipeline.graph_engine import TopologicalGraphEngine
+from config import *
+import uvicorn
+
+app = FastAPI(title="Defender V5 Sovereign Threat Analysis API")
+
+# Define node dimension configurations
+in_channels_dict = {
+    'ip': 16,
+    'domain': 32,
+    'asn': 8,
+    'cert': 16
+}
+
+# Load model directly into state memory matching training dimension maps
+model = PhishingGNN_Model(
+    metadata=GRAPH_METADATA,
+    in_channels_dict=in_channels_dict,
+    hidden_channels=HIDDEN_CHANNELS,
+    num_heads=NUM_HEADS,
+    num_layers=NUM_LAYERS,
+    dropout_rate=0.0
+)
+
+if os.path.exists(MODEL_SAVE_PATH):
+    model.load_state_dict(torch.load(MODEL_SAVE_PATH, map_location='cpu', weights_only=True))
+model.eval()
+
+# Payload model expects a URL
+class URLPayload(BaseModel):
+    url: str
+
+@app.post("/analyze")
+async def analyze_url(payload: URLPayload):
+    try:
+        # 1. Parse the URL to extract the domain
+        parsed_url = urlparse(payload.url)
+        domain = parsed_url.netloc or parsed_url.path.split('/')[0]
+        
+        if ':' in domain:
+            domain = domain.split(':')[0]
+
+        if not domain:
+            raise HTTPException(status_code=400, detail="Invalid URL format.")
+
+        # 2. Resolve the domain to an IP address
+        try:
+            ip = socket.gethostbyname(domain)
+        except socket.gaierror:
+            raise HTTPException(status_code=400, detail=f"DNS Resolution failed for domain: {domain}")
+
+        # 3. Build telemetry dictionary
+        telemetry_log = {
+            "ip": ip,
+            "domain": domain,
+            "asn": None 
+        }
+
+        # 4. Pass through Graph Engine
+        engine = TopologicalGraphEngine()
+        x_dict, edge_index_dict = engine.extract_and_build([telemetry_log])
+        
+        with torch.no_grad():
+            raw_scores = model(x_dict, edge_index_dict)
+            threat_probability = float(raw_scores.max().item())
+            
+        return {
+            "input_url": payload.url,
+            "resolved_domain": domain,
+            "resolved_ip": ip,
+            "structural_anomaly_score": round(threat_probability, 5),
+            "remediation_verdict": "ISOLATE_ROUTING" if threat_probability > 0.70 else "ALLOW"
+        }
+        
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Graph Engine Exception: {str(e)}")
+
+if __name__ == "__main__":
+    uvicorn.run(app, host="127.0.0.1", port=8005)

Version_4/build_dataset.py

@@ -0,0 +1,111 @@
+import csv
+import json
+import socket
+import random
+from urllib.parse import urlparse
+
+# ==========================================
+# DEFENDER V5 - CSV TO JSON PIPELINE (BUGFIXED)
+# ==========================================
+
+def extract_domain(raw_url):
+    """Safely extracts the domain name from a raw URL string."""
+    url = raw_url.strip()
+    if not url.startswith('http'):
+        url = 'http://' + url
+    try:
+        parsed = urlparse(url)
+        domain = parsed.netloc.split(':')[0]
+        return domain if domain else None
+    except:
+        return None
+
+def resolve_domain(domain):
+    """Attempts to resolve domain to an active IP address. Fails gracefully if offline or corrupted."""
+    try:
+        return socket.gethostbyname(domain)
+    except (socket.gaierror, UnicodeEncodeError): 
+        # BUGFIX: Added UnicodeEncodeError to prevent crashes from hidden junk characters like \x87
+        return None
+
+def build_dataset_from_csv():
+    csv_filename = "malicious_urls.csv"  
+    output_file = "network_telemetry.json"
+    
+    print(f"[*] Scanning {csv_filename} to extract 2500 'good' and 2500 'bad' URLs...")
+    
+    telemetry_data = []
+    bad_count = 0
+    good_count = 0
+    
+    try:
+        with open(csv_filename, mode='r', encoding='utf-8', errors='ignore') as file:
+            # Added errors='ignore' above to strip out any raw structural file encoding issues
+            reader = csv.DictReader(file)
+            
+            for row in reader:
+                # Stop parsing immediately if both limits are reached
+                if bad_count >= 2500 and good_count >= 2500:
+                    break
+                    
+                raw_url = row.get('URL', '')
+                label_str = row.get('Label', '').strip().lower()
+                
+                if not raw_url:
+                    continue
+                
+                # Check label and verify if we still need more samples for that category
+                if label_str == 'bad' and bad_count < 2500:
+                    is_malicious = 1.0
+                elif label_str == 'good' and good_count < 2500:
+                    is_malicious = 0.0
+                else:
+                    continue  # Skip row if label doesn't match or target category is already full
+
+                domain = extract_domain(raw_url)
+                if not domain:
+                    continue
+                    
+                # DNS Lookup
+                ip = resolve_domain(domain)
+                
+                if ip:
+                    telemetry_data.append({
+                        "ip": ip,
+                        "domain": domain,
+                        "asn": random.randint(40000, 60000) if is_malicious == 1.0 else 15169, 
+                        "is_malicious": is_malicious
+                    })
+                    
+                    if is_malicious == 1.0:
+                        bad_count += 1
+                    else:
+                        good_count += 1
+                        
+                    # Progress counter
+                    total_processed = bad_count + good_count
+                    if total_processed % 100 == 0 or (bad_count == 2500 and total_processed % 10 == 0):
+                        print(f" -> Processed {total_processed} active rows... (Good: {good_count}/2500 | Bad: {bad_count}/2500)")
+                        
+        print(f"\n[+] Extraction Complete!")
+        print(f"    - Legitimate (Good) URLs mapped: {good_count}")
+        print(f"    - Malicious (Bad) URLs mapped: {bad_count}")
+
+        if len(telemetry_data) == 0:
+            print("[-] Error: No active domains were resolved. Check internet connection or CSV headers.")
+            return
+
+        # Shuffle the list so the model mixes good and bad data
+        random.shuffle(telemetry_data)
+        
+        with open(output_file, "w") as f:
+            json.dump(telemetry_data, f, indent=4)
+            
+        print(f"[+] Balanced dataset successfully saved to: {output_file}")
+        print("[*] Ready to train! Run: python train.py")
+        
+    except FileNotFoundError:
+        print(f"[-] Error: Could not find '{csv_filename}' in this directory.")
+
+if __name__ == "__main__":
+    build_dataset_from_csv()

Version_4/checkpoints/phishing_gnn_master.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9d89a611d4655e4d430f60432caa2b7d600587dbf385dd14fdfa992412d11cb
+size 4696055

Version_4/config.py

@@ -0,0 +1,26 @@
+import os
+
+# ==========================================
+# DEFENDER V5 - GLOBAL CONFIGURATION
+# ==========================================
+
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+CHECKPOINT_DIR = os.path.join(BASE_DIR, "checkpoints")
+MODEL_SAVE_PATH = os.path.join(CHECKPOINT_DIR, "phishing_gnn_master.pt")
+
+# Topology Definition
+NODE_TYPES = ['ip', 'domain', 'asn', 'cert']
+EDGE_TYPES = [
+    ('domain', 'resolves_to', 'ip'),
+    ('ip', 'hosted_on', 'asn'),
+    ('domain', 'secured_by', 'cert'),
+    ('cert', 'issued_to', 'ip'),
+    ('domain', 'redirects_to', 'domain')
+]
+GRAPH_METADATA = (NODE_TYPES, EDGE_TYPES)
+
+# Model Deep-Stack Hyperparameters
+HIDDEN_CHANNELS = 128
+NUM_HEADS = 8
+NUM_LAYERS = 4
+DROPOUT_RATE = 0.15

Version_4/models/gnn.py

@@ -0,0 +1,72 @@
+import os
+import torch
+import torch.nn.functional as F
+from torch.nn import ModuleDict, ModuleList, Linear, LayerNorm, Dropout
+from torch_geometric.nn import HGTConv
+from typing import Dict, Tuple, List
+
+class PhishingGNN_Model(torch.nn.Module):
+    """
+    Standalone Heterogeneous Graph Transformer (HGT) for pure topological threat analysis.
+    Processes multi-relational network metadata independently of lexical features.
+    """
+    def __init__(
+        self, 
+        metadata: Tuple[List[str], List[Tuple[str, str, str]]], 
+        in_channels_dict: Dict[str, int],  # Maps node types to their specific input sizes
+        hidden_channels: int, 
+        num_heads: int, 
+        num_layers: int, 
+        dropout_rate: float
+    ):
+        super().__init__()
+        self.node_types, self.edge_types = metadata
+        self.num_layers = num_layers
+        
+        # Multi-type input embedding alignment using explicit feature dimensions
+        self.inputs_proj = ModuleDict({
+            node_type: Linear(in_channels_dict[node_type], hidden_channels) 
+            for node_type in self.node_types
+        })
+            
+        self.convs = ModuleList()
+        self.layer_norms = ModuleList()
+        self.dropouts = ModuleList()
+        
+        for _ in range(num_layers):
+            # No 'group' parameter here to ensure PyG compatibility
+            self.convs.append(HGTConv(hidden_channels, hidden_channels, metadata, num_heads))
+            self.layer_norms.append(ModuleDict({
+                node_type: LayerNorm(hidden_channels) for node_type in self.node_types
+            }))
+            self.dropouts.append(Dropout(p=dropout_rate))
+            
+        # Advanced classification projection head for targeting IP anomalies
+        self.classifier = torch.nn.Sequential(
+            Linear(hidden_channels, hidden_channels // 2),
+            torch.nn.GELU(),
+            Dropout(p=dropout_rate),
+            Linear(hidden_channels // 2, 1)
+        )
+
+    def forward(self, x_dict: Dict[str, torch.Tensor], edge_index_dict: Dict[Tuple[str, str, str], torch.Tensor]) -> torch.Tensor:
+        # Align input spaces
+        h_dict = {node_type: F.gelu(self.inputs_proj[node_type](x)) for node_type, x in x_dict.items()}
+        
+        # Deep graph convolutions with normalization and skip connections
+        for i in range(self.num_layers):
+            h_residual_dict = {node_type: h.clone() for node_type, h in h_dict.items()}
+            h_dict = self.convs[i](h_dict, edge_index_dict)
+            
+            for node_type in self.node_types:
+                h = self.dropouts[i](h_dict[node_type])
+                h = h + h_residual_dict[node_type]
+                h_dict[node_type] = self.layer_norms[i][node_type](h)
+                
+        # Isolate the 'ip' node matrix for final threat probability generation
+        return torch.sigmoid(self.classifier(h_dict['ip']))
+
+    def safe_save(self, path: str):
+        """Ensures parent directories exist before execution to prevent saving crashes."""
+        os.makedirs(os.path.dirname(path), exist_ok=True)
+        torch.save(self.state_dict(), path)

Version_4/pipeline/graph_engine.py

@@ -0,0 +1,47 @@
+import torch
+from typing import Dict, List, Tuple
+
+class TopologicalGraphEngine:
+    """Translates incoming raw telemetry logs into PyTorch Geometric HeteroData formats."""
+    def __init__(self):
+        self.ip_map, self.domain_map, self.asn_map = {}, {}, {}
+
+    def extract_and_build(self, telemetries: List[dict]) -> Tuple[Dict[str, torch.Tensor], Dict[Tuple, torch.Tensor]]:
+        # Map dynamic components to index pointers
+        for log in telemetries:
+            if log.get('ip') and log['ip'] not in self.ip_map:
+                self.ip_map[log['ip']] = len(self.ip_map)
+            if log.get('domain') and log['domain'] not in self.domain_map:
+                self.domain_map[log['domain']] = len(self.domain_map)
+            if log.get('asn') and log['asn'] not in self.asn_map:
+                self.asn_map[log['asn']] = len(self.asn_map)
+
+        # High-dimensional hidden state initializers (using structural placeholders)
+        # Note: These sizes must match in_channels_dict in app.py and train.py
+        x_dict = {
+            'ip': torch.randn((max(1, len(self.ip_map)), 16)),
+            'domain': torch.randn((max(1, len(self.domain_map)), 32)),
+            'asn': torch.randn((max(1, len(self.asn_map)), 8)),
+            'cert': torch.randn((1, 16))
+        }
+
+        domain_to_ip = [[], []]
+        ip_to_asn = [[], []]
+
+        for log in telemetries:
+            if log.get('domain') and log.get('ip'):
+                domain_to_ip[0].append(self.domain_map[log['domain']])
+                domain_to_ip[1].append(self.ip_map[log['ip']])
+            if log.get('ip') and log.get('asn'):
+                ip_to_asn[0].append(self.ip_map[log['ip']])
+                ip_to_asn[1].append(self.asn_map[log['asn']])
+
+        edge_index_dict = {
+            ('domain', 'resolves_to', 'ip'): torch.tensor(domain_to_ip, dtype=torch.long),
+            ('ip', 'hosted_on', 'asn'): torch.tensor(ip_to_asn, dtype=torch.long),
+            ('domain', 'secured_by', 'cert'): torch.empty((2, 0), dtype=torch.long),
+            ('cert', 'issued_to', 'ip'): torch.empty((2, 0), dtype=torch.long),
+            ('domain', 'redirects_to', 'domain'): torch.empty((2, 0), dtype=torch.long)
+        }
+
+        return x_dict, edge_index_dict

Version_4/requirements.txt

@@ -0,0 +1,5 @@
+fastapi
+pydantic
+uvicorn
+torch
+torch-geometric

Version_4/simulate_warfare.py

@@ -0,0 +1,70 @@
+import torch
+import torch.nn as nn
+import torch.optim as optim
+import random
+from models.gnn import PhishingGNN_Model
+from pipeline.graph_engine import TopologicalGraphEngine
+from config import *
+
+def generate_synthetic_warfare(batch_size=100):
+    """Generates synthetic network logs simulating both normal traffic and advanced attacks."""
+    logs = []
+    labels = []
+    
+    for _ in range(batch_size):
+        is_attack = random.random() > 0.5
+        
+        if is_attack:
+            logs.append({
+                'ip': f"{random.randint(1, 255)}.{random.randint(1,255)}.0.0",
+                'domain': None, 
+                'asn': random.choice([666, 9999, 5555]) 
+            })
+            labels.append([1.0]) 
+        else:
+            logs.append({
+                'ip': f"104.21.{random.randint(1,100)}.{random.randint(1,255)}",
+                'domain': f"safe-service-{random.randint(1,100)}.com",
+                'asn': 13335 
+            })
+            labels.append([0.0]) 
+            
+    return logs, torch.tensor(labels, dtype=torch.float32)
+
+def run_war_games():
+    print("[*] Initiating Autonomous Threat War Games...")
+    
+    in_channels_dict = {'ip': 16, 'domain': 32, 'asn': 8, 'cert': 16}
+    model = PhishingGNN_Model(
+        metadata=GRAPH_METADATA,
+        in_channels_dict=in_channels_dict,
+        hidden_channels=HIDDEN_CHANNELS,
+        num_heads=NUM_HEADS,
+        num_layers=NUM_LAYERS,
+        dropout_rate=DROPOUT_RATE
+    )
+    
+    optimizer = optim.AdamW(model.parameters(), lr=0.001)
+    criterion = nn.BCELoss()
+    engine = TopologicalGraphEngine()
+
+    for wave in range(50):
+        logs, labels = generate_synthetic_warfare(batch_size=200)
+        x_dict, edge_index_dict = engine.extract_and_build(logs)
+        
+        optimizer.zero_grad()
+        predictions = model(x_dict, edge_index_dict)
+        valid_preds = predictions[:len(labels)] 
+        
+        loss = criterion(valid_preds, labels)
+        loss.backward()
+        optimizer.step()
+        
+        if (wave + 1) % 10 == 0:
+            print(f"Attack Wave {wave+1:02d}/50 Defeated | Model Penetration Loss: {loss.item():.4f}")
+
+    model.safe_save(MODEL_SAVE_PATH)
+    print(f"[+] War games complete. Apex model hardened and saved to: {MODEL_SAVE_PATH}")
+
+if __name__ == "__main__":
+    run_war_games()

Version_4/train.py

@@ -0,0 +1,71 @@
+import json
+import torch
+import torch.nn as nn
+import torch.optim as optim
+from models.gnn import PhishingGNN_Model
+from pipeline.graph_engine import TopologicalGraphEngine
+from config import *
+
+def run_training_pipeline():
+    print("[*] Launching Production Defender V5 Training Run...")
+    
+    # 1. Try to load real data, fallback to mock if missing
+    try:
+        with open("network_telemetry.json", "r") as f:
+            dataset = json.load(f)
+            print(f"[+] Loaded {len(dataset)} real telemetry logs.")
+    except FileNotFoundError:
+        print("[-] network_telemetry.json not found. Falling back to simulated context.")
+        dataset = [
+            {'ip': '185.220.101.5', 'domain': 'shadow-dns-bypass.net', 'asn': 44050, 'is_malicious': 1.0},
+            {'ip': '193.56.28.14', 'asn': 57099, 'is_malicious': 1.0},
+            {'ip': '8.8.8.8', 'domain': 'dns.google', 'asn': 15169, 'is_malicious': 0.0}
+        ]
+
+    # 2. Extract logs and structure target threat labels
+    engine = TopologicalGraphEngine()
+    x_dict, edge_index_dict = engine.extract_and_build(dataset)
+    
+    labels_list = []
+    for ip_str in engine.ip_map.keys():
+        matching_logs = [log for log in dataset if log.get('ip') == ip_str]
+        label = matching_logs[0].get('is_malicious', 0.0) if matching_logs else 0.0
+        labels_list.append([label])
+        
+    labels = torch.tensor(labels_list, dtype=torch.float32)
+
+    # 3. Model setup
+    in_channels_dict = {'ip': 16, 'domain': 32, 'asn': 8, 'cert': 16}
+    model = PhishingGNN_Model(
+        metadata=GRAPH_METADATA,
+        in_channels_dict=in_channels_dict,
+        hidden_channels=HIDDEN_CHANNELS,
+        num_heads=NUM_HEADS,
+        num_layers=NUM_LAYERS,
+        dropout_rate=DROPOUT_RATE
+    )
+    
+    optimizer = optim.AdamW(model.parameters(), lr=0.0005, weight_decay=1e-3)
+    criterion = nn.BCELoss()
+
+    # 4. Optimization Loop
+    model.train()
+    for epoch in range(100):
+        optimizer.zero_grad()
+        predictions = model(x_dict, edge_index_dict)
+        
+        # Only calculate loss on the nodes we have labels for
+        valid_preds = predictions[:len(labels)]
+        loss = criterion(valid_preds, labels)
+        
+        loss.backward()
+        optimizer.step()
+        
+        if (epoch + 1) % 20 == 0:
+            print(f"Epoch {epoch+1:03d}/100 | Topological Loss: {loss.item():.5f}")
+
+    model.safe_save(MODEL_SAVE_PATH)
+    print(f"[+] Operational weights successfully frozen at: {MODEL_SAVE_PATH}")
+
+if __name__ == "__main__":
+    run_training_pipeline()