pe-malware-detector / src /chunk_model.py
adamuremek's picture
Add model code
71bf6da verified
Raw History Blame Contribute Delete
12.2 kB
"""Structure-aware chunk attention model for PE files, in PyTorch.
Idea in three steps:
1. CHUNK - read N_CHUNKS windows of CHUNK bytes, evenly spaced across the whole file
(for files up to N_CHUNKS * CHUNK bytes this is every byte).
2. ENCODE - one small convolutional network, shared by all chunks, turns each chunk into a vector.
Each chunk also gets three simple statistics and a tag for the region of the file
it comes from (header / code / data / resources / overlay).
3. ATTEND - (optional) transformer layers let chunks inform each other; then attention pooling
gives every chunk a weight (weights sum to 1) and the file summary is the weighted
average. One linear layer turns that into the malware logit.
SAFETY: files are opened read-only and parsed as bytes. Nothing is executed.
Sanity check: python src/chunk_model.py
"""
import csv
import os
import struct
import numpy as np
import torch
import torch.nn as nn
from torch.utils.data import Dataset
CHUNK = 4096 # bytes per chunk
N_CHUNKS = 256 # chunks per file (256 * 4096 = 1 MB of bytes read per file)
PAD = 0
# Region tags. 0 is reserved for "no chunk here" (file shorter than N_CHUNKS chunks).
REGIONS = ["none", "header", "code", "data", "resources", "overlay", "unknown"]
R_NONE, R_HEADER, R_CODE, R_DATA, R_RSRC, R_OVERLAY, R_UNKNOWN = range(7)
def section_map(head: bytes):
"""Read the PE section table from the start of the file.
Returns (end_of_headers, [(start, end, region), ...]) or None if the header cannot be parsed.
This only reads a table of offsets; it does not run or load anything.
"""
try:
if head[:2] != b"MZ":
return None
pe = struct.unpack_from("<I", head, 0x3C)[0]
if head[pe:pe + 4] != b"PE\0\0":
return None
n_sections = struct.unpack_from("<H", head, pe + 6)[0]
opt_size = struct.unpack_from("<H", head, pe + 20)[0]
table = pe + 24 + opt_size
sections = []
for i in range(min(n_sections, 96)):
off = table + 40 * i
name = head[off:off + 8].rstrip(b"\0")
raw_size, raw_ptr = struct.unpack_from("<II", head, off + 16)
flags = struct.unpack_from("<I", head, off + 36)[0]
if raw_size == 0:
continue
if name == b".rsrc":
region = R_RSRC
elif flags & 0x20000020: # contains code, or is executable
region = R_CODE
else:
region = R_DATA
sections.append((raw_ptr, raw_ptr + raw_size, region))
if not sections:
return None
return min(s[0] for s in sections), sections
except (struct.error, IndexError):
return None
def region_of(offset: int, smap) -> int:
"""Which region of the file does the byte at `offset` belong to?"""
if smap is None:
return R_UNKNOWN
headers_end, sections = smap
if offset < headers_end:
return R_HEADER
for start, end, region in sections:
if start <= offset < end:
return region
if offset >= max(s[1] for s in sections):
return R_OVERLAY # data appended after the last section
return R_UNKNOWN # a gap between sections
def read_chunks(path: str, n_chunks: int = N_CHUNKS, chunk: int = CHUNK):
"""Return (tokens [n, chunk], region [n], position [n], stats [n, 3]) for one file.
Chunk i starts at floor(i * size / n) when the file is larger than n * chunk bytes
(evenly spaced windows over the whole file), otherwise at i * chunk (every byte is read).
The choice is deterministic: the same file always gives the same chunks.
"""
data = np.memmap(path, dtype=np.uint8, mode="r") # bytes only, never executed
size = len(data)
tokens = np.zeros((n_chunks, chunk), dtype=np.int16) # 0 = padding
region = np.zeros(n_chunks, dtype=np.int64) # 0 = no chunk
position = np.zeros(n_chunks, dtype=np.float32)
lengths = np.zeros(n_chunks, dtype=np.int64)
smap = section_map(bytes(data[:8192]))
step = size / n_chunks if size > n_chunks * chunk else chunk
for i in range(n_chunks):
start = int(i * step)
if start >= size:
break
piece = data[start:start + chunk]
tokens[i, : len(piece)] = piece.astype(np.int16) + 1 # byte b -> token b + 1
region[i] = region_of(start, smap)
position[i] = start / size # 0.0 = start of file, 1.0 = end
lengths[i] = len(piece)
# Three cheap statistics per chunk: byte entropy (0..1), share of zero bytes, share of printable text.
stats = np.zeros((n_chunks, 3), dtype=np.float32)
used = lengths > 0
if used.any():
rows = np.arange(n_chunks)[:, None]
hist = np.bincount((tokens + rows * 257).ravel(), minlength=n_chunks * 257).reshape(n_chunks, 257)[:, 1:]
n = np.maximum(lengths, 1)[:, None]
p = hist / n
with np.errstate(divide="ignore", invalid="ignore"):
ent = -(np.where(p > 0, p * np.log2(p), 0)).sum(1) / 8.0
stats[:, 0] = ent
stats[:, 1] = hist[:, 0] / n[:, 0]
stats[:, 2] = hist[:, 32:127].sum(1) / n[:, 0]
stats[~used] = 0
return tokens, region, position, stats
class ChunkDataset(Dataset):
"""One item = a dict of chunk tokens, region tags, positions, statistics and the label."""
def __init__(self, manifest: str, data_dir: str, split: str):
self.items = []
with open(manifest, newline="") as f:
for row in csv.DictReader(f):
if row["split"] == split:
cls = "malware" if row["label"] == "1" else "benign"
self.items.append((os.path.join(data_dir, split, cls, row["sha256"]), int(row["label"])))
def __len__(self):
return len(self.items)
def __getitem__(self, i):
path, label = self.items[i]
tokens, region, position, stats = read_chunks(path)
return {"tokens": torch.from_numpy(tokens), "region": torch.from_numpy(region),
"position": torch.from_numpy(position), "stats": torch.from_numpy(stats),
"label": torch.tensor(label, dtype=torch.float32)}
class ChunkEncoder(nn.Module):
"""Turns one chunk of 4,096 byte tokens into one vector. The same weights are used for every chunk."""
def __init__(self, embed_dim=16, dim=128):
super().__init__()
self.embed = nn.Embedding(257, embed_dim, padding_idx=PAD)
# Three convolutions with small, OVERLAPPING windows (window > stride), unlike MalConv's
# single non-overlapping 500-byte window. Each layer sees a wider span of bytes:
self.convs = nn.Sequential(
nn.Conv1d(embed_dim, 64, kernel_size=8, stride=4), nn.ReLU(), # 4096 -> 1023 positions
nn.Conv1d(64, dim, kernel_size=8, stride=4), nn.ReLU(), # 1023 -> 254
nn.Conv1d(dim, dim, kernel_size=4, stride=2), nn.ReLU(), # 254 -> 126
)
self.proj = nn.Linear(2 * dim, dim)
def forward(self, tokens): # [chunks, 4096]
x = self.embed(tokens.int()).transpose(1, 2) # [chunks, 16, 4096]
x = self.convs(x) # [chunks, 128, 126]
# Summarise the chunk two ways: the average response and the strongest response.
x = torch.cat([x.mean(dim=2), x.amax(dim=2)], dim=1) # [chunks, 256]
return self.proj(x) # [chunks, 128]
class ChunkAttentionNet(nn.Module):
"""Chunk encoder -> (+stats, +region tag, +position) -> optional transformer -> attention pooling -> logit."""
def __init__(self, dim=128, use_region=True, use_stats=True, n_layers=2, n_heads=4, dropout=0.1):
super().__init__()
self.use_region, self.use_stats = use_region, use_stats
self.encoder = ChunkEncoder(dim=dim)
self.stats_proj = nn.Linear(3, dim) if use_stats else None
self.region_embed = nn.Embedding(len(REGIONS), dim, padding_idx=R_NONE) if use_region else None
self.pos_proj = nn.Linear(1, dim) # where in the file (0..1) the chunk starts
self.norm = nn.LayerNorm(dim)
self.drop = nn.Dropout(dropout)
if n_layers > 0: # chunks "talk to each other"
layer = nn.TransformerEncoderLayer(dim, n_heads, dim_feedforward=2 * dim, dropout=dropout,
batch_first=True, norm_first=True)
self.transformer = nn.TransformerEncoder(layer, n_layers, enable_nested_tensor=False)
else:
self.transformer = None
# Attention pooling: a small scorer says how important each chunk is.
self.scorer = nn.Sequential(nn.Linear(dim, dim // 2), nn.Tanh(), nn.Linear(dim // 2, 1))
self.out = nn.Sequential(nn.Linear(dim, dim), nn.ReLU(), nn.Dropout(dropout), nn.Linear(dim, 1))
def forward(self, tokens, region, position, stats, return_weights=False):
b, n, c = tokens.shape # batch, chunks per file, bytes per chunk
present = region != R_NONE # [b, n] False where the file had no chunk
x = self.encoder(tokens.reshape(b * n, c)).reshape(b, n, -1) # [b, n, dim]
x = x + self.pos_proj(position.unsqueeze(-1))
if self.use_stats:
x = x + self.stats_proj(stats)
if self.use_region:
x = x + self.region_embed(region)
x = self.drop(self.norm(x))
if self.transformer is not None:
x = self.transformer(x, src_key_padding_mask=~present)
# Attention pooling: scores -> weights that sum to 1 over the chunks that exist.
scores = self.scorer(x).squeeze(-1).float().masked_fill(~present, float("-inf"))
weights = torch.softmax(scores, dim=1) # [b, n]
summary = (weights.unsqueeze(-1) * x).sum(dim=1) # weighted average [b, dim]
logit = self.out(summary).squeeze(1) # [b]
return (logit, weights) if return_weights else logit
def count_parameters(model):
return sum(p.numel() for p in model.parameters() if p.requires_grad)
if __name__ == "__main__":
# ---- sanity checks: region parsing, shapes, parameter count, memorise 20 files ----
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
device = "cuda" if torch.cuda.is_available() else "cpu"
torch.manual_seed(0)
ds = ChunkDataset(os.path.join(root, "dataset", "manifest.csv"), os.path.join(root, "data"), "train")
benign = [i for i, (_, y) in enumerate(ds.items) if y == 0][:10]
malware = [i for i, (_, y) in enumerate(ds.items) if y == 1][:10]
items = [ds[i] for i in benign + malware]
batch = {k: torch.stack([it[k] for it in items]).to(device) for k in items[0]}
counts = torch.bincount(batch["region"].flatten().cpu(), minlength=len(REGIONS)).tolist()
print("region tags in this batch:", dict(zip(REGIONS, counts)))
model = ChunkAttentionNet().to(device)
print("trainable parameters:", f"{count_parameters(model):,}")
args = (batch["tokens"], batch["region"], batch["position"], batch["stats"])
logit, w = model(*args, return_weights=True)
print("tokens:", tuple(batch["tokens"].shape), "| logits:", tuple(logit.shape),
"| attention weights:", tuple(w.shape), "sum per file:", [round(v, 3) for v in w.sum(1)[:3].tolist()])
opt = torch.optim.AdamW(model.parameters(), lr=1e-3)
loss_fn = nn.BCEWithLogitsLoss()
for step in range(1, 61):
model.train()
loss = loss_fn(model(*args), batch["label"])
opt.zero_grad(); loss.backward(); opt.step()
if step % 20 == 0:
model.eval()
with torch.no_grad():
acc = ((model(*args) > 0).float() == batch["label"]).float().mean().item()
print(f"step {step:3d} loss {loss.item():.4f} accuracy on the 20 files {acc:.2f}")