"""Structure-aware chunk attention model for PE files, in PyTorch. Idea in three steps: 1. CHUNK - read N_CHUNKS windows of CHUNK bytes, evenly spaced across the whole file (for files up to N_CHUNKS * CHUNK bytes this is every byte). 2. ENCODE - one small convolutional network, shared by all chunks, turns each chunk into a vector. Each chunk also gets three simple statistics and a tag for the region of the file it comes from (header / code / data / resources / overlay). 3. ATTEND - (optional) transformer layers let chunks inform each other; then attention pooling gives every chunk a weight (weights sum to 1) and the file summary is the weighted average. One linear layer turns that into the malware logit. SAFETY: files are opened read-only and parsed as bytes. Nothing is executed. Sanity check: python src/chunk_model.py """ import csv import os import struct import numpy as np import torch import torch.nn as nn from torch.utils.data import Dataset CHUNK = 4096 # bytes per chunk N_CHUNKS = 256 # chunks per file (256 * 4096 = 1 MB of bytes read per file) PAD = 0 # Region tags. 0 is reserved for "no chunk here" (file shorter than N_CHUNKS chunks). REGIONS = ["none", "header", "code", "data", "resources", "overlay", "unknown"] R_NONE, R_HEADER, R_CODE, R_DATA, R_RSRC, R_OVERLAY, R_UNKNOWN = range(7) def section_map(head: bytes): """Read the PE section table from the start of the file. Returns (end_of_headers, [(start, end, region), ...]) or None if the header cannot be parsed. This only reads a table of offsets; it does not run or load anything. """ try: if head[:2] != b"MZ": return None pe = struct.unpack_from(" int: """Which region of the file does the byte at `offset` belong to?""" if smap is None: return R_UNKNOWN headers_end, sections = smap if offset < headers_end: return R_HEADER for start, end, region in sections: if start <= offset < end: return region if offset >= max(s[1] for s in sections): return R_OVERLAY # data appended after the last section return R_UNKNOWN # a gap between sections def read_chunks(path: str, n_chunks: int = N_CHUNKS, chunk: int = CHUNK): """Return (tokens [n, chunk], region [n], position [n], stats [n, 3]) for one file. Chunk i starts at floor(i * size / n) when the file is larger than n * chunk bytes (evenly spaced windows over the whole file), otherwise at i * chunk (every byte is read). The choice is deterministic: the same file always gives the same chunks. """ data = np.memmap(path, dtype=np.uint8, mode="r") # bytes only, never executed size = len(data) tokens = np.zeros((n_chunks, chunk), dtype=np.int16) # 0 = padding region = np.zeros(n_chunks, dtype=np.int64) # 0 = no chunk position = np.zeros(n_chunks, dtype=np.float32) lengths = np.zeros(n_chunks, dtype=np.int64) smap = section_map(bytes(data[:8192])) step = size / n_chunks if size > n_chunks * chunk else chunk for i in range(n_chunks): start = int(i * step) if start >= size: break piece = data[start:start + chunk] tokens[i, : len(piece)] = piece.astype(np.int16) + 1 # byte b -> token b + 1 region[i] = region_of(start, smap) position[i] = start / size # 0.0 = start of file, 1.0 = end lengths[i] = len(piece) # Three cheap statistics per chunk: byte entropy (0..1), share of zero bytes, share of printable text. stats = np.zeros((n_chunks, 3), dtype=np.float32) used = lengths > 0 if used.any(): rows = np.arange(n_chunks)[:, None] hist = np.bincount((tokens + rows * 257).ravel(), minlength=n_chunks * 257).reshape(n_chunks, 257)[:, 1:] n = np.maximum(lengths, 1)[:, None] p = hist / n with np.errstate(divide="ignore", invalid="ignore"): ent = -(np.where(p > 0, p * np.log2(p), 0)).sum(1) / 8.0 stats[:, 0] = ent stats[:, 1] = hist[:, 0] / n[:, 0] stats[:, 2] = hist[:, 32:127].sum(1) / n[:, 0] stats[~used] = 0 return tokens, region, position, stats class ChunkDataset(Dataset): """One item = a dict of chunk tokens, region tags, positions, statistics and the label.""" def __init__(self, manifest: str, data_dir: str, split: str): self.items = [] with open(manifest, newline="") as f: for row in csv.DictReader(f): if row["split"] == split: cls = "malware" if row["label"] == "1" else "benign" self.items.append((os.path.join(data_dir, split, cls, row["sha256"]), int(row["label"]))) def __len__(self): return len(self.items) def __getitem__(self, i): path, label = self.items[i] tokens, region, position, stats = read_chunks(path) return {"tokens": torch.from_numpy(tokens), "region": torch.from_numpy(region), "position": torch.from_numpy(position), "stats": torch.from_numpy(stats), "label": torch.tensor(label, dtype=torch.float32)} class ChunkEncoder(nn.Module): """Turns one chunk of 4,096 byte tokens into one vector. The same weights are used for every chunk.""" def __init__(self, embed_dim=16, dim=128): super().__init__() self.embed = nn.Embedding(257, embed_dim, padding_idx=PAD) # Three convolutions with small, OVERLAPPING windows (window > stride), unlike MalConv's # single non-overlapping 500-byte window. Each layer sees a wider span of bytes: self.convs = nn.Sequential( nn.Conv1d(embed_dim, 64, kernel_size=8, stride=4), nn.ReLU(), # 4096 -> 1023 positions nn.Conv1d(64, dim, kernel_size=8, stride=4), nn.ReLU(), # 1023 -> 254 nn.Conv1d(dim, dim, kernel_size=4, stride=2), nn.ReLU(), # 254 -> 126 ) self.proj = nn.Linear(2 * dim, dim) def forward(self, tokens): # [chunks, 4096] x = self.embed(tokens.int()).transpose(1, 2) # [chunks, 16, 4096] x = self.convs(x) # [chunks, 128, 126] # Summarise the chunk two ways: the average response and the strongest response. x = torch.cat([x.mean(dim=2), x.amax(dim=2)], dim=1) # [chunks, 256] return self.proj(x) # [chunks, 128] class ChunkAttentionNet(nn.Module): """Chunk encoder -> (+stats, +region tag, +position) -> optional transformer -> attention pooling -> logit.""" def __init__(self, dim=128, use_region=True, use_stats=True, n_layers=2, n_heads=4, dropout=0.1): super().__init__() self.use_region, self.use_stats = use_region, use_stats self.encoder = ChunkEncoder(dim=dim) self.stats_proj = nn.Linear(3, dim) if use_stats else None self.region_embed = nn.Embedding(len(REGIONS), dim, padding_idx=R_NONE) if use_region else None self.pos_proj = nn.Linear(1, dim) # where in the file (0..1) the chunk starts self.norm = nn.LayerNorm(dim) self.drop = nn.Dropout(dropout) if n_layers > 0: # chunks "talk to each other" layer = nn.TransformerEncoderLayer(dim, n_heads, dim_feedforward=2 * dim, dropout=dropout, batch_first=True, norm_first=True) self.transformer = nn.TransformerEncoder(layer, n_layers, enable_nested_tensor=False) else: self.transformer = None # Attention pooling: a small scorer says how important each chunk is. self.scorer = nn.Sequential(nn.Linear(dim, dim // 2), nn.Tanh(), nn.Linear(dim // 2, 1)) self.out = nn.Sequential(nn.Linear(dim, dim), nn.ReLU(), nn.Dropout(dropout), nn.Linear(dim, 1)) def forward(self, tokens, region, position, stats, return_weights=False): b, n, c = tokens.shape # batch, chunks per file, bytes per chunk present = region != R_NONE # [b, n] False where the file had no chunk x = self.encoder(tokens.reshape(b * n, c)).reshape(b, n, -1) # [b, n, dim] x = x + self.pos_proj(position.unsqueeze(-1)) if self.use_stats: x = x + self.stats_proj(stats) if self.use_region: x = x + self.region_embed(region) x = self.drop(self.norm(x)) if self.transformer is not None: x = self.transformer(x, src_key_padding_mask=~present) # Attention pooling: scores -> weights that sum to 1 over the chunks that exist. scores = self.scorer(x).squeeze(-1).float().masked_fill(~present, float("-inf")) weights = torch.softmax(scores, dim=1) # [b, n] summary = (weights.unsqueeze(-1) * x).sum(dim=1) # weighted average [b, dim] logit = self.out(summary).squeeze(1) # [b] return (logit, weights) if return_weights else logit def count_parameters(model): return sum(p.numel() for p in model.parameters() if p.requires_grad) if __name__ == "__main__": # ---- sanity checks: region parsing, shapes, parameter count, memorise 20 files ---- root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) device = "cuda" if torch.cuda.is_available() else "cpu" torch.manual_seed(0) ds = ChunkDataset(os.path.join(root, "dataset", "manifest.csv"), os.path.join(root, "data"), "train") benign = [i for i, (_, y) in enumerate(ds.items) if y == 0][:10] malware = [i for i, (_, y) in enumerate(ds.items) if y == 1][:10] items = [ds[i] for i in benign + malware] batch = {k: torch.stack([it[k] for it in items]).to(device) for k in items[0]} counts = torch.bincount(batch["region"].flatten().cpu(), minlength=len(REGIONS)).tolist() print("region tags in this batch:", dict(zip(REGIONS, counts))) model = ChunkAttentionNet().to(device) print("trainable parameters:", f"{count_parameters(model):,}") args = (batch["tokens"], batch["region"], batch["position"], batch["stats"]) logit, w = model(*args, return_weights=True) print("tokens:", tuple(batch["tokens"].shape), "| logits:", tuple(logit.shape), "| attention weights:", tuple(w.shape), "sum per file:", [round(v, 3) for v in w.sum(1)[:3].tolist()]) opt = torch.optim.AdamW(model.parameters(), lr=1e-3) loss_fn = nn.BCEWithLogitsLoss() for step in range(1, 61): model.train() loss = loss_fn(model(*args), batch["label"]) opt.zero_grad(); loss.backward(); opt.step() if step % 20 == 0: model.eval() with torch.no_grad(): acc = ((model(*args) > 0).float() == batch["label"]).float().mean().item() print(f"step {step:3d} loss {loss.item():.4f} accuracy on the 20 files {acc:.2f}")