Download src/chunk_model.py from adamuremek/pe-malware-detector: direct link, hf CLI and curl.
- Browser
- Download file 12.2 kB
-
https://huggingface.co/adamuremek/pe-malware-detector/resolve/main/src/chunk_model.py
- Command line
-
hf download hf://adamuremek/pe-malware-detector/src/chunk_model.py
-
curl -L -o chunk_model.py https://huggingface.co/adamuremek/pe-malware-detector/resolve/main/src/chunk_model.py
12.2 kB
| """Structure-aware chunk attention model for PE files, in PyTorch. | |
| Idea in three steps: | |
| 1. CHUNK - read N_CHUNKS windows of CHUNK bytes, evenly spaced across the whole file | |
| (for files up to N_CHUNKS * CHUNK bytes this is every byte). | |
| 2. ENCODE - one small convolutional network, shared by all chunks, turns each chunk into a vector. | |
| Each chunk also gets three simple statistics and a tag for the region of the file | |
| it comes from (header / code / data / resources / overlay). | |
| 3. ATTEND - (optional) transformer layers let chunks inform each other; then attention pooling | |
| gives every chunk a weight (weights sum to 1) and the file summary is the weighted | |
| average. One linear layer turns that into the malware logit. | |
| SAFETY: files are opened read-only and parsed as bytes. Nothing is executed. | |
| Sanity check: python src/chunk_model.py | |
| """ | |
| import csv | |
| import os | |
| import struct | |
| import numpy as np | |
| import torch | |
| import torch.nn as nn | |
| from torch.utils.data import Dataset | |
| CHUNK = 4096 # bytes per chunk | |
| N_CHUNKS = 256 # chunks per file (256 * 4096 = 1 MB of bytes read per file) | |
| PAD = 0 | |
| # Region tags. 0 is reserved for "no chunk here" (file shorter than N_CHUNKS chunks). | |
| REGIONS = ["none", "header", "code", "data", "resources", "overlay", "unknown"] | |
| R_NONE, R_HEADER, R_CODE, R_DATA, R_RSRC, R_OVERLAY, R_UNKNOWN = range(7) | |
| def section_map(head: bytes): | |
| """Read the PE section table from the start of the file. | |
| Returns (end_of_headers, [(start, end, region), ...]) or None if the header cannot be parsed. | |
| This only reads a table of offsets; it does not run or load anything. | |
| """ | |
| try: | |
| if head[:2] != b"MZ": | |
| return None | |
| pe = struct.unpack_from("<I", head, 0x3C)[0] | |
| if head[pe:pe + 4] != b"PE\0\0": | |
| return None | |
| n_sections = struct.unpack_from("<H", head, pe + 6)[0] | |
| opt_size = struct.unpack_from("<H", head, pe + 20)[0] | |
| table = pe + 24 + opt_size | |
| sections = [] | |
| for i in range(min(n_sections, 96)): | |
| off = table + 40 * i | |
| name = head[off:off + 8].rstrip(b"\0") | |
| raw_size, raw_ptr = struct.unpack_from("<II", head, off + 16) | |
| flags = struct.unpack_from("<I", head, off + 36)[0] | |
| if raw_size == 0: | |
| continue | |
| if name == b".rsrc": | |
| region = R_RSRC | |
| elif flags & 0x20000020: # contains code, or is executable | |
| region = R_CODE | |
| else: | |
| region = R_DATA | |
| sections.append((raw_ptr, raw_ptr + raw_size, region)) | |
| if not sections: | |
| return None | |
| return min(s[0] for s in sections), sections | |
| except (struct.error, IndexError): | |
| return None | |
| def region_of(offset: int, smap) -> int: | |
| """Which region of the file does the byte at `offset` belong to?""" | |
| if smap is None: | |
| return R_UNKNOWN | |
| headers_end, sections = smap | |
| if offset < headers_end: | |
| return R_HEADER | |
| for start, end, region in sections: | |
| if start <= offset < end: | |
| return region | |
| if offset >= max(s[1] for s in sections): | |
| return R_OVERLAY # data appended after the last section | |
| return R_UNKNOWN # a gap between sections | |
| def read_chunks(path: str, n_chunks: int = N_CHUNKS, chunk: int = CHUNK): | |
| """Return (tokens [n, chunk], region [n], position [n], stats [n, 3]) for one file. | |
| Chunk i starts at floor(i * size / n) when the file is larger than n * chunk bytes | |
| (evenly spaced windows over the whole file), otherwise at i * chunk (every byte is read). | |
| The choice is deterministic: the same file always gives the same chunks. | |
| """ | |
| data = np.memmap(path, dtype=np.uint8, mode="r") # bytes only, never executed | |
| size = len(data) | |
| tokens = np.zeros((n_chunks, chunk), dtype=np.int16) # 0 = padding | |
| region = np.zeros(n_chunks, dtype=np.int64) # 0 = no chunk | |
| position = np.zeros(n_chunks, dtype=np.float32) | |
| lengths = np.zeros(n_chunks, dtype=np.int64) | |
| smap = section_map(bytes(data[:8192])) | |
| step = size / n_chunks if size > n_chunks * chunk else chunk | |
| for i in range(n_chunks): | |
| start = int(i * step) | |
| if start >= size: | |
| break | |
| piece = data[start:start + chunk] | |
| tokens[i, : len(piece)] = piece.astype(np.int16) + 1 # byte b -> token b + 1 | |
| region[i] = region_of(start, smap) | |
| position[i] = start / size # 0.0 = start of file, 1.0 = end | |
| lengths[i] = len(piece) | |
| # Three cheap statistics per chunk: byte entropy (0..1), share of zero bytes, share of printable text. | |
| stats = np.zeros((n_chunks, 3), dtype=np.float32) | |
| used = lengths > 0 | |
| if used.any(): | |
| rows = np.arange(n_chunks)[:, None] | |
| hist = np.bincount((tokens + rows * 257).ravel(), minlength=n_chunks * 257).reshape(n_chunks, 257)[:, 1:] | |
| n = np.maximum(lengths, 1)[:, None] | |
| p = hist / n | |
| with np.errstate(divide="ignore", invalid="ignore"): | |
| ent = -(np.where(p > 0, p * np.log2(p), 0)).sum(1) / 8.0 | |
| stats[:, 0] = ent | |
| stats[:, 1] = hist[:, 0] / n[:, 0] | |
| stats[:, 2] = hist[:, 32:127].sum(1) / n[:, 0] | |
| stats[~used] = 0 | |
| return tokens, region, position, stats | |
| class ChunkDataset(Dataset): | |
| """One item = a dict of chunk tokens, region tags, positions, statistics and the label.""" | |
| def __init__(self, manifest: str, data_dir: str, split: str): | |
| self.items = [] | |
| with open(manifest, newline="") as f: | |
| for row in csv.DictReader(f): | |
| if row["split"] == split: | |
| cls = "malware" if row["label"] == "1" else "benign" | |
| self.items.append((os.path.join(data_dir, split, cls, row["sha256"]), int(row["label"]))) | |
| def __len__(self): | |
| return len(self.items) | |
| def __getitem__(self, i): | |
| path, label = self.items[i] | |
| tokens, region, position, stats = read_chunks(path) | |
| return {"tokens": torch.from_numpy(tokens), "region": torch.from_numpy(region), | |
| "position": torch.from_numpy(position), "stats": torch.from_numpy(stats), | |
| "label": torch.tensor(label, dtype=torch.float32)} | |
| class ChunkEncoder(nn.Module): | |
| """Turns one chunk of 4,096 byte tokens into one vector. The same weights are used for every chunk.""" | |
| def __init__(self, embed_dim=16, dim=128): | |
| super().__init__() | |
| self.embed = nn.Embedding(257, embed_dim, padding_idx=PAD) | |
| # Three convolutions with small, OVERLAPPING windows (window > stride), unlike MalConv's | |
| # single non-overlapping 500-byte window. Each layer sees a wider span of bytes: | |
| self.convs = nn.Sequential( | |
| nn.Conv1d(embed_dim, 64, kernel_size=8, stride=4), nn.ReLU(), # 4096 -> 1023 positions | |
| nn.Conv1d(64, dim, kernel_size=8, stride=4), nn.ReLU(), # 1023 -> 254 | |
| nn.Conv1d(dim, dim, kernel_size=4, stride=2), nn.ReLU(), # 254 -> 126 | |
| ) | |
| self.proj = nn.Linear(2 * dim, dim) | |
| def forward(self, tokens): # [chunks, 4096] | |
| x = self.embed(tokens.int()).transpose(1, 2) # [chunks, 16, 4096] | |
| x = self.convs(x) # [chunks, 128, 126] | |
| # Summarise the chunk two ways: the average response and the strongest response. | |
| x = torch.cat([x.mean(dim=2), x.amax(dim=2)], dim=1) # [chunks, 256] | |
| return self.proj(x) # [chunks, 128] | |
| class ChunkAttentionNet(nn.Module): | |
| """Chunk encoder -> (+stats, +region tag, +position) -> optional transformer -> attention pooling -> logit.""" | |
| def __init__(self, dim=128, use_region=True, use_stats=True, n_layers=2, n_heads=4, dropout=0.1): | |
| super().__init__() | |
| self.use_region, self.use_stats = use_region, use_stats | |
| self.encoder = ChunkEncoder(dim=dim) | |
| self.stats_proj = nn.Linear(3, dim) if use_stats else None | |
| self.region_embed = nn.Embedding(len(REGIONS), dim, padding_idx=R_NONE) if use_region else None | |
| self.pos_proj = nn.Linear(1, dim) # where in the file (0..1) the chunk starts | |
| self.norm = nn.LayerNorm(dim) | |
| self.drop = nn.Dropout(dropout) | |
| if n_layers > 0: # chunks "talk to each other" | |
| layer = nn.TransformerEncoderLayer(dim, n_heads, dim_feedforward=2 * dim, dropout=dropout, | |
| batch_first=True, norm_first=True) | |
| self.transformer = nn.TransformerEncoder(layer, n_layers, enable_nested_tensor=False) | |
| else: | |
| self.transformer = None | |
| # Attention pooling: a small scorer says how important each chunk is. | |
| self.scorer = nn.Sequential(nn.Linear(dim, dim // 2), nn.Tanh(), nn.Linear(dim // 2, 1)) | |
| self.out = nn.Sequential(nn.Linear(dim, dim), nn.ReLU(), nn.Dropout(dropout), nn.Linear(dim, 1)) | |
| def forward(self, tokens, region, position, stats, return_weights=False): | |
| b, n, c = tokens.shape # batch, chunks per file, bytes per chunk | |
| present = region != R_NONE # [b, n] False where the file had no chunk | |
| x = self.encoder(tokens.reshape(b * n, c)).reshape(b, n, -1) # [b, n, dim] | |
| x = x + self.pos_proj(position.unsqueeze(-1)) | |
| if self.use_stats: | |
| x = x + self.stats_proj(stats) | |
| if self.use_region: | |
| x = x + self.region_embed(region) | |
| x = self.drop(self.norm(x)) | |
| if self.transformer is not None: | |
| x = self.transformer(x, src_key_padding_mask=~present) | |
| # Attention pooling: scores -> weights that sum to 1 over the chunks that exist. | |
| scores = self.scorer(x).squeeze(-1).float().masked_fill(~present, float("-inf")) | |
| weights = torch.softmax(scores, dim=1) # [b, n] | |
| summary = (weights.unsqueeze(-1) * x).sum(dim=1) # weighted average [b, dim] | |
| logit = self.out(summary).squeeze(1) # [b] | |
| return (logit, weights) if return_weights else logit | |
| def count_parameters(model): | |
| return sum(p.numel() for p in model.parameters() if p.requires_grad) | |
| if __name__ == "__main__": | |
| # ---- sanity checks: region parsing, shapes, parameter count, memorise 20 files ---- | |
| root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) | |
| device = "cuda" if torch.cuda.is_available() else "cpu" | |
| torch.manual_seed(0) | |
| ds = ChunkDataset(os.path.join(root, "dataset", "manifest.csv"), os.path.join(root, "data"), "train") | |
| benign = [i for i, (_, y) in enumerate(ds.items) if y == 0][:10] | |
| malware = [i for i, (_, y) in enumerate(ds.items) if y == 1][:10] | |
| items = [ds[i] for i in benign + malware] | |
| batch = {k: torch.stack([it[k] for it in items]).to(device) for k in items[0]} | |
| counts = torch.bincount(batch["region"].flatten().cpu(), minlength=len(REGIONS)).tolist() | |
| print("region tags in this batch:", dict(zip(REGIONS, counts))) | |
| model = ChunkAttentionNet().to(device) | |
| print("trainable parameters:", f"{count_parameters(model):,}") | |
| args = (batch["tokens"], batch["region"], batch["position"], batch["stats"]) | |
| logit, w = model(*args, return_weights=True) | |
| print("tokens:", tuple(batch["tokens"].shape), "| logits:", tuple(logit.shape), | |
| "| attention weights:", tuple(w.shape), "sum per file:", [round(v, 3) for v in w.sum(1)[:3].tolist()]) | |
| opt = torch.optim.AdamW(model.parameters(), lr=1e-3) | |
| loss_fn = nn.BCEWithLogitsLoss() | |
| for step in range(1, 61): | |
| model.train() | |
| loss = loss_fn(model(*args), batch["label"]) | |
| opt.zero_grad(); loss.backward(); opt.step() | |
| if step % 20 == 0: | |
| model.eval() | |
| with torch.no_grad(): | |
| acc = ((model(*args) > 0).float() == batch["label"]).float().mean().item() | |
| print(f"step {step:3d} loss {loss.item():.4f} accuracy on the 20 files {acc:.2f}") | |