JECT_320 / modeling_ject.py
lukeingawesome's picture
Initial release
04c01e0 verified
Raw History Blame Contribute Delete
12.8 kB
"""JECT chest-CT encoder with a report-aligned readout.
The image encoder is frozen JECT pretraining. A latent-attention pooler reads its stage-4 tokens into 11 latents
(10 anatomy regions and one global latent); the global latent, projected and L2-normalised, is the image embedding.
Report text is embedded with chest2vec plus a LoRA adapter and a projection into the same space.
"""
from __future__ import annotations
import dataclasses
import json
import os
from typing import List, Optional, Sequence
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from safetensors.torch import load_file
from transformers import PreTrainedModel
try: # loaded as remote code by transformers (package-relative) or from a local folder on sys.path
from .configuration_ject import JECTConfig
from .ject_encoder import CT_TYPE_MAP, WINDOW_PARAMS, EncoderConfig, JECTEncoder
except ImportError:
from configuration_ject import JECTConfig
from ject_encoder import CT_TYPE_MAP, WINDOW_PARAMS, EncoderConfig, JECTEncoder
def window_hu(volume: torch.Tensor, window_id: int) -> torch.Tensor:
"""Raw HU to [0, 1] with the given window."""
p = WINDOW_PARAMS[window_id]
low, high = p["center"] - p["width"] / 2.0, p["center"] + p["width"] / 2.0
return ((volume.float() - low) / (high - low)).clamp_(0.0, 1.0)
def center_crop(volume: np.ndarray, size: int) -> np.ndarray:
"""Centre crop the last three axes of a cube to size^3."""
off = [(s - size) // 2 for s in volume.shape[-3:]]
if min(off) < 0:
raise ValueError(f"volume {volume.shape[-3:]} is smaller than {size}^3")
return volume[..., off[0]:off[0] + size, off[1]:off[1] + size, off[2]:off[2] + size]
class LatentAttentionPooling(nn.Module):
"""Learned queries cross-attend to the token grid; returns one vector per query."""
def __init__(self, dim: int, num_latents: int, num_heads: int, mlp_ratio: float = 4.0):
super().__init__()
self.latents = nn.Parameter(torch.zeros(1, num_latents, dim))
self.norm_latents = nn.LayerNorm(dim)
self.norm_tokens = nn.LayerNorm(dim)
self.attn = nn.MultiheadAttention(dim, num_heads, batch_first=True)
self.norm_mlp = nn.LayerNorm(dim)
hidden = int(dim * mlp_ratio)
self.mlp = nn.Sequential(nn.Linear(dim, hidden), nn.GELU(), nn.Linear(hidden, dim))
def forward(self, tokens: torch.Tensor) -> torch.Tensor:
q = self.norm_latents(self.latents).expand(tokens.shape[0], -1, -1)
kv = self.norm_tokens(tokens)
attended, _ = self.attn(q, kv, kv, need_weights=False)
attended = attended + q
return attended + self.mlp(self.norm_mlp(attended))
class JECTImageModel(nn.Module):
def __init__(self, config: dict):
super().__init__()
fields = {f.name: f for f in dataclasses.fields(EncoderConfig)}
enc = {k: tuple(v) if isinstance(v, list) else v for k, v in config["encoder"].items() if k in fields}
self.encoder = JECTEncoder(EncoderConfig(**enc))
r = config["readout"]
dim = self.encoder.pos_embed_s4.shape[-1]
self.pool = LatentAttentionPooling(dim, r["pool_latents"], r["pool_heads"])
self.proj = nn.Linear(dim, r["embed_dim"])
self.global_latent = r["global_latent"]
self.window_id = config["input"]["window_id"]
self.crop_size = config["input"]["crop_size"]
@torch.no_grad()
def forward(self, volume_hu: torch.Tensor, ct_type: Optional[Sequence[str]] = None) -> dict:
"""volume_hu: [B, 1, 224, 224, 224] raw HU; ct_type: optional per-scan 'non-contrast', 'contrast' or 'ldct'.
Returns the image embedding and intermediate features.
"""
e = self.encoder
if tuple(volume_hu.shape[-3:]) != (self.crop_size,) * 3:
raise ValueError(f"expected {self.crop_size}^3 input, got {tuple(volume_hu.shape[-3:])}")
x = window_hu(volume_hu, self.window_id)
wid = torch.full((x.shape[0],), self.window_id, dtype=torch.long, device=x.device)
cid = None
if ct_type is not None:
cid = torch.tensor([CT_TYPE_MAP[str(t).strip().lower()] for t in ct_type], dtype=torch.long, device=x.device)
x, _, _, _ = e.trunk(x, wid, cid)
x = x + e.pos_embed_s4
for blk in e.stage4:
x = blk(x, e.grid_s4)
latents = self.pool(x)
z = F.normalize(self.proj(latents).float(), dim=-1)
return {"embedding": z[:, self.global_latent], "latent_embeddings": z,
"pooled": latents[:, self.global_latent].float(), "s4_tokens": x, "s4_mean": x.mean(1).float()}
class JECTTextModel(nn.Module):
def __init__(self, config: dict, base_model: Optional[str] = None):
super().__init__()
from peft import LoraConfig, get_peft_model
from transformers import AutoModel, AutoTokenizer
t = config["text"]
name = base_model or t["base_model"]
base = AutoModel.from_pretrained(name, trust_remote_code=True, attn_implementation="eager")
self.tokenizer = AutoTokenizer.from_pretrained(name, padding_side="left", trust_remote_code=True)
if self.tokenizer.pad_token_id is None:
self.tokenizer.pad_token = self.tokenizer.eos_token
self.model = get_peft_model(base, LoraConfig(r=t["lora_r"], lora_alpha=t["lora_alpha"], lora_dropout=t.get("lora_dropout", 0.0),
target_modules=list(t["lora_targets"]), bias="none",
task_type="FEATURE_EXTRACTION"))
dim = config["readout"]["embed_dim"]
self.proj = nn.Linear(dim, dim)
self.max_length = t["max_length"]
self.pad_id = self.tokenizer.pad_token_id
eod = self.tokenizer.convert_tokens_to_ids("<|endoftext|>")
self.eod_id = eod if eod is not None and eod >= 0 else self.pad_id
def _ids(self, texts: Sequence[str]) -> List[List[int]]:
enc = self.tokenizer([str(t) for t in texts], add_special_tokens=False, truncation=True,
max_length=self.max_length - 1, padding=False, return_attention_mask=False)
return [x + [self.eod_id] for x in enc["input_ids"]]
@torch.no_grad()
def forward(self, texts: Sequence[str], batch_size: int = 16) -> torch.Tensor:
"""Texts to L2-normalised embeddings [N, D]. Texts are length-sorted and left-padded per batch."""
device = self.proj.weight.device
ids = self._ids(texts)
order = sorted(range(len(ids)), key=lambda i: len(ids[i]))
out: List[Optional[torch.Tensor]] = [None] * len(ids)
for start in range(0, len(order), batch_size):
idx = order[start:start + batch_size]
width = max(len(ids[i]) for i in idx)
input_ids = torch.tensor([[self.pad_id] * (width - len(ids[i])) + ids[i] for i in idx], device=device)
attn = torch.tensor([[0] * (width - len(ids[i])) + [1] * len(ids[i]) for i in idx], device=device)
pooled = self.model.base_model.model.encode(input_ids, attn, normalize=False).pooler_output
emb = F.normalize(self.proj(pooled.float()), dim=-1)
for j, i in enumerate(idx):
out[i] = emb[j]
return torch.stack(out)
class JECTModel(nn.Module):
"""Image and text towers with the SigLIP scale and bias used for scoring."""
def __init__(self, config: dict, with_text: bool = True, text_base_model: Optional[str] = None):
super().__init__()
self.config = config
self.image = JECTImageModel(config)
self.text = JECTTextModel(config, text_base_model) if with_text else None
self.logit_scale = float(np.exp(config["logit_scale_log"]))
self.logit_bias = float(config["logit_bias"])
@classmethod
def from_pretrained(cls, path: str, with_text: bool = True, text_base_model: Optional[str] = None,
device: str = "cpu") -> "JECTModel":
"""Load from a Hugging Face repo id or a local folder (config.json, model.safetensors, text_adapter.safetensors)."""
path = _local(path)
config = json.load(open(os.path.join(path, "config.json")))
model = cls(config, with_text, text_base_model)
_load_image_weights(model.image, os.path.join(path, "model.safetensors"))
if model.text is not None:
_load_text_adapter(model.text, os.path.join(path, "text_adapter.safetensors"))
return model.to(device).eval()
def similarity(self, image_embedding: torch.Tensor, text_embedding: torch.Tensor) -> torch.Tensor:
"""Pairwise SigLIP probabilities that each image matches each text."""
return torch.sigmoid(self.logit_scale * image_embedding @ text_embedding.T + self.logit_bias)
def _local(path: str) -> str:
if os.path.isdir(path):
return path
from huggingface_hub import snapshot_download
return snapshot_download(path)
def _relative_position_index(window) -> torch.Tensor:
wd, wh, ww = window
coords = torch.stack(torch.meshgrid(torch.arange(wd), torch.arange(wh), torch.arange(ww), indexing="ij")).reshape(3, -1)
rel = (coords[:, :, None] - coords[:, None, :]).permute(1, 2, 0).contiguous()
rel[..., 0] += wd - 1
rel[..., 1] += wh - 1
rel[..., 2] += ww - 1
rel[..., 0] *= (2 * wh - 1) * (2 * ww - 1)
rel[..., 1] *= 2 * ww - 1
return rel.sum(-1).view(-1)
def _rebuild_buffers(module: nn.Module) -> None:
"""Recompute the windowed-attention index buffers, which are not stored in the weights."""
for m in module.modules():
if "relative_position_index" in m._buffers and hasattr(m, "window"):
dev = m.relative_position_bias_table.device
m._buffers["relative_position_index"] = _relative_position_index(m.window).to(dev)
def _load_image_weights(image: nn.Module, file: str) -> None:
sd = {k[len("image."):]: v for k, v in load_file(file).items() if k.startswith("image.")}
missing, unexpected = image.load_state_dict(sd, strict=False)
missing = [k for k in missing if not k.endswith("relative_position_index")]
if missing or unexpected:
raise RuntimeError(f"image weights do not match: missing={missing[:5]} unexpected={unexpected[:5]}")
def _load_text_adapter(text: nn.Module, file: str) -> None:
sd = load_file(file)
own = dict(text.named_parameters())
bad = [k for k in sd if k not in own]
if bad:
raise RuntimeError(f"text adapter weights do not match: {bad[:5]}")
with torch.no_grad():
for k, v in sd.items():
own[k].copy_(v)
class JECTImageEncoderModel(PreTrainedModel):
"""The JECT image encoder and readout, loadable with ``AutoModel.from_pretrained(..., trust_remote_code=True)``."""
config_class = JECTConfig
base_model_prefix = "image"
main_input_name = "pixel_values"
def __init__(self, config: JECTConfig):
super().__init__(config)
self.image = JECTImageModel({"encoder": config.encoder, "readout": config.readout, "input": config.input})
self.post_init()
def _init_weights(self, module):
pass # all weights come from model.safetensors
@classmethod
def from_pretrained(cls, *args, **kwargs):
model = super().from_pretrained(*args, **kwargs)
_rebuild_buffers(model)
return model.eval()
def forward(self, pixel_values: torch.Tensor, ct_type: Optional[Sequence[str]] = None) -> dict:
"""pixel_values: [B, 1, S, S, S] raw HU (see prepare.load_volume). Returns the same dict as JECTImageModel."""
return self.image(pixel_values, ct_type)
def similarity(self, image_embedding: torch.Tensor, text_embedding: torch.Tensor) -> torch.Tensor:
scale = float(np.exp(self.config.logit_scale_log))
return torch.sigmoid(scale * image_embedding @ text_embedding.T + self.config.logit_bias)
def load_text_model(repo: str, device: str = "cpu", text_base_model: Optional[str] = None) -> "JECTTextModel":
"""chest2vec 0.6B with this model's LoRA adapter and projection, for report and prompt embeddings."""
path = _local(repo)
config = json.load(open(os.path.join(path, "config.json")))
text = JECTTextModel(config, text_base_model)
_load_text_adapter(text, os.path.join(path, "text_adapter.safetensors"))
return text.to(device).eval()
def load_cube(path: str, crop_size: int = 224) -> torch.Tensor:
"""Read a preprocessed npz cube (key ``ct``, raw HU) and return a [1, 1, 224, 224, 224] tensor."""
with np.load(path) as d:
v = d["ct"]
if v.ndim == 4:
v = v[0]
return torch.from_numpy(np.ascontiguousarray(center_crop(v, crop_size))).unsqueeze(0).unsqueeze(0)