Download modeling_ject.py from lukeingawesome/JECT_320_aligned: direct link, hf CLI and curl.
- Browser
- Download file 12.8 kB
-
https://huggingface.co/lukeingawesome/JECT_320_aligned/resolve/main/modeling_ject.py
- Command line
-
hf download hf://lukeingawesome/JECT_320_aligned/modeling_ject.py
-
curl -L -o modeling_ject.py https://huggingface.co/lukeingawesome/JECT_320_aligned/resolve/main/modeling_ject.py
12.8 kB
| """JECT chest-CT encoder with a report-aligned readout. | |
| The image encoder is frozen JECT pretraining. A latent-attention pooler reads its stage-4 tokens into 11 latents | |
| (10 anatomy regions and one global latent); the global latent, projected and L2-normalised, is the image embedding. | |
| Report text is embedded with chest2vec plus a LoRA adapter and a projection into the same space. | |
| """ | |
| from __future__ import annotations | |
| import dataclasses | |
| import json | |
| import os | |
| from typing import List, Optional, Sequence | |
| import numpy as np | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| from safetensors.torch import load_file | |
| from transformers import PreTrainedModel | |
| try: # loaded as remote code by transformers (package-relative) or from a local folder on sys.path | |
| from .configuration_ject import JECTConfig | |
| from .ject_encoder import CT_TYPE_MAP, WINDOW_PARAMS, EncoderConfig, JECTEncoder | |
| except ImportError: | |
| from configuration_ject import JECTConfig | |
| from ject_encoder import CT_TYPE_MAP, WINDOW_PARAMS, EncoderConfig, JECTEncoder | |
| def window_hu(volume: torch.Tensor, window_id: int) -> torch.Tensor: | |
| """Raw HU to [0, 1] with the given window.""" | |
| p = WINDOW_PARAMS[window_id] | |
| low, high = p["center"] - p["width"] / 2.0, p["center"] + p["width"] / 2.0 | |
| return ((volume.float() - low) / (high - low)).clamp_(0.0, 1.0) | |
| def center_crop(volume: np.ndarray, size: int) -> np.ndarray: | |
| """Centre crop the last three axes of a cube to size^3.""" | |
| off = [(s - size) // 2 for s in volume.shape[-3:]] | |
| if min(off) < 0: | |
| raise ValueError(f"volume {volume.shape[-3:]} is smaller than {size}^3") | |
| return volume[..., off[0]:off[0] + size, off[1]:off[1] + size, off[2]:off[2] + size] | |
| class LatentAttentionPooling(nn.Module): | |
| """Learned queries cross-attend to the token grid; returns one vector per query.""" | |
| def __init__(self, dim: int, num_latents: int, num_heads: int, mlp_ratio: float = 4.0): | |
| super().__init__() | |
| self.latents = nn.Parameter(torch.zeros(1, num_latents, dim)) | |
| self.norm_latents = nn.LayerNorm(dim) | |
| self.norm_tokens = nn.LayerNorm(dim) | |
| self.attn = nn.MultiheadAttention(dim, num_heads, batch_first=True) | |
| self.norm_mlp = nn.LayerNorm(dim) | |
| hidden = int(dim * mlp_ratio) | |
| self.mlp = nn.Sequential(nn.Linear(dim, hidden), nn.GELU(), nn.Linear(hidden, dim)) | |
| def forward(self, tokens: torch.Tensor) -> torch.Tensor: | |
| q = self.norm_latents(self.latents).expand(tokens.shape[0], -1, -1) | |
| kv = self.norm_tokens(tokens) | |
| attended, _ = self.attn(q, kv, kv, need_weights=False) | |
| attended = attended + q | |
| return attended + self.mlp(self.norm_mlp(attended)) | |
| class JECTImageModel(nn.Module): | |
| def __init__(self, config: dict): | |
| super().__init__() | |
| fields = {f.name: f for f in dataclasses.fields(EncoderConfig)} | |
| enc = {k: tuple(v) if isinstance(v, list) else v for k, v in config["encoder"].items() if k in fields} | |
| self.encoder = JECTEncoder(EncoderConfig(**enc)) | |
| r = config["readout"] | |
| dim = self.encoder.pos_embed_s4.shape[-1] | |
| self.pool = LatentAttentionPooling(dim, r["pool_latents"], r["pool_heads"]) | |
| self.proj = nn.Linear(dim, r["embed_dim"]) | |
| self.global_latent = r["global_latent"] | |
| self.window_id = config["input"]["window_id"] | |
| self.crop_size = config["input"]["crop_size"] | |
| def forward(self, volume_hu: torch.Tensor, ct_type: Optional[Sequence[str]] = None) -> dict: | |
| """volume_hu: [B, 1, 224, 224, 224] raw HU; ct_type: optional per-scan 'non-contrast', 'contrast' or 'ldct'. | |
| Returns the image embedding and intermediate features. | |
| """ | |
| e = self.encoder | |
| if tuple(volume_hu.shape[-3:]) != (self.crop_size,) * 3: | |
| raise ValueError(f"expected {self.crop_size}^3 input, got {tuple(volume_hu.shape[-3:])}") | |
| x = window_hu(volume_hu, self.window_id) | |
| wid = torch.full((x.shape[0],), self.window_id, dtype=torch.long, device=x.device) | |
| cid = None | |
| if ct_type is not None: | |
| cid = torch.tensor([CT_TYPE_MAP[str(t).strip().lower()] for t in ct_type], dtype=torch.long, device=x.device) | |
| x, _, _, _ = e.trunk(x, wid, cid) | |
| x = x + e.pos_embed_s4 | |
| for blk in e.stage4: | |
| x = blk(x, e.grid_s4) | |
| latents = self.pool(x) | |
| z = F.normalize(self.proj(latents).float(), dim=-1) | |
| return {"embedding": z[:, self.global_latent], "latent_embeddings": z, | |
| "pooled": latents[:, self.global_latent].float(), "s4_tokens": x, "s4_mean": x.mean(1).float()} | |
| class JECTTextModel(nn.Module): | |
| def __init__(self, config: dict, base_model: Optional[str] = None): | |
| super().__init__() | |
| from peft import LoraConfig, get_peft_model | |
| from transformers import AutoModel, AutoTokenizer | |
| t = config["text"] | |
| name = base_model or t["base_model"] | |
| base = AutoModel.from_pretrained(name, trust_remote_code=True, attn_implementation="eager") | |
| self.tokenizer = AutoTokenizer.from_pretrained(name, padding_side="left", trust_remote_code=True) | |
| if self.tokenizer.pad_token_id is None: | |
| self.tokenizer.pad_token = self.tokenizer.eos_token | |
| self.model = get_peft_model(base, LoraConfig(r=t["lora_r"], lora_alpha=t["lora_alpha"], lora_dropout=t.get("lora_dropout", 0.0), | |
| target_modules=list(t["lora_targets"]), bias="none", | |
| task_type="FEATURE_EXTRACTION")) | |
| dim = config["readout"]["embed_dim"] | |
| self.proj = nn.Linear(dim, dim) | |
| self.max_length = t["max_length"] | |
| self.pad_id = self.tokenizer.pad_token_id | |
| eod = self.tokenizer.convert_tokens_to_ids("<|endoftext|>") | |
| self.eod_id = eod if eod is not None and eod >= 0 else self.pad_id | |
| def _ids(self, texts: Sequence[str]) -> List[List[int]]: | |
| enc = self.tokenizer([str(t) for t in texts], add_special_tokens=False, truncation=True, | |
| max_length=self.max_length - 1, padding=False, return_attention_mask=False) | |
| return [x + [self.eod_id] for x in enc["input_ids"]] | |
| def forward(self, texts: Sequence[str], batch_size: int = 16) -> torch.Tensor: | |
| """Texts to L2-normalised embeddings [N, D]. Texts are length-sorted and left-padded per batch.""" | |
| device = self.proj.weight.device | |
| ids = self._ids(texts) | |
| order = sorted(range(len(ids)), key=lambda i: len(ids[i])) | |
| out: List[Optional[torch.Tensor]] = [None] * len(ids) | |
| for start in range(0, len(order), batch_size): | |
| idx = order[start:start + batch_size] | |
| width = max(len(ids[i]) for i in idx) | |
| input_ids = torch.tensor([[self.pad_id] * (width - len(ids[i])) + ids[i] for i in idx], device=device) | |
| attn = torch.tensor([[0] * (width - len(ids[i])) + [1] * len(ids[i]) for i in idx], device=device) | |
| pooled = self.model.base_model.model.encode(input_ids, attn, normalize=False).pooler_output | |
| emb = F.normalize(self.proj(pooled.float()), dim=-1) | |
| for j, i in enumerate(idx): | |
| out[i] = emb[j] | |
| return torch.stack(out) | |
| class JECTModel(nn.Module): | |
| """Image and text towers with the SigLIP scale and bias used for scoring.""" | |
| def __init__(self, config: dict, with_text: bool = True, text_base_model: Optional[str] = None): | |
| super().__init__() | |
| self.config = config | |
| self.image = JECTImageModel(config) | |
| self.text = JECTTextModel(config, text_base_model) if with_text else None | |
| self.logit_scale = float(np.exp(config["logit_scale_log"])) | |
| self.logit_bias = float(config["logit_bias"]) | |
| def from_pretrained(cls, path: str, with_text: bool = True, text_base_model: Optional[str] = None, | |
| device: str = "cpu") -> "JECTModel": | |
| """Load from a Hugging Face repo id or a local folder (config.json, model.safetensors, text_adapter.safetensors).""" | |
| path = _local(path) | |
| config = json.load(open(os.path.join(path, "config.json"))) | |
| model = cls(config, with_text, text_base_model) | |
| _load_image_weights(model.image, os.path.join(path, "model.safetensors")) | |
| if model.text is not None: | |
| _load_text_adapter(model.text, os.path.join(path, "text_adapter.safetensors")) | |
| return model.to(device).eval() | |
| def similarity(self, image_embedding: torch.Tensor, text_embedding: torch.Tensor) -> torch.Tensor: | |
| """Pairwise SigLIP probabilities that each image matches each text.""" | |
| return torch.sigmoid(self.logit_scale * image_embedding @ text_embedding.T + self.logit_bias) | |
| def _local(path: str) -> str: | |
| if os.path.isdir(path): | |
| return path | |
| from huggingface_hub import snapshot_download | |
| return snapshot_download(path) | |
| def _relative_position_index(window) -> torch.Tensor: | |
| wd, wh, ww = window | |
| coords = torch.stack(torch.meshgrid(torch.arange(wd), torch.arange(wh), torch.arange(ww), indexing="ij")).reshape(3, -1) | |
| rel = (coords[:, :, None] - coords[:, None, :]).permute(1, 2, 0).contiguous() | |
| rel[..., 0] += wd - 1 | |
| rel[..., 1] += wh - 1 | |
| rel[..., 2] += ww - 1 | |
| rel[..., 0] *= (2 * wh - 1) * (2 * ww - 1) | |
| rel[..., 1] *= 2 * ww - 1 | |
| return rel.sum(-1).view(-1) | |
| def _rebuild_buffers(module: nn.Module) -> None: | |
| """Recompute the windowed-attention index buffers, which are not stored in the weights.""" | |
| for m in module.modules(): | |
| if "relative_position_index" in m._buffers and hasattr(m, "window"): | |
| dev = m.relative_position_bias_table.device | |
| m._buffers["relative_position_index"] = _relative_position_index(m.window).to(dev) | |
| def _load_image_weights(image: nn.Module, file: str) -> None: | |
| sd = {k[len("image."):]: v for k, v in load_file(file).items() if k.startswith("image.")} | |
| missing, unexpected = image.load_state_dict(sd, strict=False) | |
| missing = [k for k in missing if not k.endswith("relative_position_index")] | |
| if missing or unexpected: | |
| raise RuntimeError(f"image weights do not match: missing={missing[:5]} unexpected={unexpected[:5]}") | |
| def _load_text_adapter(text: nn.Module, file: str) -> None: | |
| sd = load_file(file) | |
| own = dict(text.named_parameters()) | |
| bad = [k for k in sd if k not in own] | |
| if bad: | |
| raise RuntimeError(f"text adapter weights do not match: {bad[:5]}") | |
| with torch.no_grad(): | |
| for k, v in sd.items(): | |
| own[k].copy_(v) | |
| class JECTImageEncoderModel(PreTrainedModel): | |
| """The JECT image encoder and readout, loadable with ``AutoModel.from_pretrained(..., trust_remote_code=True)``.""" | |
| config_class = JECTConfig | |
| base_model_prefix = "image" | |
| main_input_name = "pixel_values" | |
| def __init__(self, config: JECTConfig): | |
| super().__init__(config) | |
| self.image = JECTImageModel({"encoder": config.encoder, "readout": config.readout, "input": config.input}) | |
| self.post_init() | |
| def _init_weights(self, module): | |
| pass # all weights come from model.safetensors | |
| def from_pretrained(cls, *args, **kwargs): | |
| model = super().from_pretrained(*args, **kwargs) | |
| _rebuild_buffers(model) | |
| return model.eval() | |
| def forward(self, pixel_values: torch.Tensor, ct_type: Optional[Sequence[str]] = None) -> dict: | |
| """pixel_values: [B, 1, S, S, S] raw HU (see prepare.load_volume). Returns the same dict as JECTImageModel.""" | |
| return self.image(pixel_values, ct_type) | |
| def similarity(self, image_embedding: torch.Tensor, text_embedding: torch.Tensor) -> torch.Tensor: | |
| scale = float(np.exp(self.config.logit_scale_log)) | |
| return torch.sigmoid(scale * image_embedding @ text_embedding.T + self.config.logit_bias) | |
| def load_text_model(repo: str, device: str = "cpu", text_base_model: Optional[str] = None) -> "JECTTextModel": | |
| """chest2vec 0.6B with this model's LoRA adapter and projection, for report and prompt embeddings.""" | |
| path = _local(repo) | |
| config = json.load(open(os.path.join(path, "config.json"))) | |
| text = JECTTextModel(config, text_base_model) | |
| _load_text_adapter(text, os.path.join(path, "text_adapter.safetensors")) | |
| return text.to(device).eval() | |
| def load_cube(path: str, crop_size: int = 224) -> torch.Tensor: | |
| """Read a preprocessed npz cube (key ``ct``, raw HU) and return a [1, 1, 224, 224, 224] tensor.""" | |
| with np.load(path) as d: | |
| v = d["ct"] | |
| if v.ndim == 4: | |
| v = v[0] | |
| return torch.from_numpy(np.ascontiguousarray(center_crop(v, crop_size))).unsqueeze(0).unsqueeze(0) | |