"""JECT chest-CT encoder with a report-aligned readout. The image encoder is frozen JECT pretraining. A latent-attention pooler reads its stage-4 tokens into 11 latents (10 anatomy regions and one global latent); the global latent, projected and L2-normalised, is the image embedding. Report text is embedded with chest2vec plus a LoRA adapter and a projection into the same space. """ from __future__ import annotations import dataclasses import json import os from typing import List, Optional, Sequence import numpy as np import torch import torch.nn as nn import torch.nn.functional as F from safetensors.torch import load_file from transformers import PreTrainedModel try: # loaded as remote code by transformers (package-relative) or from a local folder on sys.path from .configuration_ject import JECTConfig from .ject_encoder import CT_TYPE_MAP, WINDOW_PARAMS, EncoderConfig, JECTEncoder except ImportError: from configuration_ject import JECTConfig from ject_encoder import CT_TYPE_MAP, WINDOW_PARAMS, EncoderConfig, JECTEncoder def window_hu(volume: torch.Tensor, window_id: int) -> torch.Tensor: """Raw HU to [0, 1] with the given window.""" p = WINDOW_PARAMS[window_id] low, high = p["center"] - p["width"] / 2.0, p["center"] + p["width"] / 2.0 return ((volume.float() - low) / (high - low)).clamp_(0.0, 1.0) def center_crop(volume: np.ndarray, size: int) -> np.ndarray: """Centre crop the last three axes of a cube to size^3.""" off = [(s - size) // 2 for s in volume.shape[-3:]] if min(off) < 0: raise ValueError(f"volume {volume.shape[-3:]} is smaller than {size}^3") return volume[..., off[0]:off[0] + size, off[1]:off[1] + size, off[2]:off[2] + size] class LatentAttentionPooling(nn.Module): """Learned queries cross-attend to the token grid; returns one vector per query.""" def __init__(self, dim: int, num_latents: int, num_heads: int, mlp_ratio: float = 4.0): super().__init__() self.latents = nn.Parameter(torch.zeros(1, num_latents, dim)) self.norm_latents = nn.LayerNorm(dim) self.norm_tokens = nn.LayerNorm(dim) self.attn = nn.MultiheadAttention(dim, num_heads, batch_first=True) self.norm_mlp = nn.LayerNorm(dim) hidden = int(dim * mlp_ratio) self.mlp = nn.Sequential(nn.Linear(dim, hidden), nn.GELU(), nn.Linear(hidden, dim)) def forward(self, tokens: torch.Tensor) -> torch.Tensor: q = self.norm_latents(self.latents).expand(tokens.shape[0], -1, -1) kv = self.norm_tokens(tokens) attended, _ = self.attn(q, kv, kv, need_weights=False) attended = attended + q return attended + self.mlp(self.norm_mlp(attended)) class JECTImageModel(nn.Module): def __init__(self, config: dict): super().__init__() fields = {f.name: f for f in dataclasses.fields(EncoderConfig)} enc = {k: tuple(v) if isinstance(v, list) else v for k, v in config["encoder"].items() if k in fields} self.encoder = JECTEncoder(EncoderConfig(**enc)) r = config["readout"] dim = self.encoder.pos_embed_s4.shape[-1] self.pool = LatentAttentionPooling(dim, r["pool_latents"], r["pool_heads"]) self.proj = nn.Linear(dim, r["embed_dim"]) self.global_latent = r["global_latent"] self.window_id = config["input"]["window_id"] self.crop_size = config["input"]["crop_size"] @torch.no_grad() def forward(self, volume_hu: torch.Tensor, ct_type: Optional[Sequence[str]] = None) -> dict: """volume_hu: [B, 1, 224, 224, 224] raw HU; ct_type: optional per-scan 'non-contrast', 'contrast' or 'ldct'. Returns the image embedding and intermediate features. """ e = self.encoder if tuple(volume_hu.shape[-3:]) != (self.crop_size,) * 3: raise ValueError(f"expected {self.crop_size}^3 input, got {tuple(volume_hu.shape[-3:])}") x = window_hu(volume_hu, self.window_id) wid = torch.full((x.shape[0],), self.window_id, dtype=torch.long, device=x.device) cid = None if ct_type is not None: cid = torch.tensor([CT_TYPE_MAP[str(t).strip().lower()] for t in ct_type], dtype=torch.long, device=x.device) x, _, _, _ = e.trunk(x, wid, cid) x = x + e.pos_embed_s4 for blk in e.stage4: x = blk(x, e.grid_s4) latents = self.pool(x) z = F.normalize(self.proj(latents).float(), dim=-1) return {"embedding": z[:, self.global_latent], "latent_embeddings": z, "pooled": latents[:, self.global_latent].float(), "s4_tokens": x, "s4_mean": x.mean(1).float()} class JECTTextModel(nn.Module): def __init__(self, config: dict, base_model: Optional[str] = None): super().__init__() from peft import LoraConfig, get_peft_model from transformers import AutoModel, AutoTokenizer t = config["text"] name = base_model or t["base_model"] base = AutoModel.from_pretrained(name, trust_remote_code=True, attn_implementation="eager") self.tokenizer = AutoTokenizer.from_pretrained(name, padding_side="left", trust_remote_code=True) if self.tokenizer.pad_token_id is None: self.tokenizer.pad_token = self.tokenizer.eos_token self.model = get_peft_model(base, LoraConfig(r=t["lora_r"], lora_alpha=t["lora_alpha"], lora_dropout=t.get("lora_dropout", 0.0), target_modules=list(t["lora_targets"]), bias="none", task_type="FEATURE_EXTRACTION")) dim = config["readout"]["embed_dim"] self.proj = nn.Linear(dim, dim) self.max_length = t["max_length"] self.pad_id = self.tokenizer.pad_token_id eod = self.tokenizer.convert_tokens_to_ids("<|endoftext|>") self.eod_id = eod if eod is not None and eod >= 0 else self.pad_id def _ids(self, texts: Sequence[str]) -> List[List[int]]: enc = self.tokenizer([str(t) for t in texts], add_special_tokens=False, truncation=True, max_length=self.max_length - 1, padding=False, return_attention_mask=False) return [x + [self.eod_id] for x in enc["input_ids"]] @torch.no_grad() def forward(self, texts: Sequence[str], batch_size: int = 16) -> torch.Tensor: """Texts to L2-normalised embeddings [N, D]. Texts are length-sorted and left-padded per batch.""" device = self.proj.weight.device ids = self._ids(texts) order = sorted(range(len(ids)), key=lambda i: len(ids[i])) out: List[Optional[torch.Tensor]] = [None] * len(ids) for start in range(0, len(order), batch_size): idx = order[start:start + batch_size] width = max(len(ids[i]) for i in idx) input_ids = torch.tensor([[self.pad_id] * (width - len(ids[i])) + ids[i] for i in idx], device=device) attn = torch.tensor([[0] * (width - len(ids[i])) + [1] * len(ids[i]) for i in idx], device=device) pooled = self.model.base_model.model.encode(input_ids, attn, normalize=False).pooler_output emb = F.normalize(self.proj(pooled.float()), dim=-1) for j, i in enumerate(idx): out[i] = emb[j] return torch.stack(out) class JECTModel(nn.Module): """Image and text towers with the SigLIP scale and bias used for scoring.""" def __init__(self, config: dict, with_text: bool = True, text_base_model: Optional[str] = None): super().__init__() self.config = config self.image = JECTImageModel(config) self.text = JECTTextModel(config, text_base_model) if with_text else None self.logit_scale = float(np.exp(config["logit_scale_log"])) self.logit_bias = float(config["logit_bias"]) @classmethod def from_pretrained(cls, path: str, with_text: bool = True, text_base_model: Optional[str] = None, device: str = "cpu") -> "JECTModel": """Load from a Hugging Face repo id or a local folder (config.json, model.safetensors, text_adapter.safetensors).""" path = _local(path) config = json.load(open(os.path.join(path, "config.json"))) model = cls(config, with_text, text_base_model) _load_image_weights(model.image, os.path.join(path, "model.safetensors")) if model.text is not None: _load_text_adapter(model.text, os.path.join(path, "text_adapter.safetensors")) return model.to(device).eval() def similarity(self, image_embedding: torch.Tensor, text_embedding: torch.Tensor) -> torch.Tensor: """Pairwise SigLIP probabilities that each image matches each text.""" return torch.sigmoid(self.logit_scale * image_embedding @ text_embedding.T + self.logit_bias) def _local(path: str) -> str: if os.path.isdir(path): return path from huggingface_hub import snapshot_download return snapshot_download(path) def _relative_position_index(window) -> torch.Tensor: wd, wh, ww = window coords = torch.stack(torch.meshgrid(torch.arange(wd), torch.arange(wh), torch.arange(ww), indexing="ij")).reshape(3, -1) rel = (coords[:, :, None] - coords[:, None, :]).permute(1, 2, 0).contiguous() rel[..., 0] += wd - 1 rel[..., 1] += wh - 1 rel[..., 2] += ww - 1 rel[..., 0] *= (2 * wh - 1) * (2 * ww - 1) rel[..., 1] *= 2 * ww - 1 return rel.sum(-1).view(-1) def _rebuild_buffers(module: nn.Module) -> None: """Recompute the windowed-attention index buffers, which are not stored in the weights.""" for m in module.modules(): if "relative_position_index" in m._buffers and hasattr(m, "window"): dev = m.relative_position_bias_table.device m._buffers["relative_position_index"] = _relative_position_index(m.window).to(dev) def _load_image_weights(image: nn.Module, file: str) -> None: sd = {k[len("image."):]: v for k, v in load_file(file).items() if k.startswith("image.")} missing, unexpected = image.load_state_dict(sd, strict=False) missing = [k for k in missing if not k.endswith("relative_position_index")] if missing or unexpected: raise RuntimeError(f"image weights do not match: missing={missing[:5]} unexpected={unexpected[:5]}") def _load_text_adapter(text: nn.Module, file: str) -> None: sd = load_file(file) own = dict(text.named_parameters()) bad = [k for k in sd if k not in own] if bad: raise RuntimeError(f"text adapter weights do not match: {bad[:5]}") with torch.no_grad(): for k, v in sd.items(): own[k].copy_(v) class JECTImageEncoderModel(PreTrainedModel): """The JECT image encoder and readout, loadable with ``AutoModel.from_pretrained(..., trust_remote_code=True)``.""" config_class = JECTConfig base_model_prefix = "image" main_input_name = "pixel_values" def __init__(self, config: JECTConfig): super().__init__(config) self.image = JECTImageModel({"encoder": config.encoder, "readout": config.readout, "input": config.input}) self.post_init() def _init_weights(self, module): pass # all weights come from model.safetensors @classmethod def from_pretrained(cls, *args, **kwargs): model = super().from_pretrained(*args, **kwargs) _rebuild_buffers(model) return model.eval() def forward(self, pixel_values: torch.Tensor, ct_type: Optional[Sequence[str]] = None) -> dict: """pixel_values: [B, 1, S, S, S] raw HU (see prepare.load_volume). Returns the same dict as JECTImageModel.""" return self.image(pixel_values, ct_type) def similarity(self, image_embedding: torch.Tensor, text_embedding: torch.Tensor) -> torch.Tensor: scale = float(np.exp(self.config.logit_scale_log)) return torch.sigmoid(scale * image_embedding @ text_embedding.T + self.config.logit_bias) def load_text_model(repo: str, device: str = "cpu", text_base_model: Optional[str] = None) -> "JECTTextModel": """chest2vec 0.6B with this model's LoRA adapter and projection, for report and prompt embeddings.""" path = _local(repo) config = json.load(open(os.path.join(path, "config.json"))) text = JECTTextModel(config, text_base_model) _load_text_adapter(text, os.path.join(path, "text_adapter.safetensors")) return text.to(device).eval() def load_cube(path: str, crop_size: int = 224) -> torch.Tensor: """Read a preprocessed npz cube (key ``ct``, raw HU) and return a [1, 1, 224, 224, 224] tensor.""" with np.load(path) as d: v = d["ct"] if v.ndim == 4: v = v[0] return torch.from_numpy(np.ascontiguousarray(center_crop(v, crop_size))).unsqueeze(0).unsqueeze(0)