File size: 12,802 Bytes
0fc3616
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
"""JECT chest-CT encoder with a report-aligned readout.

The image encoder is frozen JECT pretraining. A latent-attention pooler reads its stage-4 tokens into 11 latents
(10 anatomy regions and one global latent); the global latent, projected and L2-normalised, is the image embedding.
Report text is embedded with chest2vec plus a LoRA adapter and a projection into the same space.
"""
from __future__ import annotations

import dataclasses
import json
import os
from typing import List, Optional, Sequence

import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from safetensors.torch import load_file

from transformers import PreTrainedModel

try:  # loaded as remote code by transformers (package-relative) or from a local folder on sys.path
    from .configuration_ject import JECTConfig
    from .ject_encoder import CT_TYPE_MAP, WINDOW_PARAMS, EncoderConfig, JECTEncoder
except ImportError:
    from configuration_ject import JECTConfig
    from ject_encoder import CT_TYPE_MAP, WINDOW_PARAMS, EncoderConfig, JECTEncoder


def window_hu(volume: torch.Tensor, window_id: int) -> torch.Tensor:
    """Raw HU to [0, 1] with the given window."""
    p = WINDOW_PARAMS[window_id]
    low, high = p["center"] - p["width"] / 2.0, p["center"] + p["width"] / 2.0
    return ((volume.float() - low) / (high - low)).clamp_(0.0, 1.0)


def center_crop(volume: np.ndarray, size: int) -> np.ndarray:
    """Centre crop the last three axes of a cube to size^3."""
    off = [(s - size) // 2 for s in volume.shape[-3:]]
    if min(off) < 0:
        raise ValueError(f"volume {volume.shape[-3:]} is smaller than {size}^3")
    return volume[..., off[0]:off[0] + size, off[1]:off[1] + size, off[2]:off[2] + size]


class LatentAttentionPooling(nn.Module):
    """Learned queries cross-attend to the token grid; returns one vector per query."""

    def __init__(self, dim: int, num_latents: int, num_heads: int, mlp_ratio: float = 4.0):
        super().__init__()
        self.latents = nn.Parameter(torch.zeros(1, num_latents, dim))
        self.norm_latents = nn.LayerNorm(dim)
        self.norm_tokens = nn.LayerNorm(dim)
        self.attn = nn.MultiheadAttention(dim, num_heads, batch_first=True)
        self.norm_mlp = nn.LayerNorm(dim)
        hidden = int(dim * mlp_ratio)
        self.mlp = nn.Sequential(nn.Linear(dim, hidden), nn.GELU(), nn.Linear(hidden, dim))

    def forward(self, tokens: torch.Tensor) -> torch.Tensor:
        q = self.norm_latents(self.latents).expand(tokens.shape[0], -1, -1)
        kv = self.norm_tokens(tokens)
        attended, _ = self.attn(q, kv, kv, need_weights=False)
        attended = attended + q
        return attended + self.mlp(self.norm_mlp(attended))


class JECTImageModel(nn.Module):
    def __init__(self, config: dict):
        super().__init__()
        fields = {f.name: f for f in dataclasses.fields(EncoderConfig)}
        enc = {k: tuple(v) if isinstance(v, list) else v for k, v in config["encoder"].items() if k in fields}
        self.encoder = JECTEncoder(EncoderConfig(**enc))
        r = config["readout"]
        dim = self.encoder.pos_embed_s4.shape[-1]
        self.pool = LatentAttentionPooling(dim, r["pool_latents"], r["pool_heads"])
        self.proj = nn.Linear(dim, r["embed_dim"])
        self.global_latent = r["global_latent"]
        self.window_id = config["input"]["window_id"]
        self.crop_size = config["input"]["crop_size"]

    @torch.no_grad()
    def forward(self, volume_hu: torch.Tensor, ct_type: Optional[Sequence[str]] = None) -> dict:
        """volume_hu: [B, 1, 224, 224, 224] raw HU; ct_type: optional per-scan 'non-contrast', 'contrast' or 'ldct'.

        Returns the image embedding and intermediate features.
        """
        e = self.encoder
        if tuple(volume_hu.shape[-3:]) != (self.crop_size,) * 3:
            raise ValueError(f"expected {self.crop_size}^3 input, got {tuple(volume_hu.shape[-3:])}")
        x = window_hu(volume_hu, self.window_id)
        wid = torch.full((x.shape[0],), self.window_id, dtype=torch.long, device=x.device)
        cid = None
        if ct_type is not None:
            cid = torch.tensor([CT_TYPE_MAP[str(t).strip().lower()] for t in ct_type], dtype=torch.long, device=x.device)
        x, _, _, _ = e.trunk(x, wid, cid)
        x = x + e.pos_embed_s4
        for blk in e.stage4:
            x = blk(x, e.grid_s4)
        latents = self.pool(x)
        z = F.normalize(self.proj(latents).float(), dim=-1)
        return {"embedding": z[:, self.global_latent], "latent_embeddings": z,
                "pooled": latents[:, self.global_latent].float(), "s4_tokens": x, "s4_mean": x.mean(1).float()}


class JECTTextModel(nn.Module):
    def __init__(self, config: dict, base_model: Optional[str] = None):
        super().__init__()
        from peft import LoraConfig, get_peft_model
        from transformers import AutoModel, AutoTokenizer

        t = config["text"]
        name = base_model or t["base_model"]
        base = AutoModel.from_pretrained(name, trust_remote_code=True, attn_implementation="eager")
        self.tokenizer = AutoTokenizer.from_pretrained(name, padding_side="left", trust_remote_code=True)
        if self.tokenizer.pad_token_id is None:
            self.tokenizer.pad_token = self.tokenizer.eos_token
        self.model = get_peft_model(base, LoraConfig(r=t["lora_r"], lora_alpha=t["lora_alpha"], lora_dropout=t.get("lora_dropout", 0.0),
                                                     target_modules=list(t["lora_targets"]), bias="none",
                                                     task_type="FEATURE_EXTRACTION"))
        dim = config["readout"]["embed_dim"]
        self.proj = nn.Linear(dim, dim)
        self.max_length = t["max_length"]
        self.pad_id = self.tokenizer.pad_token_id
        eod = self.tokenizer.convert_tokens_to_ids("<|endoftext|>")
        self.eod_id = eod if eod is not None and eod >= 0 else self.pad_id

    def _ids(self, texts: Sequence[str]) -> List[List[int]]:
        enc = self.tokenizer([str(t) for t in texts], add_special_tokens=False, truncation=True,
                             max_length=self.max_length - 1, padding=False, return_attention_mask=False)
        return [x + [self.eod_id] for x in enc["input_ids"]]

    @torch.no_grad()
    def forward(self, texts: Sequence[str], batch_size: int = 16) -> torch.Tensor:
        """Texts to L2-normalised embeddings [N, D]. Texts are length-sorted and left-padded per batch."""
        device = self.proj.weight.device
        ids = self._ids(texts)
        order = sorted(range(len(ids)), key=lambda i: len(ids[i]))
        out: List[Optional[torch.Tensor]] = [None] * len(ids)
        for start in range(0, len(order), batch_size):
            idx = order[start:start + batch_size]
            width = max(len(ids[i]) for i in idx)
            input_ids = torch.tensor([[self.pad_id] * (width - len(ids[i])) + ids[i] for i in idx], device=device)
            attn = torch.tensor([[0] * (width - len(ids[i])) + [1] * len(ids[i]) for i in idx], device=device)
            pooled = self.model.base_model.model.encode(input_ids, attn, normalize=False).pooler_output
            emb = F.normalize(self.proj(pooled.float()), dim=-1)
            for j, i in enumerate(idx):
                out[i] = emb[j]
        return torch.stack(out)


class JECTModel(nn.Module):
    """Image and text towers with the SigLIP scale and bias used for scoring."""

    def __init__(self, config: dict, with_text: bool = True, text_base_model: Optional[str] = None):
        super().__init__()
        self.config = config
        self.image = JECTImageModel(config)
        self.text = JECTTextModel(config, text_base_model) if with_text else None
        self.logit_scale = float(np.exp(config["logit_scale_log"]))
        self.logit_bias = float(config["logit_bias"])

    @classmethod
    def from_pretrained(cls, path: str, with_text: bool = True, text_base_model: Optional[str] = None,
                        device: str = "cpu") -> "JECTModel":
        """Load from a Hugging Face repo id or a local folder (config.json, model.safetensors, text_adapter.safetensors)."""
        path = _local(path)
        config = json.load(open(os.path.join(path, "config.json")))
        model = cls(config, with_text, text_base_model)
        _load_image_weights(model.image, os.path.join(path, "model.safetensors"))
        if model.text is not None:
            _load_text_adapter(model.text, os.path.join(path, "text_adapter.safetensors"))
        return model.to(device).eval()

    def similarity(self, image_embedding: torch.Tensor, text_embedding: torch.Tensor) -> torch.Tensor:
        """Pairwise SigLIP probabilities that each image matches each text."""
        return torch.sigmoid(self.logit_scale * image_embedding @ text_embedding.T + self.logit_bias)


def _local(path: str) -> str:
    if os.path.isdir(path):
        return path
    from huggingface_hub import snapshot_download
    return snapshot_download(path)


def _relative_position_index(window) -> torch.Tensor:
    wd, wh, ww = window
    coords = torch.stack(torch.meshgrid(torch.arange(wd), torch.arange(wh), torch.arange(ww), indexing="ij")).reshape(3, -1)
    rel = (coords[:, :, None] - coords[:, None, :]).permute(1, 2, 0).contiguous()
    rel[..., 0] += wd - 1
    rel[..., 1] += wh - 1
    rel[..., 2] += ww - 1
    rel[..., 0] *= (2 * wh - 1) * (2 * ww - 1)
    rel[..., 1] *= 2 * ww - 1
    return rel.sum(-1).view(-1)


def _rebuild_buffers(module: nn.Module) -> None:
    """Recompute the windowed-attention index buffers, which are not stored in the weights."""
    for m in module.modules():
        if "relative_position_index" in m._buffers and hasattr(m, "window"):
            dev = m.relative_position_bias_table.device
            m._buffers["relative_position_index"] = _relative_position_index(m.window).to(dev)


def _load_image_weights(image: nn.Module, file: str) -> None:
    sd = {k[len("image."):]: v for k, v in load_file(file).items() if k.startswith("image.")}
    missing, unexpected = image.load_state_dict(sd, strict=False)
    missing = [k for k in missing if not k.endswith("relative_position_index")]
    if missing or unexpected:
        raise RuntimeError(f"image weights do not match: missing={missing[:5]} unexpected={unexpected[:5]}")


def _load_text_adapter(text: nn.Module, file: str) -> None:
    sd = load_file(file)
    own = dict(text.named_parameters())
    bad = [k for k in sd if k not in own]
    if bad:
        raise RuntimeError(f"text adapter weights do not match: {bad[:5]}")
    with torch.no_grad():
        for k, v in sd.items():
            own[k].copy_(v)


class JECTImageEncoderModel(PreTrainedModel):
    """The JECT image encoder and readout, loadable with ``AutoModel.from_pretrained(..., trust_remote_code=True)``."""

    config_class = JECTConfig
    base_model_prefix = "image"
    main_input_name = "pixel_values"

    def __init__(self, config: JECTConfig):
        super().__init__(config)
        self.image = JECTImageModel({"encoder": config.encoder, "readout": config.readout, "input": config.input})
        self.post_init()

    def _init_weights(self, module):
        pass  # all weights come from model.safetensors

    @classmethod
    def from_pretrained(cls, *args, **kwargs):
        model = super().from_pretrained(*args, **kwargs)
        _rebuild_buffers(model)
        return model.eval()

    def forward(self, pixel_values: torch.Tensor, ct_type: Optional[Sequence[str]] = None) -> dict:
        """pixel_values: [B, 1, S, S, S] raw HU (see prepare.load_volume). Returns the same dict as JECTImageModel."""
        return self.image(pixel_values, ct_type)

    def similarity(self, image_embedding: torch.Tensor, text_embedding: torch.Tensor) -> torch.Tensor:
        scale = float(np.exp(self.config.logit_scale_log))
        return torch.sigmoid(scale * image_embedding @ text_embedding.T + self.config.logit_bias)


def load_text_model(repo: str, device: str = "cpu", text_base_model: Optional[str] = None) -> "JECTTextModel":
    """chest2vec 0.6B with this model's LoRA adapter and projection, for report and prompt embeddings."""
    path = _local(repo)
    config = json.load(open(os.path.join(path, "config.json")))
    text = JECTTextModel(config, text_base_model)
    _load_text_adapter(text, os.path.join(path, "text_adapter.safetensors"))
    return text.to(device).eval()


def load_cube(path: str, crop_size: int = 224) -> torch.Tensor:
    """Read a preprocessed npz cube (key ``ct``, raw HU) and return a [1, 1, 224, 224, 224] tensor."""
    with np.load(path) as d:
        v = d["ct"]
    if v.ndim == 4:
        v = v[0]
    return torch.from_numpy(np.ascontiguousarray(center_crop(v, crop_size))).unsqueeze(0).unsqueeze(0)