lukeingawesome commited on
Commit
0fc3616
·
verified ·
1 Parent(s): ff2990b

Initial release

Browse files
Files changed (14) hide show
  1. .gitattributes +0 -34
  2. .gitignore +9 -0
  3. LICENSE +12 -0
  4. LICENSE-CODE +21 -0
  5. README.md +125 -0
  6. config.json +105 -0
  7. configuration_ject.py +15 -0
  8. ject_encoder.py +734 -0
  9. model.safetensors +3 -0
  10. modeling_ject.py +269 -0
  11. prepare.py +160 -0
  12. preprocess.py +155 -0
  13. requirements.txt +12 -0
  14. text_adapter.safetensors +3 -0
.gitattributes CHANGED
@@ -1,35 +1 @@
1
- *.7z filter=lfs diff=lfs merge=lfs -text
2
- *.arrow filter=lfs diff=lfs merge=lfs -text
3
- *.bin filter=lfs diff=lfs merge=lfs -text
4
- *.bz2 filter=lfs diff=lfs merge=lfs -text
5
- *.ckpt filter=lfs diff=lfs merge=lfs -text
6
- *.ftz filter=lfs diff=lfs merge=lfs -text
7
- *.gz filter=lfs diff=lfs merge=lfs -text
8
- *.h5 filter=lfs diff=lfs merge=lfs -text
9
- *.joblib filter=lfs diff=lfs merge=lfs -text
10
- *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
- *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
- *.model filter=lfs diff=lfs merge=lfs -text
13
- *.msgpack filter=lfs diff=lfs merge=lfs -text
14
- *.npy filter=lfs diff=lfs merge=lfs -text
15
- *.npz filter=lfs diff=lfs merge=lfs -text
16
- *.onnx filter=lfs diff=lfs merge=lfs -text
17
- *.ot filter=lfs diff=lfs merge=lfs -text
18
- *.parquet filter=lfs diff=lfs merge=lfs -text
19
- *.pb filter=lfs diff=lfs merge=lfs -text
20
- *.pickle filter=lfs diff=lfs merge=lfs -text
21
- *.pkl filter=lfs diff=lfs merge=lfs -text
22
- *.pt filter=lfs diff=lfs merge=lfs -text
23
- *.pth filter=lfs diff=lfs merge=lfs -text
24
- *.rar filter=lfs diff=lfs merge=lfs -text
25
  *.safetensors filter=lfs diff=lfs merge=lfs -text
26
- saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
- *.tar.* filter=lfs diff=lfs merge=lfs -text
28
- *.tar filter=lfs diff=lfs merge=lfs -text
29
- *.tflite filter=lfs diff=lfs merge=lfs -text
30
- *.tgz filter=lfs diff=lfs merge=lfs -text
31
- *.wasm filter=lfs diff=lfs merge=lfs -text
32
- *.xz filter=lfs diff=lfs merge=lfs -text
33
- *.zip filter=lfs diff=lfs merge=lfs -text
34
- *.zst filter=lfs diff=lfs merge=lfs -text
35
- *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  *.safetensors filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
 
 
.gitignore ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ # weights, features and tables
2
+ *.pt
3
+ *.csv
4
+
5
+ # caches
6
+ __pycache__/
7
+ *.py[cod]
8
+ .cache/
9
+ cache/
LICENSE ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Model weights: Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International (CC BY-NC-SA 4.0)
2
+
3
+ Copyright (c) 2026 The JECT Authors
4
+
5
+ The files model.safetensors and text_adapter.safetensors are licensed under
6
+ CC BY-NC-SA 4.0. You may share and adapt them for non-commercial purposes, with attribution, and must distribute
7
+ adaptations under the same license. Commercial use is not permitted.
8
+
9
+ Full legal text: https://creativecommons.org/licenses/by-nc-sa/4.0/legalcode
10
+
11
+ The Python files in this repository (configuration_ject.py, modeling_ject.py, ject_encoder.py, preprocess.py, prepare.py) are licensed under the
12
+ MIT License; see LICENSE-CODE.
LICENSE-CODE ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ MIT License
2
+
3
+ Copyright (c) 2026 The JECT Authors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
README.md ADDED
@@ -0,0 +1,125 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: cc-by-nc-sa-4.0
3
+ library_name: pytorch
4
+ pipeline_tag: feature-extraction
5
+ tags:
6
+ - medical-imaging
7
+ - chest-ct
8
+ - 3d
9
+ - foundation-model
10
+ - jepa
11
+ - siglip
12
+ ---
13
+
14
+ # JECT_320_aligned
15
+
16
+ Model weights for *JECT: Addressing the Context Shortcut in Joint-Embedding Predictive Learning for Chest CT*.
17
+
18
+ A 3D chest-CT encoder pretrained with JECT and cooled down at 320³ on 1 mm volumes, then aligned end to end with
19
+ radiology reports: the whole image encoder, a latent-attention pooler, a projection and a text adapter were trained
20
+ together. A variant with the encoder kept frozen is released as JECT_320. Use it for CT embeddings, report retrieval and
21
+ zero-shot finding scores.
22
+
23
+ | component | parameters | trained in the alignment stage |
24
+ |---|---|---|
25
+ | image encoder (JECT, 320³ input) | 161 M | yes |
26
+ | latent-attention pooler + projection | 13.7 M | yes |
27
+ | text: chest2vec 0.6B + LoRA (r 16) + projection | 11.1 M adapter | adapter only |
28
+
29
+ ## Input
30
+
31
+ A chest CT as NIfTI (`.nii.gz`) or as an already preprocessed cube (`.npz`). `prepare.load_volume` turns either into the
32
+ model input: reorient to LPS, resample to 1 mm, pad with air (-1024 HU), crop a 384³ cube around the lungs, keep raw HU,
33
+ then centre-crop 320³. The window is applied inside the model.
34
+
35
+ | mode | how the lungs are located | speed | needs |
36
+ |---|---|---|---|
37
+ | `fast` (default for NIfTI) | air inside the body, found by HU threshold | seconds | `monai`, `nibabel`, `scipy` |
38
+ | `totalseg` | TotalSegmentator lung lobes, as in training | about a minute on a GPU | `TotalSegmentator` |
39
+ | `center` | middle of the scan, no lung search | seconds | `monai`, `nibabel` |
40
+ | `npz` | cube already preprocessed for this model | instant | none |
41
+
42
+ `totalseg` reproduces the training input exactly. Most chest CTs fit inside the 384³ cube, so all modes give the same input;
43
+ they differ only on scans that extend beyond it (for example chest-abdomen series), where `center` can miss the lungs.
44
+
45
+ Measured with JECT_224 on 414 CT-RATE validation scans available as NIfTI, against the exact training input (CT-RATE-18
46
+ linear probe, macro AUROC; paired bootstrap over scans):
47
+
48
+ | mode | macro AUROC | difference to exact [95 % CI] | embedding cosine to exact, median / 5th percentile |
49
+ |---|---|---|---|
50
+ | exact (`totalseg` or `npz`) | 0.875 | | 1 / 1 |
51
+ | `fast` | 0.875 | -0.0005 [-0.0015, +0.0005] | 1.000 / 0.977 |
52
+ | `center` | 0.875 | -0.0002 [-0.0018, +0.0014] | 1.000 / 0.955 |
53
+ Scan type (`"non-contrast"`, `"contrast"` or `"ldct"`): Pass the scan type; the model was trained with it.
54
+
55
+ ## Usage
56
+
57
+ ```python
58
+ import torch
59
+ from transformers import AutoModel
60
+
61
+ repo = "<user>/JECT_320_aligned"
62
+ model = AutoModel.from_pretrained(repo, trust_remote_code=True).cuda()
63
+
64
+ from huggingface_hub import snapshot_download # input preparation and the text tower ship as code in the repo
65
+ import sys; sys.path.insert(0, snapshot_download(repo))
66
+ from prepare import load_volume
67
+ from modeling_ject import load_text_model
68
+
69
+ volume = load_volume("scan.nii.gz", model.config).cuda() # mode="fast"; also "totalseg", "center", or an .npz path
70
+ with torch.no_grad(), torch.autocast("cuda", dtype=torch.bfloat16):
71
+ out = model(volume, ct_type=["non-contrast"])
72
+ text = load_text_model(repo, device="cuda") # needs chest2vec/chest2vec_0.6B
73
+ with torch.no_grad(), torch.autocast("cuda", dtype=torch.bfloat16):
74
+ texts = text(["[Lungs] - A 9 mm solid nodule in the right upper lobe.", "[Lungs] - The lungs are clear."])
75
+ print(model.similarity(out["embedding"].float(), texts.float()))
76
+ ```
77
+
78
+ The model returns `embedding` (the 1,024-d retrieval vector), `latent_embeddings` (all 11 latents: 10 anatomy regions and
79
+ the global one), `pooled`, `s4_tokens` (10³ = 1,000 tokens × 1,024) and `s4_mean`. Scores follow SigLIP:
80
+ `sigmoid(exp(logit_scale_log) · cos + logit_bias)`.
81
+
82
+ ## Results
83
+
84
+ CT-RATE validation, 1,564 scans and 1,503 distinct reports (identical reports count as matches), evaluated at the end of
85
+ alignment training:
86
+
87
+ | task | R@1 | R@5 | R@10 |
88
+ |---|---|---|---|
89
+ | image-to-report retrieval | 0.330 | 0.580 | 0.668 |
90
+ | report-to-image retrieval | 0.359 | 0.593 | 0.685 |
91
+
92
+ ## Training data
93
+
94
+ Encoder: JECT pretraining at 224³ on about 173,000 chest CTs with reports (CT-RATE and a private institutional cohort),
95
+ then a cooldown at 320³ on 1 mm volumes. Alignment: CT-RATE training pairs (findings section), all encoder weights trained.
96
+
97
+ ## Limitations
98
+
99
+ Trained on adult chest CT; reports in English. Not a medical device and not validated for clinical use. Retrieval scores
100
+ reflect CT-RATE-style findings text; other report styles may score lower. The model expects the input contract above;
101
+ other spacings, crops or orientations run without error but give wrong features.
102
+
103
+ ## Files
104
+
105
+ | file | content |
106
+ |---|---|
107
+ | `config.json` | encoder architecture, readout and text settings, SigLIP scale and bias |
108
+ | `model.safetensors` | image encoder, latent-attention pooler and projection |
109
+ | `text_adapter.safetensors` | LoRA and projection for chest2vec 0.6B |
110
+ | `configuration_ject.py`, `modeling_ject.py`, `ject_encoder.py`, `preprocess.py` | model code (`AutoModel`, `trust_remote_code=True`) |
111
+ | `prepare.py` | NIfTI and npz input preparation |
112
+
113
+ ## Citation
114
+
115
+ ```bibtex
116
+ @misc{ject2026,
117
+ title = {{JECT}: Addressing the Context Shortcut in Joint-Embedding Predictive Learning for Chest {CT}},
118
+ author = {TBD},
119
+ year = {2026}
120
+ }
121
+ ```
122
+
123
+ ## License
124
+
125
+ Model weights: CC BY-NC-SA 4.0 (non-commercial use only; see `LICENSE`). Code: MIT (see `LICENSE-CODE`).
config.json ADDED
@@ -0,0 +1,105 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_type": "ject-ct",
3
+ "encoder": {
4
+ "input_size": [
5
+ 320,
6
+ 320,
7
+ 320
8
+ ],
9
+ "patch_size": [
10
+ 4,
11
+ 4,
12
+ 4
13
+ ],
14
+ "in_channels": 1,
15
+ "variant": "huge",
16
+ "local_block_type": "swin",
17
+ "stage3_block_type": "swin",
18
+ "stage4_block_type": "global",
19
+ "window": [
20
+ 7,
21
+ 7,
22
+ 7
23
+ ],
24
+ "mlp_ratio": 4.0,
25
+ "drop": 0.0,
26
+ "num_ct_types": 3,
27
+ "skip_fine": true,
28
+ "skip_pool_s3": "attention",
29
+ "skip_pool_s2s4": "attention",
30
+ "dims": [
31
+ 128,
32
+ 256,
33
+ 512,
34
+ 1024
35
+ ],
36
+ "heads": [
37
+ 8,
38
+ 16,
39
+ 32,
40
+ 32
41
+ ],
42
+ "depths": [
43
+ 2,
44
+ 4,
45
+ 9,
46
+ 9
47
+ ],
48
+ "grid_s1": [
49
+ 80,
50
+ 80,
51
+ 80
52
+ ],
53
+ "grid_s2": [
54
+ 40,
55
+ 40,
56
+ 40
57
+ ],
58
+ "grid_s3": [
59
+ 20,
60
+ 20,
61
+ 20
62
+ ],
63
+ "grid_s4": [
64
+ 10,
65
+ 10,
66
+ 10
67
+ ]
68
+ },
69
+ "readout": {
70
+ "embed_dim": 1024,
71
+ "pool_latents": 11,
72
+ "pool_heads": 8,
73
+ "global_latent": 10
74
+ },
75
+ "input": {
76
+ "crop_size": 320,
77
+ "cube_size": 384,
78
+ "spacing_mm": 1.0,
79
+ "window_id": 2
80
+ },
81
+ "text": {
82
+ "base_model": "chest2vec/chest2vec_0.6B",
83
+ "lora_r": 16,
84
+ "lora_alpha": 32,
85
+ "lora_targets": [
86
+ "q_proj",
87
+ "k_proj",
88
+ "v_proj",
89
+ "o_proj",
90
+ "gate_proj",
91
+ "up_proj",
92
+ "down_proj"
93
+ ],
94
+ "max_length": 1024
95
+ },
96
+ "logit_scale_log": 3.847844123840332,
97
+ "logit_bias": -10.863999366760254,
98
+ "architectures": [
99
+ "JECTImageEncoderModel"
100
+ ],
101
+ "auto_map": {
102
+ "AutoConfig": "configuration_ject.JECTConfig",
103
+ "AutoModel": "modeling_ject.JECTImageEncoderModel"
104
+ }
105
+ }
configuration_ject.py ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Configuration for the JECT chest-CT encoder."""
2
+ from transformers import PretrainedConfig
3
+
4
+
5
+ class JECTConfig(PretrainedConfig):
6
+ model_type = "ject-ct"
7
+
8
+ def __init__(self, encoder=None, readout=None, input=None, text=None, logit_scale_log=0.0, logit_bias=0.0, **kwargs):
9
+ self.encoder = encoder or {}
10
+ self.readout = readout or {}
11
+ self.input = input or {}
12
+ self.text = text or {}
13
+ self.logit_scale_log = logit_scale_log
14
+ self.logit_bias = logit_bias
15
+ super().__init__(**kwargs)
ject_encoder.py ADDED
@@ -0,0 +1,734 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """JECT image encoder: standalone architecture definition.
2
+
3
+ Depends only on ``torch``. Defines the 3D Swin-UNETR-style hierarchical encoder,
4
+ the dense latent decoder on top of it, and the config needed to rebuild both from
5
+ a training checkpoint. Training-only components (EMA, predictors, text heads,
6
+ losses) are not included.
7
+
8
+ Encoder layout (224 input, patch 4, `huge` variant)::
9
+
10
+ volume [B,1,224,224,224]
11
+ -> PatchEmbed3D (conv4x4x4) -> e0 56^3 x 128
12
+ -> stage1 (2x swin, window 7) -> e1 56^3 x 128
13
+ -> PatchMerge -> stage2 (4x swin) -> e2 28^3 x 256
14
+ -> PatchMerge -> +pos_embed -> stage3 (9x swin) -> e3 14^3 x 512
15
+ -> PatchMerge -> +pos_embed_s4 -> stage4 (9x global attn) -> e4 7^3 x 1024
16
+
17
+ At 336 input the grids are 84/42/21/11. The learned s3/s4 absolute position
18
+ embeddings are resolution-specific, so each input size has its own checkpoint.
19
+ """
20
+
21
+ from __future__ import annotations
22
+
23
+ from dataclasses import dataclass, field
24
+ from typing import Dict, Optional, Tuple
25
+
26
+ import torch
27
+ import torch.nn as nn
28
+ import torch.nn.functional as F
29
+
30
+ Triple = Tuple[int, int, int]
31
+
32
+ # Per-variant (dims, heads, depths) for scales (s1, s2, s3, s4).
33
+ VARIANTS = {
34
+ "tiny": dict(dims=(32, 64, 128, 256), heads=(2, 4, 8, 8), depths=(2, 2, 2, 2)),
35
+ "small": dict(dims=(48, 96, 192, 384), heads=(3, 6, 12, 12), depths=(2, 2, 4, 2)),
36
+ "base": dict(dims=(64, 128, 256, 512), heads=(4, 8, 16, 16), depths=(2, 2, 4, 2)),
37
+ "large": dict(dims=(96, 192, 384, 768), heads=(6, 12, 24, 24), depths=(2, 2, 8, 4)),
38
+ "huge": dict(dims=(128, 256, 512, 1024), heads=(8, 16, 32, 32), depths=(2, 4, 9, 9)),
39
+ }
40
+
41
+ # HU windows the encoder was trained with. `window_id` is an additive embedding on the
42
+ # patch tokens, so the id passed at inference must match the window actually applied.
43
+ WINDOW_PARAMS = {
44
+ 0: {"center": 450, "width": 2100}, # lung: -600 .. 1500
45
+ 1: {"center": 195, "width": 310}, # mediastinal: 40 .. 350
46
+ 2: {"center": 0, "width": 2000}, # general: -1000 .. 1000 (default)
47
+ 3: {"center": 1500, "width": 5000}, # full: -1000 .. 4000
48
+ }
49
+ GENERAL_WINDOW_ID = 2
50
+
51
+ # ct_type to additive embedding index.
52
+ CT_TYPE_MAP = {"non-contrast": 0, "contrast": 1, "ldct": 2}
53
+ DEFAULT_CT_TYPE = 0
54
+
55
+
56
+ def _triple(x) -> Triple:
57
+ if isinstance(x, int):
58
+ return (x, x, x)
59
+ vals = tuple(int(v) for v in x)
60
+ if len(vals) == 1:
61
+ return (vals[0],) * 3
62
+ if len(vals) != 3:
63
+ raise ValueError(f"expected an int or a 1/3-length iterable, got {x!r}")
64
+ return vals
65
+
66
+
67
+ # ---------------------------------------------------------------------------
68
+ # Config
69
+ # ---------------------------------------------------------------------------
70
+ @dataclass
71
+ class EncoderConfig:
72
+ """Encoder geometry and width; normally built with ``from_training_config``."""
73
+
74
+ input_size: Triple = (224, 224, 224)
75
+ patch_size: Triple = (4, 4, 4)
76
+ in_channels: int = 1
77
+ variant: str = "huge"
78
+
79
+ local_block_type: str = "swin" # stages 1-2
80
+ stage3_block_type: str = "swin"
81
+ stage4_block_type: str = "global" # full attention over the 7^3 / 11^3 grid
82
+ window: Triple = (7, 7, 7)
83
+ mlp_ratio: float = 4.0
84
+ drop: float = 0.0
85
+ num_ct_types: int = 3
86
+ # Cumulative bottom-up skips of pooled finer features into the s2/s3/s4 stage entries.
87
+ skip_fine: bool = False
88
+ # Pooling for the s3 skip: "maxmean" ([max, mean] cell pools) or "attention" (LocalAttentionPool3D).
89
+ skip_pool_s3: str = "maxmean"
90
+ # Pooling for the s2 and s4 skips: "maxmean" or "attention".
91
+ skip_pool_s2s4: str = "maxmean"
92
+
93
+ # filled by __post_init__
94
+ dims: Tuple[int, ...] = field(default_factory=tuple)
95
+ heads: Tuple[int, ...] = field(default_factory=tuple)
96
+ depths: Tuple[int, ...] = field(default_factory=tuple)
97
+ grid_s1: Triple = (0, 0, 0)
98
+ grid_s2: Triple = (0, 0, 0)
99
+ grid_s3: Triple = (0, 0, 0)
100
+ grid_s4: Triple = (0, 0, 0)
101
+
102
+ def __post_init__(self):
103
+ self.input_size = _triple(self.input_size)
104
+ self.patch_size = _triple(self.patch_size)
105
+ self.window = _triple(self.window)
106
+ if self.variant not in VARIANTS:
107
+ raise ValueError(f"unknown variant {self.variant!r}; expected one of {sorted(VARIANTS)}")
108
+ spec = VARIANTS[self.variant]
109
+ if self.skip_pool_s3 not in ("maxmean", "attention"):
110
+ raise ValueError(f"unknown skip_pool_s3 {self.skip_pool_s3!r}; expected 'maxmean' or 'attention'")
111
+ if self.skip_pool_s3 != "maxmean" and not self.skip_fine:
112
+ raise ValueError("skip_pool_s3='attention' requires skip_fine (mirrors pretrain)")
113
+ if self.skip_pool_s2s4 not in ("maxmean", "attention"):
114
+ raise ValueError(f"unknown skip_pool_s2s4 {self.skip_pool_s2s4!r}; expected 'maxmean' or 'attention'")
115
+ if self.skip_pool_s2s4 != "maxmean" and not self.skip_fine:
116
+ raise ValueError("skip_pool_s2s4='attention' requires skip_fine (mirrors pretrain)")
117
+ self.dims = tuple(spec["dims"])
118
+ self.heads = tuple(spec["heads"])
119
+ self.depths = tuple(spec["depths"])
120
+ for name, v, p in zip("DHW", self.input_size, self.patch_size):
121
+ if v % p:
122
+ raise ValueError(f"input_size {name}={v} not divisible by patch_size {p}")
123
+ s1 = tuple(v // p for v, p in zip(self.input_size, self.patch_size))
124
+ s2 = tuple(g // 2 for g in s1)
125
+ s3 = tuple(g // 4 for g in s1)
126
+ # PatchMerge3D pads odd grids before halving, so s4 = ceil(s3 / 2) (e.g. s3 21 gives s4 11).
127
+ s4 = tuple((g + 1) // 2 for g in s3)
128
+ self.grid_s1, self.grid_s2, self.grid_s3, self.grid_s4 = s1, s2, s3, s4
129
+
130
+ # -- convenience ---------------------------------------------------------
131
+ @property
132
+ def num_tokens(self) -> Dict[str, int]:
133
+ return {f"s{i}": g[0] * g[1] * g[2]
134
+ for i, g in enumerate((self.grid_s1, self.grid_s2, self.grid_s3, self.grid_s4), 1)}
135
+
136
+ @property
137
+ def grids(self) -> Dict[str, Triple]:
138
+ return {"s1": self.grid_s1, "s2": self.grid_s2, "s3": self.grid_s3, "s4": self.grid_s4}
139
+
140
+ @property
141
+ def channels(self) -> Dict[str, int]:
142
+ return {f"s{i}": d for i, d in enumerate(self.dims, 1)}
143
+
144
+ @classmethod
145
+ def from_training_config(cls, saved: dict) -> "EncoderConfig":
146
+ """Rebuild from the `config` dict stored inside a JECT training checkpoint.
147
+
148
+ The input size is derived from ``grid_size_s1 * patch_size`` rather than the ``volume``
149
+ block, and the derived grids and widths are checked against the saved ones.
150
+ """
151
+ patch = saved.get("patch", {})
152
+ model = saved.get("model", {})
153
+ psize = _triple(patch.get("patch_size", (4, 4, 4)))
154
+ g1 = _triple(patch["grid_size_s1"])
155
+ cfg = cls(
156
+ input_size=tuple(g * p for g, p in zip(g1, psize)),
157
+ patch_size=psize,
158
+ in_channels=int(saved.get("volume", {}).get("channels", 1)),
159
+ variant=str(model.get("variant", "huge")),
160
+ local_block_type=str(model.get("local_block_type", "swin")),
161
+ stage3_block_type=str(model.get("stage3_block_type", "swin")),
162
+ stage4_block_type=str(model.get("stage4_block_type", "global")),
163
+ window=_triple(model.get("swin_window_size", (7, 7, 7))),
164
+ mlp_ratio=float(model.get("mlp_ratio", 4.0)),
165
+ drop=0.0, # no dropout at inference
166
+ num_ct_types=int(model.get("num_ct_types", 3)),
167
+ skip_fine=bool(model.get("skip_fine", False)),
168
+ skip_pool_s3=str(model.get("skip_pool_s3", "maxmean")),
169
+ skip_pool_s2s4=str(model.get("skip_pool_s2s4", "maxmean")),
170
+ )
171
+ for name, derived in (("grid_size_s2", cfg.grid_s2), ("grid_size_s3", cfg.grid_s3),
172
+ ("grid_size_s4", cfg.grid_s4)):
173
+ if name in patch and _triple(patch[name]) != derived:
174
+ raise ValueError(f"geometry mismatch: checkpoint {name}={tuple(patch[name])} "
175
+ f"but input_size {cfg.input_size} implies {derived}")
176
+ for key, want in (("dim_s1", cfg.dims[0]), ("dim_s2", cfg.dims[1]),
177
+ ("dim_s3", cfg.dims[2]), ("dim_s4", cfg.dims[3])):
178
+ if key in model and int(model[key]) != want:
179
+ raise ValueError(f"width mismatch: checkpoint {key}={model[key]} but "
180
+ f"variant {cfg.variant!r} implies {want}")
181
+ return cfg
182
+
183
+
184
+ # ---------------------------------------------------------------------------
185
+ # Shared pieces
186
+ # ---------------------------------------------------------------------------
187
+ class Mlp(nn.Module):
188
+ def __init__(self, dim: int, mlp_ratio: float = 4.0, drop: float = 0.0):
189
+ super().__init__()
190
+ hidden = int(dim * mlp_ratio)
191
+ self.fc1 = nn.Linear(dim, hidden)
192
+ self.act = nn.GELU()
193
+ self.fc2 = nn.Linear(hidden, dim)
194
+ self.drop = nn.Dropout(drop)
195
+
196
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
197
+ return self.drop(self.fc2(self.drop(self.act(self.fc1(x)))))
198
+
199
+
200
+ def to_grid(x: torch.Tensor, grid: Triple) -> torch.Tensor:
201
+ """[B, N, C] -> [B, C, D, H, W]."""
202
+ B, N, C = x.shape
203
+ D, H, W = grid
204
+ if N != D * H * W:
205
+ raise ValueError(f"token count {N} does not match grid {grid} ({D * H * W})")
206
+ return x.transpose(1, 2).reshape(B, C, D, H, W)
207
+
208
+
209
+ def to_tokens(x: torch.Tensor) -> torch.Tensor:
210
+ """[B, C, D, H, W] -> [B, N, C]."""
211
+ return x.flatten(2).transpose(1, 2)
212
+
213
+
214
+ class Attention(nn.Module):
215
+ """Multi-head self-attention over all tokens (no positional bias)."""
216
+
217
+ def __init__(self, dim: int, num_heads: int, attn_drop: float = 0.0, proj_drop: float = 0.0):
218
+ super().__init__()
219
+ if dim % num_heads:
220
+ raise ValueError(f"dim={dim} not divisible by num_heads={num_heads}")
221
+ self.num_heads = num_heads
222
+ self.head_dim = dim // num_heads
223
+ self.qkv = nn.Linear(dim, dim * 3)
224
+ self.attn_drop_p = float(attn_drop)
225
+ self.proj = nn.Linear(dim, dim)
226
+ self.proj_drop = nn.Dropout(proj_drop)
227
+
228
+ def forward(self, x: torch.Tensor, bias: Optional[torch.Tensor] = None) -> torch.Tensor:
229
+ B, N, C = x.shape
230
+ qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, self.head_dim).permute(2, 0, 3, 1, 4)
231
+ q, k, v = qkv.unbind(0)
232
+ attn_mask = bias.unsqueeze(0).to(q.dtype) if bias is not None else None
233
+ out = F.scaled_dot_product_attention(
234
+ q, k, v, attn_mask=attn_mask, dropout_p=self.attn_drop_p if self.training else 0.0)
235
+ out = out.transpose(1, 2).reshape(B, N, C)
236
+ return self.proj_drop(self.proj(out))
237
+
238
+
239
+ class TransformerBlock3D(nn.Module):
240
+ """Pre-norm global transformer block (stage 4)."""
241
+
242
+ def __init__(self, dim: int, num_heads: int, mlp_ratio: float = 4.0, drop: float = 0.0):
243
+ super().__init__()
244
+ self.norm1 = nn.LayerNorm(dim)
245
+ self.attn = Attention(dim, num_heads, attn_drop=drop, proj_drop=drop)
246
+ self.norm2 = nn.LayerNorm(dim)
247
+ self.mlp = Mlp(dim, mlp_ratio, drop)
248
+
249
+ def forward(self, x: torch.Tensor, grid: Triple) -> torch.Tensor:
250
+ x = x + self.attn(self.norm1(x))
251
+ x = x + self.mlp(self.norm2(x))
252
+ return x
253
+
254
+
255
+ def _pad_to_window(x: torch.Tensor, window: Triple):
256
+ """Pad a [B, D, H, W, C] grid up to a multiple of the window on each spatial axis."""
257
+ _, D, H, W, _ = x.shape
258
+ wd, wh, ww = window
259
+ pd, ph, pw = (wd - D % wd) % wd, (wh - H % wh) % wh, (ww - W % ww) % ww
260
+ if pd or ph or pw:
261
+ x = F.pad(x, (0, 0, 0, pw, 0, ph, 0, pd)) # pads W, H, D (C last, untouched)
262
+ return x, (D, H, W)
263
+
264
+
265
+ def _win_partition(x: torch.Tensor, window: Triple) -> torch.Tensor:
266
+ B, D, H, W, C = x.shape
267
+ wd, wh, ww = window
268
+ x = x.view(B, D // wd, wd, H // wh, wh, W // ww, ww, C)
269
+ x = x.permute(0, 1, 3, 5, 2, 4, 6, 7).contiguous()
270
+ return x.view(-1, wd * wh * ww, C)
271
+
272
+
273
+ def _win_merge(windows: torch.Tensor, gridp: Triple, window: Triple, B: int) -> torch.Tensor:
274
+ Dp, Hp, Wp = gridp
275
+ wd, wh, ww = window
276
+ C = windows.shape[-1]
277
+ x = windows.view(B, Dp // wd, Hp // wh, Wp // ww, wd, wh, ww, C)
278
+ x = x.permute(0, 1, 4, 2, 5, 3, 6, 7).contiguous()
279
+ return x.view(B, Dp, Hp, Wp, C)
280
+
281
+
282
+ class WindowAttention3D(nn.Module):
283
+ """Windowed multi-head attention with a 3D relative-position-bias table."""
284
+
285
+ def __init__(self, dim: int, num_heads: int, window: Triple,
286
+ attn_drop: float = 0.0, proj_drop: float = 0.0):
287
+ super().__init__()
288
+ if dim % num_heads:
289
+ raise ValueError(f"dim={dim} not divisible by num_heads={num_heads}")
290
+ self.num_heads = num_heads
291
+ self.head_dim = dim // num_heads
292
+ self.window = window
293
+ wd, wh, ww = window
294
+ self.relative_position_bias_table = nn.Parameter(
295
+ torch.zeros((2 * wd - 1) * (2 * wh - 1) * (2 * ww - 1), num_heads))
296
+ coords = torch.stack(torch.meshgrid(
297
+ torch.arange(wd), torch.arange(wh), torch.arange(ww), indexing="ij"))
298
+ coords_flat = coords.reshape(3, -1)
299
+ rel = (coords_flat[:, :, None] - coords_flat[:, None, :]).permute(1, 2, 0).contiguous()
300
+ rel[..., 0] += wd - 1
301
+ rel[..., 1] += wh - 1
302
+ rel[..., 2] += ww - 1
303
+ rel[..., 0] *= (2 * wh - 1) * (2 * ww - 1)
304
+ rel[..., 1] *= (2 * ww - 1)
305
+ self.register_buffer("relative_position_index", rel.sum(-1).view(-1), persistent=False)
306
+ self.qkv = nn.Linear(dim, dim * 3)
307
+ self.attn_drop_p = float(attn_drop)
308
+ self.proj = nn.Linear(dim, dim)
309
+ self.proj_drop = nn.Dropout(proj_drop)
310
+
311
+ def _bias(self) -> torch.Tensor:
312
+ ws = int(round(self.relative_position_index.numel() ** 0.5))
313
+ table = self.relative_position_bias_table[self.relative_position_index]
314
+ return table.view(ws, ws, -1).permute(2, 0, 1).contiguous() # [heads, ws, ws]
315
+
316
+ def forward(self, windows: torch.Tensor) -> torch.Tensor:
317
+ Bn, ws, C = windows.shape
318
+ qkv = self.qkv(windows).reshape(Bn, ws, 3, self.num_heads, self.head_dim).permute(2, 0, 3, 1, 4)
319
+ q, k, v = qkv.unbind(0)
320
+ attn_mask = self._bias().unsqueeze(0).to(q.dtype)
321
+ out = F.scaled_dot_product_attention(
322
+ q, k, v, attn_mask=attn_mask, dropout_p=self.attn_drop_p if self.training else 0.0)
323
+ out = out.transpose(1, 2).reshape(Bn, ws, C)
324
+ return self.proj_drop(self.proj(out))
325
+
326
+
327
+ class SwinBlock3D(nn.Module):
328
+ """Shifted-window 3D attention block; pads internally so any grid size works."""
329
+
330
+ def __init__(self, dim: int, num_heads: int, window: Triple,
331
+ mlp_ratio: float = 4.0, drop: float = 0.0, shift: bool = False):
332
+ super().__init__()
333
+ self.window = window
334
+ self.shift_size = tuple(w // 2 for w in window) if shift else (0, 0, 0)
335
+ self.norm1 = nn.LayerNorm(dim)
336
+ self.attn = WindowAttention3D(dim, num_heads, window, attn_drop=drop, proj_drop=drop)
337
+ self.norm2 = nn.LayerNorm(dim)
338
+ self.mlp = Mlp(dim, mlp_ratio, drop)
339
+
340
+ def forward(self, x: torch.Tensor, grid: Triple) -> torch.Tensor:
341
+ B, N, C = x.shape
342
+ D, H, W = grid
343
+ h = self.norm1(x).view(B, D, H, W, C)
344
+ h, _ = _pad_to_window(h, self.window)
345
+ Dp, Hp, Wp = h.shape[1:4]
346
+ if any(s > 0 for s in self.shift_size):
347
+ h = torch.roll(h, shifts=tuple(-s for s in self.shift_size), dims=(1, 2, 3))
348
+ windows = self.attn(_win_partition(h, self.window))
349
+ h = _win_merge(windows, (Dp, Hp, Wp), self.window, B)
350
+ if any(s > 0 for s in self.shift_size):
351
+ h = torch.roll(h, shifts=self.shift_size, dims=(1, 2, 3))
352
+ h = h[:, :D, :H, :W].reshape(B, N, C) # crop the window padding
353
+ x = x + h
354
+ x = x + self.mlp(self.norm2(x))
355
+ return x
356
+
357
+
358
+ class ConvNeXtBlock3D(nn.Module):
359
+ """Depthwise 3D ConvNeXt block."""
360
+
361
+ def __init__(self, dim: int, mlp_ratio: float = 4.0, drop: float = 0.0):
362
+ super().__init__()
363
+ self.dwconv = nn.Conv3d(dim, dim, kernel_size=7, padding=3, groups=dim)
364
+ self.norm = nn.LayerNorm(dim)
365
+ hidden = int(dim * mlp_ratio)
366
+ self.pw1 = nn.Linear(dim, hidden)
367
+ self.act = nn.GELU()
368
+ self.pw2 = nn.Linear(hidden, dim)
369
+ self.gamma = nn.Parameter(1e-6 * torch.ones(dim))
370
+
371
+ def forward(self, x: torch.Tensor, grid: Triple) -> torch.Tensor:
372
+ y = to_tokens(self.dwconv(to_grid(x, grid)))
373
+ y = self.pw2(self.act(self.pw1(self.norm(y))))
374
+ return x + self.gamma * y
375
+
376
+
377
+ def build_block(block_type: str, dim: int, num_heads: int, window: Triple,
378
+ mlp_ratio: float, drop: float, shift: bool = False) -> nn.Module:
379
+ if block_type == "swin":
380
+ return SwinBlock3D(dim, num_heads, window, mlp_ratio, drop, shift=shift)
381
+ if block_type in ("transformer", "global"):
382
+ return TransformerBlock3D(dim, num_heads, mlp_ratio, drop)
383
+ if block_type == "convnext":
384
+ return ConvNeXtBlock3D(dim, mlp_ratio, drop)
385
+ raise ValueError(f"unknown block type {block_type!r}")
386
+
387
+
388
+ class PatchEmbed3D(nn.Module):
389
+ def __init__(self, channels: int, dim: int, patch_size: Triple):
390
+ super().__init__()
391
+ self.proj = nn.Conv3d(channels, dim, kernel_size=patch_size, stride=patch_size)
392
+ self.norm = nn.LayerNorm(dim)
393
+
394
+ def forward(self, volume: torch.Tensor) -> torch.Tensor:
395
+ return self.norm(to_tokens(self.proj(volume)))
396
+
397
+
398
+ def _pool_fine(x: torch.Tensor, grid: Triple, grid_to: Triple) -> torch.Tensor:
399
+ """Max- and mean-pool z-major tokens on `grid` into the cells of `grid_to`.
400
+
401
+ [B, N, C] -> [B, N_to, 2C], pooling the k^3 fine tokens of each coarse cell (k = grid / grid_to)."""
402
+ B, N, C = x.shape
403
+ D, H, W = grid
404
+ k = D // grid_to[0]
405
+ if (D, H, W) != (k * grid_to[0], k * grid_to[1], k * grid_to[2]):
406
+ raise ValueError(f"skip_fine needs grid {grid} to be an integer multiple of the coarser grid {grid_to}")
407
+ v = x.transpose(1, 2).reshape(B, C, D, H, W)
408
+ return torch.cat([F.max_pool3d(v, k), F.avg_pool3d(v, k)], dim=1).flatten(2).transpose(1, 2)
409
+
410
+
411
+ class LocalAttentionPool3D(nn.Module):
412
+ """Coarse-query attention over one cell's fine tokens, with a learned 3D-offset bias.
413
+
414
+ Four heads, 128 internal channels, and the same 2C output width as max/mean pooling.
415
+ Token order within each cell is z/y/x, so each bias index is a distinct relative 3D
416
+ position. Attention never crosses cells.
417
+ """
418
+
419
+ SDPA_CHUNK = 32768 # max cells per SDPA call
420
+
421
+ def __init__(self, fine_dim: int, coarse_dim: int, factor: int):
422
+ super().__init__()
423
+ self.factor = factor
424
+ self.heads, self.head_dim = 4, 32
425
+ self.fine_norm = nn.LayerNorm(fine_dim)
426
+ self.query_norm = nn.LayerNorm(coarse_dim)
427
+ self.query = nn.Linear(coarse_dim, 128)
428
+ self.key_value = nn.Linear(fine_dim, 256)
429
+ self.relative_bias = nn.Parameter(torch.zeros(4, factor ** 3))
430
+ self.output = nn.Linear(128, 2 * fine_dim)
431
+
432
+ def forward(self, fine: torch.Tensor, coarse: torch.Tensor, grid: Triple,
433
+ grid_to: Triple) -> torch.Tensor:
434
+ B, N, C = fine.shape
435
+ D, H, W = grid_to
436
+ k = self.factor
437
+ if tuple(grid) != (D * k, H * k, W * k) or N != D * H * W * k ** 3:
438
+ raise ValueError("attention skip requires exactly nested fine/coarse grids")
439
+ if coarse.shape[:2] != (B, D * H * W):
440
+ raise ValueError("attention skip query count does not match its coarse grid")
441
+ cells = fine.reshape(B, D, k, H, k, W, k, C)
442
+ cells = cells.permute(0, 1, 3, 5, 2, 4, 6, 7).reshape(B * D * H * W, k ** 3, C)
443
+ kv = self.key_value(self.fine_norm(cells))
444
+ kv = kv.reshape(-1, k ** 3, 2, self.heads, self.head_dim).permute(2, 0, 3, 1, 4)
445
+ q = self.query(self.query_norm(coarse)).reshape(-1, self.heads, 1, self.head_dim)
446
+ bias = self.relative_bias.to(q.dtype)[None, :, None, :]
447
+ # One SDPA batch entry per cell (B*D*H*W). Above ~65k entries the fused kernel exceeds the CUDA
448
+ # launch-grid limit, so large inputs are processed in chunks; the result is unchanged.
449
+ n = q.shape[0]
450
+ if n <= self.SDPA_CHUNK:
451
+ pooled = F.scaled_dot_product_attention(q, kv[0], kv[1], attn_mask=bias, dropout_p=0.0)
452
+ else:
453
+ pooled = torch.cat([F.scaled_dot_product_attention(q[i:i + self.SDPA_CHUNK], kv[0][i:i + self.SDPA_CHUNK],
454
+ kv[1][i:i + self.SDPA_CHUNK], attn_mask=bias, dropout_p=0.0)
455
+ for i in range(0, n, self.SDPA_CHUNK)], dim=0)
456
+ return self.output(pooled.reshape(B, D * H * W, 128))
457
+
458
+
459
+ class PatchMerge3D(nn.Module):
460
+ """2x2x2 patch merging: [B, N, C] -> [B, N/8, out_dim] (odd grids are padded first)."""
461
+
462
+ def __init__(self, in_dim: int, out_dim: int):
463
+ super().__init__()
464
+ self.norm = nn.LayerNorm(8 * in_dim)
465
+ self.reduce = nn.Linear(8 * in_dim, out_dim, bias=False)
466
+
467
+ def forward(self, x: torch.Tensor, grid: Triple) -> Tuple[torch.Tensor, Triple]:
468
+ B, N, C = x.shape
469
+ D, H, W = grid
470
+ if D % 2 or H % 2 or W % 2:
471
+ x = x.view(B, D, H, W, C)
472
+ x = F.pad(x, (0, 0, 0, W % 2, 0, H % 2, 0, D % 2))
473
+ D, H, W = D + D % 2, H + H % 2, W + W % 2
474
+ x = x.reshape(B, D * H * W, C)
475
+ x = x.view(B, D // 2, 2, H // 2, 2, W // 2, 2, C)
476
+ x = x.permute(0, 1, 3, 5, 2, 4, 6, 7).contiguous()
477
+ x = x.view(B, (D // 2) * (H // 2) * (W // 2), 8 * C)
478
+ return self.reduce(self.norm(x)), (D // 2, H // 2, W // 2)
479
+
480
+
481
+ # ---------------------------------------------------------------------------
482
+ # Encoder / dense decoder
483
+ # ---------------------------------------------------------------------------
484
+ class JECTEncoder(nn.Module):
485
+ """Hierarchical 3D encoder producing token features at the s1/s2/s3/s4 grids.
486
+
487
+ Module and parameter names match the pretraining checkpoint exactly, so
488
+ ``state_dict`` keys load without remapping (strip the ``target_encoder.``
489
+ or ``student_encoder.`` prefix first).
490
+ """
491
+
492
+ def __init__(self, config: EncoderConfig):
493
+ super().__init__()
494
+ self.config = config
495
+ d1, d2, d3, d4 = config.dims
496
+ h1, h2, h3, h4 = config.heads
497
+ n1, n2, n3, n4 = config.depths
498
+ win = config.window
499
+ self.grid_s1, self.grid_s2 = config.grid_s1, config.grid_s2
500
+ self.grid_s3, self.grid_s4 = config.grid_s3, config.grid_s4
501
+
502
+ self.patch_embed = PatchEmbed3D(config.in_channels, d1, config.patch_size)
503
+ self.window_embed = nn.Embedding(4, d1) # HU-window id (see WINDOW_PARAMS)
504
+ self.ct_type_embed = nn.Embedding(config.num_ct_types, d1)
505
+ self.mask_token = nn.Parameter(torch.zeros(1, 1, d1)) # pretraining only; unused here
506
+
507
+ self.stage1 = nn.ModuleList([
508
+ build_block(config.local_block_type, d1, h1, win, config.mlp_ratio, config.drop,
509
+ shift=(i % 2 == 1)) for i in range(n1)])
510
+ self.merge1 = PatchMerge3D(d1, d2)
511
+ self.stage2 = nn.ModuleList([
512
+ build_block(config.local_block_type, d2, h2, win, config.mlp_ratio, config.drop,
513
+ shift=(i % 2 == 1)) for i in range(n2)])
514
+ self.merge2 = PatchMerge3D(d2, d3)
515
+ # skip_fine: at each stage entry, a projection of pooled features from every finer representation
516
+ # (patch embeddings, e1, e2, e3).
517
+ self.skip_fine = bool(config.skip_fine)
518
+ if self.skip_fine:
519
+ self.skip2_norm = nn.LayerNorm(2 * (d1 + d1)); self.skip2_proj = nn.Linear(2 * (d1 + d1), d2)
520
+ self.skip3_norm = nn.LayerNorm(2 * (d1 + d1 + d2)); self.skip3_proj = nn.Linear(2 * (d1 + d1 + d2), d3)
521
+ self.skip4_norm = nn.LayerNorm(2 * (d1 + d1 + d2 + d3)); self.skip4_proj = nn.Linear(2 * (d1 + d1 + d2 + d3), d4)
522
+ # skip_pool_s3="attention": one LocalAttentionPool3D per s3 skip source instead of max/mean pooling.
523
+ self.skip_pool_s3 = str(config.skip_pool_s3)
524
+ if self.skip_fine and self.skip_pool_s3 == "attention":
525
+ self.skip3_attention = nn.ModuleList([
526
+ LocalAttentionPool3D(d1, d3, 4),
527
+ LocalAttentionPool3D(d1, d3, 4),
528
+ LocalAttentionPool3D(d2, d3, 2),
529
+ ])
530
+ # skip_pool_s2s4="attention": the same per-cell attention pooling at the s2 and s4 entries.
531
+ self.skip_pool_s2s4 = str(getattr(config, "skip_pool_s2s4", "maxmean"))
532
+ if self.skip_fine and self.skip_pool_s2s4 == "attention":
533
+ self.skip2_attention = nn.ModuleList([
534
+ LocalAttentionPool3D(d1, d2, 2),
535
+ LocalAttentionPool3D(d1, d2, 2),
536
+ ])
537
+ self.skip4_attention = nn.ModuleList([
538
+ LocalAttentionPool3D(d1, d4, 8),
539
+ LocalAttentionPool3D(d1, d4, 8),
540
+ LocalAttentionPool3D(d2, d4, 4),
541
+ LocalAttentionPool3D(d3, d4, 2),
542
+ ])
543
+ self.pos_embed = nn.Parameter(torch.zeros(1, config.num_tokens["s3"], d3))
544
+ self.stage3 = nn.ModuleList([
545
+ build_block(config.stage3_block_type, d3, h3, win, config.mlp_ratio, config.drop,
546
+ shift=(i % 2 == 1)) for i in range(n3)])
547
+ self.merge3 = PatchMerge3D(d3, d4)
548
+ self.pos_embed_s4 = nn.Parameter(torch.zeros(1, config.num_tokens["s4"], d4))
549
+ self.stage4 = nn.ModuleList([
550
+ build_block(config.stage4_block_type, d4, h4, win, config.mlp_ratio, config.drop,
551
+ shift=(i % 2 == 1)) for i in range(n4)])
552
+
553
+ def trunk(self, volume: torch.Tensor,
554
+ window_id: Optional[torch.Tensor] = None,
555
+ ct_type_id: Optional[torch.Tensor] = None,
556
+ mask_s1: Optional[torch.Tensor] = None) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
557
+ """Run stages 1-3 and merge3 (with skips), stopping before pos_embed_s4 and stage 4.
558
+
559
+ Useful for fine-tuning only stage 4. Returns ``(x_s4_in, e1, e2, e3)``.
560
+ """
561
+ expected = self.config.input_size
562
+ if tuple(volume.shape[-3:]) != tuple(expected):
563
+ raise ValueError(f"expected volume {expected}, got {tuple(volume.shape[-3:])}. "
564
+ f"This checkpoint is fixed-resolution (learned s3/s4 position "
565
+ f"embeddings); use the checkpoint matching your input size.")
566
+ x = self.patch_embed(volume)
567
+ if window_id is not None:
568
+ x = x + self.window_embed(window_id).unsqueeze(1)
569
+ if ct_type_id is not None:
570
+ x = x + self.ct_type_embed(ct_type_id).unsqueeze(1)
571
+ if mask_s1 is not None:
572
+ x = torch.where(mask_s1.unsqueeze(-1), self.mask_token.to(x.dtype), x)
573
+ p0 = x if self.skip_fine else None
574
+
575
+ for blk in self.stage1:
576
+ x = blk(x, self.grid_s1)
577
+ e1 = x
578
+ x, _ = self.merge1(x, self.grid_s1)
579
+ if self.skip_fine:
580
+ if self.skip_pool_s2s4 == "attention":
581
+ sources = ((p0, self.grid_s1), (e1, self.grid_s1))
582
+ fine = torch.cat([pool(f, x, grid, self.grid_s2)
583
+ for pool, (f, grid) in zip(self.skip2_attention, sources)], dim=-1)
584
+ else:
585
+ fine = torch.cat([_pool_fine(p0, self.grid_s1, self.grid_s2), _pool_fine(e1, self.grid_s1, self.grid_s2)], dim=-1)
586
+ x = x + self.skip2_proj(self.skip2_norm(fine)).to(x.dtype)
587
+ for blk in self.stage2:
588
+ x = blk(x, self.grid_s2)
589
+ e2 = x
590
+ x, _ = self.merge2(x, self.grid_s2)
591
+ if self.skip_fine:
592
+ if self.skip_pool_s3 == "attention":
593
+ sources = ((p0, self.grid_s1), (e1, self.grid_s1), (e2, self.grid_s2))
594
+ fine = torch.cat([pool(f, x, grid, self.grid_s3)
595
+ for pool, (f, grid) in zip(self.skip3_attention, sources)], dim=-1)
596
+ else:
597
+ fine = torch.cat([_pool_fine(p0, self.grid_s1, self.grid_s3), _pool_fine(e1, self.grid_s1, self.grid_s3),
598
+ _pool_fine(e2, self.grid_s2, self.grid_s3)], dim=-1)
599
+ x = x + self.skip3_proj(self.skip3_norm(fine)).to(x.dtype)
600
+ x = x + self.pos_embed
601
+ for blk in self.stage3:
602
+ x = blk(x, self.grid_s3)
603
+ e3 = x
604
+ x, _ = self.merge3(x, self.grid_s3)
605
+ if self.skip_fine:
606
+ if self.skip_pool_s2s4 == "attention":
607
+ sources = ((p0, self.grid_s1), (e1, self.grid_s1), (e2, self.grid_s2), (e3, self.grid_s3))
608
+ fine = torch.cat([pool(f, x, grid, self.grid_s4)
609
+ for pool, (f, grid) in zip(self.skip4_attention, sources)], dim=-1)
610
+ else:
611
+ fine = torch.cat([_pool_fine(p0, self.grid_s1, self.grid_s4), _pool_fine(e1, self.grid_s1, self.grid_s4),
612
+ _pool_fine(e2, self.grid_s2, self.grid_s4), _pool_fine(e3, self.grid_s3, self.grid_s4)], dim=-1)
613
+ x = x + self.skip4_proj(self.skip4_norm(fine)).to(x.dtype)
614
+ return x, e1, e2, e3
615
+
616
+ def forward(self, volume: torch.Tensor,
617
+ window_id: Optional[torch.Tensor] = None,
618
+ ct_type_id: Optional[torch.Tensor] = None,
619
+ mask_s1: Optional[torch.Tensor] = None) -> Dict[str, torch.Tensor]:
620
+ """Encode a volume.
621
+
622
+ Args:
623
+ volume: [B, 1, D, H, W], HU-windowed to [0, 1] (see ``preprocess``).
624
+ window_id: [B] long, the HU window applied (see WINDOW_PARAMS).
625
+ ct_type_id: [B] long, optional (see CT_TYPE_MAP).
626
+ mask_s1: [B, N_s1] bool, optional; replaces tokens with the mask token.
627
+
628
+ Returns:
629
+ dict of token features ``{e1, e2, e3, e4}``, each [B, N_s, C_s].
630
+ """
631
+ x, e1, e2, e3 = self.trunk(volume, window_id, ct_type_id, mask_s1)
632
+ x = x + self.pos_embed_s4
633
+ for blk in self.stage4:
634
+ x = blk(x, self.grid_s4)
635
+ e4 = x
636
+ return {"e1": e1, "e2": e2, "e3": e3, "e4": e4}
637
+
638
+
639
+ class DenseLatentDecoder(nn.Module):
640
+ """Upsampling decoder producing dense latents at the s3/s2/s1 grids.
641
+
642
+ These latents are the JEPA targets during pretraining. Optional at inference; the
643
+ encoder features are the recommended readout for probing and retrieval.
644
+ """
645
+
646
+ def __init__(self, config: EncoderConfig):
647
+ super().__init__()
648
+ d1, d2, d3, _ = config.dims
649
+ self.grid_s1, self.grid_s2, self.grid_s3 = config.grid_s1, config.grid_s2, config.grid_s3
650
+ self.block3 = Mlp(d3, config.mlp_ratio, config.drop)
651
+ self.norm3 = nn.LayerNorm(d3)
652
+ self.up3to2 = nn.Linear(d3, d2)
653
+ self.fuse2 = nn.LayerNorm(d2)
654
+ self.block2 = Mlp(d2, config.mlp_ratio, config.drop)
655
+ self.up2to1 = nn.Linear(d2, d1)
656
+ self.fuse1 = nn.LayerNorm(d1)
657
+ self.block1 = Mlp(d1, config.mlp_ratio, config.drop)
658
+ self.out1 = nn.LayerNorm(d1)
659
+ self.out2 = nn.LayerNorm(d2)
660
+ self.out3 = nn.LayerNorm(d3)
661
+
662
+ @staticmethod
663
+ def _upsample(x: torch.Tensor, src_grid: Triple, dst_grid: Triple) -> torch.Tensor:
664
+ y = F.interpolate(to_grid(x, src_grid), size=tuple(dst_grid),
665
+ mode="trilinear", align_corners=False)
666
+ return to_tokens(y)
667
+
668
+ def forward(self, enc: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]:
669
+ e1, e2, e3 = enc["e1"], enc["e2"], enc["e3"]
670
+ d3 = e3 + self.block3(self.norm3(e3))
671
+ u2 = self.up3to2(self._upsample(d3, self.grid_s3, self.grid_s2))
672
+ d2 = u2 + e2
673
+ d2 = d2 + self.block2(self.fuse2(d2))
674
+ u1 = self.up2to1(self._upsample(d2, self.grid_s2, self.grid_s1))
675
+ d1 = u1 + e1
676
+ d1 = d1 + self.block1(self.fuse1(d1))
677
+ return {"s1": self.out1(d1), "s2": self.out2(d2), "s3": self.out3(d3)}
678
+
679
+
680
+ # ---------------------------------------------------------------------------
681
+ # Checkpoint loading
682
+ # ---------------------------------------------------------------------------
683
+ def _substate(state: Dict[str, torch.Tensor], prefix: str) -> Dict[str, torch.Tensor]:
684
+ p = prefix if prefix.endswith(".") else prefix + "."
685
+ return {k[len(p):]: v for k, v in state.items() if k.startswith(p)}
686
+
687
+
688
+ def load_pretrained(checkpoint_path, stream: str = "target", with_decoder: bool = False,
689
+ map_location: str = "cpu"):
690
+ """Rebuild the encoder (and optionally the dense decoder) from a training checkpoint.
691
+
692
+ Args:
693
+ checkpoint_path: path to ``checkpoint_epoch_*.pt``.
694
+ stream: ``"target"`` (EMA teacher, recommended) or ``"student"`` (online weights).
695
+ with_decoder: also build and load the dense latent decoder.
696
+ map_location: device passed to ``torch.load``.
697
+
698
+ Returns:
699
+ ``(encoder, config, info)``, or ``(encoder, decoder, config, info)`` when
700
+ ``with_decoder``. Modules are in eval mode. ``info`` holds the epoch, step,
701
+ stream, checkpoint path and the raw training config.
702
+ """
703
+ if stream not in ("target", "student"):
704
+ raise ValueError(f"stream must be 'target' or 'student', got {stream!r}")
705
+ ck = torch.load(str(checkpoint_path), map_location=map_location, weights_only=False)
706
+ if "model" not in ck or "config" not in ck:
707
+ raise ValueError(f"{checkpoint_path} is not an JECT training checkpoint "
708
+ f"(missing 'model'/'config'; found {sorted(ck)[:6]})")
709
+ cfg = EncoderConfig.from_training_config(ck["config"])
710
+
711
+ encoder = JECTEncoder(cfg)
712
+ enc_state = _substate(ck["model"], f"{stream}_encoder")
713
+ if not enc_state:
714
+ raise ValueError(f"checkpoint has no '{stream}_encoder.*' weights")
715
+ missing, unexpected = encoder.load_state_dict(enc_state, strict=False)
716
+ # `relative_position_index` is a non-persistent buffer rebuilt in __init__.
717
+ missing = [k for k in missing if not k.endswith("relative_position_index")]
718
+ if missing or unexpected:
719
+ raise RuntimeError(f"encoder state mismatch: missing={missing[:5]} "
720
+ f"unexpected={unexpected[:5]}")
721
+ encoder.eval()
722
+
723
+ info = {"epoch": ck.get("epoch"), "step": ck.get("step"), "stream": stream,
724
+ "checkpoint": str(checkpoint_path), "training_config": ck["config"]}
725
+ if not with_decoder:
726
+ return encoder, cfg, info
727
+
728
+ decoder = DenseLatentDecoder(cfg)
729
+ dec_state = _substate(ck["model"], f"{stream}_decoder")
730
+ if not dec_state:
731
+ raise ValueError(f"checkpoint has no '{stream}_decoder.*' weights")
732
+ missing, unexpected = decoder.load_state_dict(dec_state, strict=True)
733
+ decoder.eval()
734
+ return encoder, decoder, cfg, info
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3d675ae5eb24f0c689f351de3df32f6d3d138cad2599267185cd0376f15f63fb
3
+ size 699660744
modeling_ject.py ADDED
@@ -0,0 +1,269 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """JECT chest-CT encoder with a report-aligned readout.
2
+
3
+ The image encoder is frozen JECT pretraining. A latent-attention pooler reads its stage-4 tokens into 11 latents
4
+ (10 anatomy regions and one global latent); the global latent, projected and L2-normalised, is the image embedding.
5
+ Report text is embedded with chest2vec plus a LoRA adapter and a projection into the same space.
6
+ """
7
+ from __future__ import annotations
8
+
9
+ import dataclasses
10
+ import json
11
+ import os
12
+ from typing import List, Optional, Sequence
13
+
14
+ import numpy as np
15
+ import torch
16
+ import torch.nn as nn
17
+ import torch.nn.functional as F
18
+ from safetensors.torch import load_file
19
+
20
+ from transformers import PreTrainedModel
21
+
22
+ try: # loaded as remote code by transformers (package-relative) or from a local folder on sys.path
23
+ from .configuration_ject import JECTConfig
24
+ from .ject_encoder import CT_TYPE_MAP, WINDOW_PARAMS, EncoderConfig, JECTEncoder
25
+ except ImportError:
26
+ from configuration_ject import JECTConfig
27
+ from ject_encoder import CT_TYPE_MAP, WINDOW_PARAMS, EncoderConfig, JECTEncoder
28
+
29
+
30
+ def window_hu(volume: torch.Tensor, window_id: int) -> torch.Tensor:
31
+ """Raw HU to [0, 1] with the given window."""
32
+ p = WINDOW_PARAMS[window_id]
33
+ low, high = p["center"] - p["width"] / 2.0, p["center"] + p["width"] / 2.0
34
+ return ((volume.float() - low) / (high - low)).clamp_(0.0, 1.0)
35
+
36
+
37
+ def center_crop(volume: np.ndarray, size: int) -> np.ndarray:
38
+ """Centre crop the last three axes of a cube to size^3."""
39
+ off = [(s - size) // 2 for s in volume.shape[-3:]]
40
+ if min(off) < 0:
41
+ raise ValueError(f"volume {volume.shape[-3:]} is smaller than {size}^3")
42
+ return volume[..., off[0]:off[0] + size, off[1]:off[1] + size, off[2]:off[2] + size]
43
+
44
+
45
+ class LatentAttentionPooling(nn.Module):
46
+ """Learned queries cross-attend to the token grid; returns one vector per query."""
47
+
48
+ def __init__(self, dim: int, num_latents: int, num_heads: int, mlp_ratio: float = 4.0):
49
+ super().__init__()
50
+ self.latents = nn.Parameter(torch.zeros(1, num_latents, dim))
51
+ self.norm_latents = nn.LayerNorm(dim)
52
+ self.norm_tokens = nn.LayerNorm(dim)
53
+ self.attn = nn.MultiheadAttention(dim, num_heads, batch_first=True)
54
+ self.norm_mlp = nn.LayerNorm(dim)
55
+ hidden = int(dim * mlp_ratio)
56
+ self.mlp = nn.Sequential(nn.Linear(dim, hidden), nn.GELU(), nn.Linear(hidden, dim))
57
+
58
+ def forward(self, tokens: torch.Tensor) -> torch.Tensor:
59
+ q = self.norm_latents(self.latents).expand(tokens.shape[0], -1, -1)
60
+ kv = self.norm_tokens(tokens)
61
+ attended, _ = self.attn(q, kv, kv, need_weights=False)
62
+ attended = attended + q
63
+ return attended + self.mlp(self.norm_mlp(attended))
64
+
65
+
66
+ class JECTImageModel(nn.Module):
67
+ def __init__(self, config: dict):
68
+ super().__init__()
69
+ fields = {f.name: f for f in dataclasses.fields(EncoderConfig)}
70
+ enc = {k: tuple(v) if isinstance(v, list) else v for k, v in config["encoder"].items() if k in fields}
71
+ self.encoder = JECTEncoder(EncoderConfig(**enc))
72
+ r = config["readout"]
73
+ dim = self.encoder.pos_embed_s4.shape[-1]
74
+ self.pool = LatentAttentionPooling(dim, r["pool_latents"], r["pool_heads"])
75
+ self.proj = nn.Linear(dim, r["embed_dim"])
76
+ self.global_latent = r["global_latent"]
77
+ self.window_id = config["input"]["window_id"]
78
+ self.crop_size = config["input"]["crop_size"]
79
+
80
+ @torch.no_grad()
81
+ def forward(self, volume_hu: torch.Tensor, ct_type: Optional[Sequence[str]] = None) -> dict:
82
+ """volume_hu: [B, 1, 224, 224, 224] raw HU; ct_type: optional per-scan 'non-contrast', 'contrast' or 'ldct'.
83
+
84
+ Returns the image embedding and intermediate features.
85
+ """
86
+ e = self.encoder
87
+ if tuple(volume_hu.shape[-3:]) != (self.crop_size,) * 3:
88
+ raise ValueError(f"expected {self.crop_size}^3 input, got {tuple(volume_hu.shape[-3:])}")
89
+ x = window_hu(volume_hu, self.window_id)
90
+ wid = torch.full((x.shape[0],), self.window_id, dtype=torch.long, device=x.device)
91
+ cid = None
92
+ if ct_type is not None:
93
+ cid = torch.tensor([CT_TYPE_MAP[str(t).strip().lower()] for t in ct_type], dtype=torch.long, device=x.device)
94
+ x, _, _, _ = e.trunk(x, wid, cid)
95
+ x = x + e.pos_embed_s4
96
+ for blk in e.stage4:
97
+ x = blk(x, e.grid_s4)
98
+ latents = self.pool(x)
99
+ z = F.normalize(self.proj(latents).float(), dim=-1)
100
+ return {"embedding": z[:, self.global_latent], "latent_embeddings": z,
101
+ "pooled": latents[:, self.global_latent].float(), "s4_tokens": x, "s4_mean": x.mean(1).float()}
102
+
103
+
104
+ class JECTTextModel(nn.Module):
105
+ def __init__(self, config: dict, base_model: Optional[str] = None):
106
+ super().__init__()
107
+ from peft import LoraConfig, get_peft_model
108
+ from transformers import AutoModel, AutoTokenizer
109
+
110
+ t = config["text"]
111
+ name = base_model or t["base_model"]
112
+ base = AutoModel.from_pretrained(name, trust_remote_code=True, attn_implementation="eager")
113
+ self.tokenizer = AutoTokenizer.from_pretrained(name, padding_side="left", trust_remote_code=True)
114
+ if self.tokenizer.pad_token_id is None:
115
+ self.tokenizer.pad_token = self.tokenizer.eos_token
116
+ self.model = get_peft_model(base, LoraConfig(r=t["lora_r"], lora_alpha=t["lora_alpha"], lora_dropout=t.get("lora_dropout", 0.0),
117
+ target_modules=list(t["lora_targets"]), bias="none",
118
+ task_type="FEATURE_EXTRACTION"))
119
+ dim = config["readout"]["embed_dim"]
120
+ self.proj = nn.Linear(dim, dim)
121
+ self.max_length = t["max_length"]
122
+ self.pad_id = self.tokenizer.pad_token_id
123
+ eod = self.tokenizer.convert_tokens_to_ids("<|endoftext|>")
124
+ self.eod_id = eod if eod is not None and eod >= 0 else self.pad_id
125
+
126
+ def _ids(self, texts: Sequence[str]) -> List[List[int]]:
127
+ enc = self.tokenizer([str(t) for t in texts], add_special_tokens=False, truncation=True,
128
+ max_length=self.max_length - 1, padding=False, return_attention_mask=False)
129
+ return [x + [self.eod_id] for x in enc["input_ids"]]
130
+
131
+ @torch.no_grad()
132
+ def forward(self, texts: Sequence[str], batch_size: int = 16) -> torch.Tensor:
133
+ """Texts to L2-normalised embeddings [N, D]. Texts are length-sorted and left-padded per batch."""
134
+ device = self.proj.weight.device
135
+ ids = self._ids(texts)
136
+ order = sorted(range(len(ids)), key=lambda i: len(ids[i]))
137
+ out: List[Optional[torch.Tensor]] = [None] * len(ids)
138
+ for start in range(0, len(order), batch_size):
139
+ idx = order[start:start + batch_size]
140
+ width = max(len(ids[i]) for i in idx)
141
+ input_ids = torch.tensor([[self.pad_id] * (width - len(ids[i])) + ids[i] for i in idx], device=device)
142
+ attn = torch.tensor([[0] * (width - len(ids[i])) + [1] * len(ids[i]) for i in idx], device=device)
143
+ pooled = self.model.base_model.model.encode(input_ids, attn, normalize=False).pooler_output
144
+ emb = F.normalize(self.proj(pooled.float()), dim=-1)
145
+ for j, i in enumerate(idx):
146
+ out[i] = emb[j]
147
+ return torch.stack(out)
148
+
149
+
150
+ class JECTModel(nn.Module):
151
+ """Image and text towers with the SigLIP scale and bias used for scoring."""
152
+
153
+ def __init__(self, config: dict, with_text: bool = True, text_base_model: Optional[str] = None):
154
+ super().__init__()
155
+ self.config = config
156
+ self.image = JECTImageModel(config)
157
+ self.text = JECTTextModel(config, text_base_model) if with_text else None
158
+ self.logit_scale = float(np.exp(config["logit_scale_log"]))
159
+ self.logit_bias = float(config["logit_bias"])
160
+
161
+ @classmethod
162
+ def from_pretrained(cls, path: str, with_text: bool = True, text_base_model: Optional[str] = None,
163
+ device: str = "cpu") -> "JECTModel":
164
+ """Load from a Hugging Face repo id or a local folder (config.json, model.safetensors, text_adapter.safetensors)."""
165
+ path = _local(path)
166
+ config = json.load(open(os.path.join(path, "config.json")))
167
+ model = cls(config, with_text, text_base_model)
168
+ _load_image_weights(model.image, os.path.join(path, "model.safetensors"))
169
+ if model.text is not None:
170
+ _load_text_adapter(model.text, os.path.join(path, "text_adapter.safetensors"))
171
+ return model.to(device).eval()
172
+
173
+ def similarity(self, image_embedding: torch.Tensor, text_embedding: torch.Tensor) -> torch.Tensor:
174
+ """Pairwise SigLIP probabilities that each image matches each text."""
175
+ return torch.sigmoid(self.logit_scale * image_embedding @ text_embedding.T + self.logit_bias)
176
+
177
+
178
+ def _local(path: str) -> str:
179
+ if os.path.isdir(path):
180
+ return path
181
+ from huggingface_hub import snapshot_download
182
+ return snapshot_download(path)
183
+
184
+
185
+ def _relative_position_index(window) -> torch.Tensor:
186
+ wd, wh, ww = window
187
+ coords = torch.stack(torch.meshgrid(torch.arange(wd), torch.arange(wh), torch.arange(ww), indexing="ij")).reshape(3, -1)
188
+ rel = (coords[:, :, None] - coords[:, None, :]).permute(1, 2, 0).contiguous()
189
+ rel[..., 0] += wd - 1
190
+ rel[..., 1] += wh - 1
191
+ rel[..., 2] += ww - 1
192
+ rel[..., 0] *= (2 * wh - 1) * (2 * ww - 1)
193
+ rel[..., 1] *= 2 * ww - 1
194
+ return rel.sum(-1).view(-1)
195
+
196
+
197
+ def _rebuild_buffers(module: nn.Module) -> None:
198
+ """Recompute the windowed-attention index buffers, which are not stored in the weights."""
199
+ for m in module.modules():
200
+ if "relative_position_index" in m._buffers and hasattr(m, "window"):
201
+ dev = m.relative_position_bias_table.device
202
+ m._buffers["relative_position_index"] = _relative_position_index(m.window).to(dev)
203
+
204
+
205
+ def _load_image_weights(image: nn.Module, file: str) -> None:
206
+ sd = {k[len("image."):]: v for k, v in load_file(file).items() if k.startswith("image.")}
207
+ missing, unexpected = image.load_state_dict(sd, strict=False)
208
+ missing = [k for k in missing if not k.endswith("relative_position_index")]
209
+ if missing or unexpected:
210
+ raise RuntimeError(f"image weights do not match: missing={missing[:5]} unexpected={unexpected[:5]}")
211
+
212
+
213
+ def _load_text_adapter(text: nn.Module, file: str) -> None:
214
+ sd = load_file(file)
215
+ own = dict(text.named_parameters())
216
+ bad = [k for k in sd if k not in own]
217
+ if bad:
218
+ raise RuntimeError(f"text adapter weights do not match: {bad[:5]}")
219
+ with torch.no_grad():
220
+ for k, v in sd.items():
221
+ own[k].copy_(v)
222
+
223
+
224
+ class JECTImageEncoderModel(PreTrainedModel):
225
+ """The JECT image encoder and readout, loadable with ``AutoModel.from_pretrained(..., trust_remote_code=True)``."""
226
+
227
+ config_class = JECTConfig
228
+ base_model_prefix = "image"
229
+ main_input_name = "pixel_values"
230
+
231
+ def __init__(self, config: JECTConfig):
232
+ super().__init__(config)
233
+ self.image = JECTImageModel({"encoder": config.encoder, "readout": config.readout, "input": config.input})
234
+ self.post_init()
235
+
236
+ def _init_weights(self, module):
237
+ pass # all weights come from model.safetensors
238
+
239
+ @classmethod
240
+ def from_pretrained(cls, *args, **kwargs):
241
+ model = super().from_pretrained(*args, **kwargs)
242
+ _rebuild_buffers(model)
243
+ return model.eval()
244
+
245
+ def forward(self, pixel_values: torch.Tensor, ct_type: Optional[Sequence[str]] = None) -> dict:
246
+ """pixel_values: [B, 1, S, S, S] raw HU (see prepare.load_volume). Returns the same dict as JECTImageModel."""
247
+ return self.image(pixel_values, ct_type)
248
+
249
+ def similarity(self, image_embedding: torch.Tensor, text_embedding: torch.Tensor) -> torch.Tensor:
250
+ scale = float(np.exp(self.config.logit_scale_log))
251
+ return torch.sigmoid(scale * image_embedding @ text_embedding.T + self.config.logit_bias)
252
+
253
+
254
+ def load_text_model(repo: str, device: str = "cpu", text_base_model: Optional[str] = None) -> "JECTTextModel":
255
+ """chest2vec 0.6B with this model's LoRA adapter and projection, for report and prompt embeddings."""
256
+ path = _local(repo)
257
+ config = json.load(open(os.path.join(path, "config.json")))
258
+ text = JECTTextModel(config, text_base_model)
259
+ _load_text_adapter(text, os.path.join(path, "text_adapter.safetensors"))
260
+ return text.to(device).eval()
261
+
262
+
263
+ def load_cube(path: str, crop_size: int = 224) -> torch.Tensor:
264
+ """Read a preprocessed npz cube (key ``ct``, raw HU) and return a [1, 1, 224, 224, 224] tensor."""
265
+ with np.load(path) as d:
266
+ v = d["ct"]
267
+ if v.ndim == 4:
268
+ v = v[0]
269
+ return torch.from_numpy(np.ascontiguousarray(center_crop(v, crop_size))).unsqueeze(0).unsqueeze(0)
prepare.py ADDED
@@ -0,0 +1,160 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Turn a chest CT into model input.
2
+
3
+ Modes:
4
+ npz a cube already preprocessed for this model (key ``ct``); centre-cropped to the input size
5
+ totalseg NIfTI; lungs segmented with TotalSegmentator, then the exact preprocessing used in training
6
+ fast NIfTI; same preprocessing, but the lungs are found by HU threshold instead of TotalSegmentator
7
+ center NIfTI; same preprocessing, but the cube is centred on the middle of the volume
8
+
9
+ NIfTI preprocessing: reorient to LPS, resample to the model's spacing (trilinear), pad with air, crop a cube
10
+ around the lungs, store depth-first as raw HU int16, then centre-crop to the model's input size.
11
+ ``totalseg`` needs the ``TotalSegmentator`` package and a GPU for reasonable speed; the other modes need
12
+ ``monai`` and ``nibabel`` only.
13
+ """
14
+ from __future__ import annotations
15
+
16
+ import os
17
+ import tempfile
18
+ from typing import Optional, Sequence, Tuple
19
+
20
+ import numpy as np
21
+ import torch
22
+
23
+ AIR_HU = -1024.0
24
+ LUNG_LABELS = (10, 11, 12, 13, 14) # TotalSegmentator lung lobes
25
+ MODES = ("npz", "totalseg", "fast", "center")
26
+
27
+
28
+ def crop_starts(shape: Sequence[int], roi: Sequence[int], lo: Sequence[int], hi: Sequence[int]) -> Tuple[int, ...]:
29
+ """Start per axis of a crop of size ``roi`` around the box ``lo``..``hi``, kept inside the volume."""
30
+ starts = []
31
+ for size, r, a, b in zip(shape, roi, lo, hi):
32
+ if size <= r:
33
+ starts.append(0)
34
+ continue
35
+ a, b = int(a), int(b)
36
+ extent = b - a + 1
37
+ start = (a + b) // 2 - r // 2 if extent >= r else a - (r - extent) // 2
38
+ starts.append(int(np.clip(start, 0, size - r)))
39
+ return tuple(starts)
40
+
41
+
42
+ def hu_lung_box(vol: np.ndarray) -> Optional[Tuple[np.ndarray, np.ndarray]]:
43
+ """Bounding box of the lungs found by HU threshold, or None.
44
+
45
+ The body is the largest connected region above -500 HU (this drops the table), filled slice by slice
46
+ along the last (I-S) axis; the lungs are air below -400 HU strictly inside it, keeping the two
47
+ largest connected regions.
48
+ """
49
+ from scipy import ndimage
50
+ lab, n = ndimage.label(vol > -500)
51
+ if n == 0:
52
+ return None
53
+ sizes = ndimage.sum_labels(np.ones_like(lab, dtype=np.int8), lab, index=np.arange(1, n + 1))
54
+ body = lab == (int(np.argmax(sizes)) + 1)
55
+ filled = np.zeros_like(body)
56
+ for z in range(body.shape[2]):
57
+ filled[:, :, z] = ndimage.binary_fill_holes(body[:, :, z])
58
+ air = filled & ~body & (vol < -400)
59
+ lab, n = ndimage.label(air)
60
+ if n == 0:
61
+ return None
62
+ sizes = ndimage.sum_labels(np.ones_like(lab, dtype=np.int8), lab, index=np.arange(1, n + 1))
63
+ keep = np.argsort(sizes)[::-1][:2] + 1
64
+ lungs = np.isin(lab, keep)
65
+ if lungs.sum() < 10000:
66
+ return None
67
+ coords = np.argwhere(lungs)
68
+ return coords.min(0), coords.max(0)
69
+
70
+
71
+ def run_totalsegmentator(nifti_path: str, out_path: str, device: str = "gpu") -> str:
72
+ """Segment the scan with TotalSegmentator (multi-label output) and return the mask path."""
73
+ from totalsegmentator.python_api import totalsegmentator
74
+ totalsegmentator(nifti_path, out_path, ml=True, device=device, quiet=True)
75
+ return out_path
76
+
77
+
78
+ def preprocess_nifti(nifti_path: str, spacing_mm: float, cube_size: int, mode: str = "fast",
79
+ totalseg_mask: Optional[str] = None, device: str = "gpu", return_volume: bool = False):
80
+ """NIfTI to a raw-HU int16 cube [cube, cube, cube] in the model's layout (LPS, depth-first).
81
+
82
+ With return_volume, also returns the resampled volume (depth-first) and the crop start per axis.
83
+ """
84
+ from monai.transforms import (Compose, EnsureChannelFirstd, LoadImaged, Orientationd, Spacingd,
85
+ SpatialPadd)
86
+ if mode not in ("totalseg", "fast", "center"):
87
+ raise ValueError(f"mode must be totalseg, fast or center for NIfTI input, got {mode!r}")
88
+ tmp = None
89
+ if mode == "totalseg" and totalseg_mask is None:
90
+ tmp = tempfile.mkdtemp()
91
+ totalseg_mask = run_totalsegmentator(nifti_path, os.path.join(tmp, "seg.nii.gz"), device)
92
+ keys = ("ct", "seg") if mode == "totalseg" else ("ct",)
93
+ roi = (cube_size,) * 3
94
+ steps = [LoadImaged(keys=keys, reader="NibabelReader"), EnsureChannelFirstd(keys=keys),
95
+ Orientationd(keys=keys, axcodes="LPS", labels=(("L", "R"), ("P", "A"), ("I", "S"))),
96
+ Spacingd(keys=("ct",), pixdim=(spacing_mm,) * 3, mode="trilinear", align_corners=True)]
97
+ if mode == "totalseg":
98
+ steps.append(Spacingd(keys=("seg",), pixdim=(spacing_mm,) * 3, mode="nearest", align_corners=True))
99
+ steps.append(SpatialPadd(keys=("ct",), spatial_size=roi, constant_values=AIR_HU))
100
+ if mode == "totalseg":
101
+ steps.append(SpatialPadd(keys=("seg",), spatial_size=roi, constant_values=0.0))
102
+ paths = {"ct": nifti_path, **({"seg": totalseg_mask} if mode == "totalseg" else {})}
103
+ d = Compose(steps)(paths)
104
+ ct = np.asarray(d["ct"])[0]
105
+ if mode == "totalseg":
106
+ lungs = np.isin(np.asarray(d["seg"])[0], LUNG_LABELS)
107
+ if not lungs.any():
108
+ raise ValueError(f"{nifti_path}: TotalSegmentator found no lungs")
109
+ coords = np.argwhere(lungs)
110
+ starts = crop_starts(ct.shape, roi, coords.min(0), coords.max(0))
111
+ elif mode == "fast":
112
+ box = hu_lung_box(ct)
113
+ if box is None:
114
+ c = tuple(x // 2 for x in ct.shape)
115
+ box = (c, c)
116
+ starts = crop_starts(ct.shape, roi, box[0], box[1])
117
+ else:
118
+ c = tuple(s // 2 for s in ct.shape)
119
+ starts = crop_starts(ct.shape, roi, c, c)
120
+ full = ct
121
+ ct = ct[tuple(slice(s, s + r) for s, r in zip(starts, roi))]
122
+ ct = np.clip(np.round(ct), np.iinfo(np.int16).min, np.iinfo(np.int16).max)
123
+ if tmp is not None:
124
+ for f in os.listdir(tmp):
125
+ os.remove(os.path.join(tmp, f))
126
+ os.rmdir(tmp)
127
+ cube = np.ascontiguousarray(np.transpose(ct, (2, 0, 1))).astype(np.int16)
128
+ if return_volume:
129
+ return cube, np.transpose(full, (2, 0, 1)), (starts[2], starts[0], starts[1])
130
+ return cube
131
+
132
+
133
+ def center_crop(volume: np.ndarray, size: int) -> np.ndarray:
134
+ off = [(s - size) // 2 for s in volume.shape[-3:]]
135
+ if min(off) < 0:
136
+ raise ValueError(f"volume {volume.shape[-3:]} is smaller than {size}^3")
137
+ return volume[..., off[0]:off[0] + size, off[1]:off[1] + size, off[2]:off[2] + size]
138
+
139
+
140
+ def load_volume(path: str, config: dict, mode: Optional[str] = None, totalseg_mask: Optional[str] = None,
141
+ device: str = "gpu") -> torch.Tensor:
142
+ """Read an npz cube or a NIfTI scan and return a [1, 1, S, S, S] raw-HU tensor for the model.
143
+
144
+ ``config`` is the model config (a dict or the loaded config object; its ``input`` block sets spacing, cube and
145
+ crop size). ``mode``
146
+ defaults to ``npz`` for .npz files and ``fast`` for NIfTI.
147
+ """
148
+ inp = config["input"] if isinstance(config, dict) else config.input
149
+ if mode is None:
150
+ mode = "npz" if path.endswith(".npz") else "fast"
151
+ if mode not in MODES:
152
+ raise ValueError(f"mode must be one of {MODES}, got {mode!r}")
153
+ if mode == "npz":
154
+ with np.load(path) as d:
155
+ cube = d["ct"]
156
+ cube = cube[0] if cube.ndim == 4 else cube
157
+ else:
158
+ cube = preprocess_nifti(path, inp["spacing_mm"], inp["cube_size"], mode, totalseg_mask, device)
159
+ cube = center_crop(cube, inp["crop_size"])
160
+ return torch.from_numpy(np.ascontiguousarray(cube)).unsqueeze(0).unsqueeze(0)
preprocess.py ADDED
@@ -0,0 +1,155 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Input preprocessing for the JECT encoder.
2
+
3
+ Input contract used during pretraining (features degrade without warning otherwise):
4
+
5
+ 1. Orientation LPS, axes ordered (D, H, W) = (R-L, A-P, I-S), as produced by MONAI's
6
+ ``Orientationd(axcodes="LPS")``.
7
+ 2. Isotropic resampling, then a lung-centred crop/pad to a fixed cube of raw HU
8
+ (int16, padded with -1024 HU):
9
+ * 1.5 mm spacing, 256^3 cube: input for the 224 checkpoint
10
+ * 1.0 mm spacing, 384^3 cube: input for the 336 checkpoint
11
+ Both give a ~336 mm physical field of view at the model input.
12
+ 3. Centre crop / pad (with -1000 HU) from that cube to the model's input size.
13
+ 4. HU window, then linear scale to [0, 1]. The default is the general window
14
+ (id 2, centre 0, width 2000, i.e. [-1000, 1000]). Pass the same id to the encoder
15
+ as ``window_id``; it selects an additive token embedding.
16
+
17
+ ``preprocess_hu_volume`` performs steps 3-4 on an already-resampled cube and is the
18
+ usual entry point. ``resample_to_spacing`` covers the resampling in step 2 for raw
19
+ NIfTI input; it is a plain trilinear resize without the lung-bounding-box crop, so
20
+ features differ slightly from those of the original pipeline.
21
+ """
22
+
23
+ from __future__ import annotations
24
+
25
+ import os
26
+ from typing import Optional, Sequence, Tuple
27
+
28
+ import numpy as np
29
+ import torch
30
+
31
+ try: # importable as a package module or as a plain script in this directory
32
+ from .ject_encoder import CT_TYPE_MAP, DEFAULT_CT_TYPE, GENERAL_WINDOW_ID, WINDOW_PARAMS
33
+ except ImportError:
34
+ from ject_encoder import CT_TYPE_MAP, DEFAULT_CT_TYPE, GENERAL_WINDOW_ID, WINDOW_PARAMS
35
+
36
+ Triple = Tuple[int, int, int]
37
+
38
+ AIR_HU = -1000.0 # pad value for the centre crop/pad
39
+
40
+
41
+ # ---------------------------------------------------------------------------
42
+ # Windowing
43
+ # ---------------------------------------------------------------------------
44
+ def apply_window(volume_hu: np.ndarray, window_id: int = GENERAL_WINDOW_ID) -> np.ndarray:
45
+ """Clamp raw HU to a window and rescale to [0, 1]."""
46
+ if window_id not in WINDOW_PARAMS:
47
+ raise ValueError(f"window_id must be one of {sorted(WINDOW_PARAMS)}, got {window_id}")
48
+ p = WINDOW_PARAMS[window_id]
49
+ low = p["center"] - p["width"] / 2.0
50
+ high = p["center"] + p["width"] / 2.0
51
+ out = (volume_hu.astype(np.float32) - low) / (high - low)
52
+ return np.clip(out, 0.0, 1.0)
53
+
54
+
55
+ def ct_type_to_id(ct_type) -> int:
56
+ """Map a ct_type string ('non-contrast' | 'contrast' | 'ldct') to its embedding index."""
57
+ if ct_type is None:
58
+ return DEFAULT_CT_TYPE
59
+ if isinstance(ct_type, (int, np.integer)):
60
+ return int(ct_type)
61
+ return CT_TYPE_MAP.get(str(ct_type).strip().lower(), DEFAULT_CT_TYPE)
62
+
63
+
64
+ # ---------------------------------------------------------------------------
65
+ # Crop / pad
66
+ # ---------------------------------------------------------------------------
67
+ def make_crop_pad_transform(src_shape: Triple, dst_shape: Triple, random_crop: bool = False):
68
+ """Slices + pads that take ``src_shape`` to exactly ``dst_shape`` (centre crop by default)."""
69
+ slices, pads = [], []
70
+ for src, dst in zip(src_shape, dst_shape):
71
+ if src >= dst:
72
+ start = (src - dst) // 2
73
+ if random_crop and src > dst:
74
+ start = int(np.random.randint(0, src - dst + 1))
75
+ slices.append(slice(start, start + dst))
76
+ pads.append((0, 0))
77
+ else:
78
+ slices.append(slice(0, src))
79
+ before = (dst - src) // 2
80
+ pads.append((before, dst - src - before))
81
+ return tuple(slices), tuple(pads)
82
+
83
+
84
+ def apply_crop_pad_3d(arr: np.ndarray, slices, pads, constant: float = AIR_HU) -> np.ndarray:
85
+ """Apply crop/pad to the last three dims of ``arr``."""
86
+ cropped = arr[(...,) + tuple(slices)]
87
+ if any(a > 0 or b > 0 for a, b in pads):
88
+ pad_width = [(0, 0)] * (cropped.ndim - 3) + list(pads)
89
+ cropped = np.pad(cropped, pad_width=pad_width, mode="constant", constant_values=constant)
90
+ return cropped
91
+
92
+
93
+ def center_crop_pad(volume_hu: np.ndarray, target: Triple) -> np.ndarray:
94
+ """Centre crop/pad a raw-HU volume (last three dims) to ``target``, padding with air."""
95
+ slices, pads = make_crop_pad_transform(tuple(volume_hu.shape[-3:]), tuple(target))
96
+ return apply_crop_pad_3d(volume_hu, slices, pads, constant=AIR_HU)
97
+
98
+
99
+ # ---------------------------------------------------------------------------
100
+ # Entry points
101
+ # ---------------------------------------------------------------------------
102
+ def preprocess_hu_volume(volume_hu: np.ndarray, input_size: Triple,
103
+ window_id: int = GENERAL_WINDOW_ID) -> torch.Tensor:
104
+ """Convert a raw-HU cube to a model input tensor ``[1, D, H, W]`` in [0, 1].
105
+
106
+ Crop/pad happens in HU space (padding with air) before windowing, so padded
107
+ borders map to 0.0 in the general window.
108
+ """
109
+ vol = np.asarray(volume_hu)
110
+ if vol.ndim == 4:
111
+ if vol.shape[0] != 1:
112
+ raise ValueError(f"expected a single-channel volume, got shape {vol.shape}")
113
+ vol = vol[0]
114
+ if vol.ndim != 3:
115
+ raise ValueError(f"expected a 3D HU volume, got shape {vol.shape}")
116
+ vol = center_crop_pad(vol, tuple(input_size))
117
+ return torch.from_numpy(apply_window(vol, window_id)).unsqueeze(0)
118
+
119
+
120
+ def load_npz_volume(path: str, input_size: Triple, key: str = "ct",
121
+ window_id: int = GENERAL_WINDOW_ID) -> torch.Tensor:
122
+ """Load a preprocessed ``.npz`` (raw int16 HU under ``key``) into a model input tensor."""
123
+ if not os.path.exists(path) and os.path.exists(path + ".npz"):
124
+ path = path + ".npz"
125
+ with np.load(path) as d:
126
+ if key not in d:
127
+ raise KeyError(f"{path} has no '{key}' array (found {list(d.files)})")
128
+ ct = d[key]
129
+ return preprocess_hu_volume(ct, input_size, window_id=window_id)
130
+
131
+
132
+ def resample_to_spacing(volume_hu: np.ndarray, src_spacing: Sequence[float],
133
+ dst_spacing: float = 1.5) -> np.ndarray:
134
+ """Trilinear-resample a raw-HU volume to isotropic ``dst_spacing`` mm.
135
+
136
+ Convenience for raw NIfTI input. It does not apply the lung-bounding-box crop
137
+ used in pretraining; see the module docstring.
138
+ """
139
+ vol = np.asarray(volume_hu, dtype=np.float32)
140
+ if vol.ndim != 3:
141
+ raise ValueError(f"expected a 3D HU volume, got shape {vol.shape}")
142
+ if len(src_spacing) != 3:
143
+ raise ValueError(f"src_spacing must have 3 entries, got {src_spacing!r}")
144
+ scale = [float(s) / float(dst_spacing) for s in src_spacing]
145
+ out_shape = tuple(max(1, int(round(n * s))) for n, s in zip(vol.shape, scale))
146
+ t = torch.from_numpy(vol)[None, None]
147
+ t = torch.nn.functional.interpolate(t, size=out_shape, mode="trilinear", align_corners=True)
148
+ return t[0, 0].numpy()
149
+
150
+
151
+ def stack_batch(volumes: Sequence[torch.Tensor]) -> torch.Tensor:
152
+ """Stack [1, D, H, W] tensors into a [B, 1, D, H, W] batch."""
153
+ if not volumes:
154
+ raise ValueError("stack_batch got an empty list")
155
+ return torch.stack([v if v.ndim == 4 else v.unsqueeze(0) for v in volumes], dim=0)
requirements.txt ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ torch>=2.4
2
+ numpy>=1.24
3
+ safetensors>=0.4
4
+ huggingface_hub>=0.30
5
+ transformers>=4.57
6
+ peft>=0.18
7
+ # NIfTI input (prepare.py)
8
+ monai>=1.5
9
+ nibabel>=5.0
10
+ scipy>=1.10
11
+ # optional: prepare.py mode "totalseg"
12
+ # TotalSegmentator>=2.0
text_adapter.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e0069b975fbdd32278e2aa6dab00e3daed5db45786ec869bf0a8faf15814392c
3
+ size 44626192