Initial release
Browse files- .gitattributes +0 -34
- .gitignore +9 -0
- LICENSE +12 -0
- LICENSE-CODE +21 -0
- README.md +125 -0
- config.json +105 -0
- configuration_ject.py +15 -0
- ject_encoder.py +734 -0
- model.safetensors +3 -0
- modeling_ject.py +269 -0
- prepare.py +160 -0
- preprocess.py +155 -0
- requirements.txt +12 -0
- text_adapter.safetensors +3 -0
.gitattributes
CHANGED
|
@@ -1,35 +1 @@
|
|
| 1 |
-
*.7z filter=lfs diff=lfs merge=lfs -text
|
| 2 |
-
*.arrow filter=lfs diff=lfs merge=lfs -text
|
| 3 |
-
*.bin filter=lfs diff=lfs merge=lfs -text
|
| 4 |
-
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
| 5 |
-
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
| 6 |
-
*.ftz filter=lfs diff=lfs merge=lfs -text
|
| 7 |
-
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 8 |
-
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 9 |
-
*.joblib filter=lfs diff=lfs merge=lfs -text
|
| 10 |
-
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
| 11 |
-
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
| 12 |
-
*.model filter=lfs diff=lfs merge=lfs -text
|
| 13 |
-
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
| 14 |
-
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 15 |
-
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 16 |
-
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 17 |
-
*.ot filter=lfs diff=lfs merge=lfs -text
|
| 18 |
-
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 19 |
-
*.pb filter=lfs diff=lfs merge=lfs -text
|
| 20 |
-
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 21 |
-
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 22 |
-
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 23 |
-
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 24 |
-
*.rar filter=lfs diff=lfs merge=lfs -text
|
| 25 |
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 26 |
-
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
| 27 |
-
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
| 28 |
-
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 29 |
-
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 30 |
-
*.tgz filter=lfs diff=lfs merge=lfs -text
|
| 31 |
-
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 32 |
-
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 33 |
-
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
-
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
-
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
.gitignore
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# weights, features and tables
|
| 2 |
+
*.pt
|
| 3 |
+
*.csv
|
| 4 |
+
|
| 5 |
+
# caches
|
| 6 |
+
__pycache__/
|
| 7 |
+
*.py[cod]
|
| 8 |
+
.cache/
|
| 9 |
+
cache/
|
LICENSE
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Model weights: Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International (CC BY-NC-SA 4.0)
|
| 2 |
+
|
| 3 |
+
Copyright (c) 2026 The JECT Authors
|
| 4 |
+
|
| 5 |
+
The files model.safetensors and text_adapter.safetensors are licensed under
|
| 6 |
+
CC BY-NC-SA 4.0. You may share and adapt them for non-commercial purposes, with attribution, and must distribute
|
| 7 |
+
adaptations under the same license. Commercial use is not permitted.
|
| 8 |
+
|
| 9 |
+
Full legal text: https://creativecommons.org/licenses/by-nc-sa/4.0/legalcode
|
| 10 |
+
|
| 11 |
+
The Python files in this repository (configuration_ject.py, modeling_ject.py, ject_encoder.py, preprocess.py, prepare.py) are licensed under the
|
| 12 |
+
MIT License; see LICENSE-CODE.
|
LICENSE-CODE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
MIT License
|
| 2 |
+
|
| 3 |
+
Copyright (c) 2026 The JECT Authors
|
| 4 |
+
|
| 5 |
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
| 6 |
+
of this software and associated documentation files (the "Software"), to deal
|
| 7 |
+
in the Software without restriction, including without limitation the rights
|
| 8 |
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
| 9 |
+
copies of the Software, and to permit persons to whom the Software is
|
| 10 |
+
furnished to do so, subject to the following conditions:
|
| 11 |
+
|
| 12 |
+
The above copyright notice and this permission notice shall be included in all
|
| 13 |
+
copies or substantial portions of the Software.
|
| 14 |
+
|
| 15 |
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
| 16 |
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
| 17 |
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
| 18 |
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
| 19 |
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
| 20 |
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
| 21 |
+
SOFTWARE.
|
README.md
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: cc-by-nc-sa-4.0
|
| 3 |
+
library_name: pytorch
|
| 4 |
+
pipeline_tag: feature-extraction
|
| 5 |
+
tags:
|
| 6 |
+
- medical-imaging
|
| 7 |
+
- chest-ct
|
| 8 |
+
- 3d
|
| 9 |
+
- foundation-model
|
| 10 |
+
- jepa
|
| 11 |
+
- siglip
|
| 12 |
+
---
|
| 13 |
+
|
| 14 |
+
# JECT_320_aligned
|
| 15 |
+
|
| 16 |
+
Model weights for *JECT: Addressing the Context Shortcut in Joint-Embedding Predictive Learning for Chest CT*.
|
| 17 |
+
|
| 18 |
+
A 3D chest-CT encoder pretrained with JECT and cooled down at 320³ on 1 mm volumes, then aligned end to end with
|
| 19 |
+
radiology reports: the whole image encoder, a latent-attention pooler, a projection and a text adapter were trained
|
| 20 |
+
together. A variant with the encoder kept frozen is released as JECT_320. Use it for CT embeddings, report retrieval and
|
| 21 |
+
zero-shot finding scores.
|
| 22 |
+
|
| 23 |
+
| component | parameters | trained in the alignment stage |
|
| 24 |
+
|---|---|---|
|
| 25 |
+
| image encoder (JECT, 320³ input) | 161 M | yes |
|
| 26 |
+
| latent-attention pooler + projection | 13.7 M | yes |
|
| 27 |
+
| text: chest2vec 0.6B + LoRA (r 16) + projection | 11.1 M adapter | adapter only |
|
| 28 |
+
|
| 29 |
+
## Input
|
| 30 |
+
|
| 31 |
+
A chest CT as NIfTI (`.nii.gz`) or as an already preprocessed cube (`.npz`). `prepare.load_volume` turns either into the
|
| 32 |
+
model input: reorient to LPS, resample to 1 mm, pad with air (-1024 HU), crop a 384³ cube around the lungs, keep raw HU,
|
| 33 |
+
then centre-crop 320³. The window is applied inside the model.
|
| 34 |
+
|
| 35 |
+
| mode | how the lungs are located | speed | needs |
|
| 36 |
+
|---|---|---|---|
|
| 37 |
+
| `fast` (default for NIfTI) | air inside the body, found by HU threshold | seconds | `monai`, `nibabel`, `scipy` |
|
| 38 |
+
| `totalseg` | TotalSegmentator lung lobes, as in training | about a minute on a GPU | `TotalSegmentator` |
|
| 39 |
+
| `center` | middle of the scan, no lung search | seconds | `monai`, `nibabel` |
|
| 40 |
+
| `npz` | cube already preprocessed for this model | instant | none |
|
| 41 |
+
|
| 42 |
+
`totalseg` reproduces the training input exactly. Most chest CTs fit inside the 384³ cube, so all modes give the same input;
|
| 43 |
+
they differ only on scans that extend beyond it (for example chest-abdomen series), where `center` can miss the lungs.
|
| 44 |
+
|
| 45 |
+
Measured with JECT_224 on 414 CT-RATE validation scans available as NIfTI, against the exact training input (CT-RATE-18
|
| 46 |
+
linear probe, macro AUROC; paired bootstrap over scans):
|
| 47 |
+
|
| 48 |
+
| mode | macro AUROC | difference to exact [95 % CI] | embedding cosine to exact, median / 5th percentile |
|
| 49 |
+
|---|---|---|---|
|
| 50 |
+
| exact (`totalseg` or `npz`) | 0.875 | | 1 / 1 |
|
| 51 |
+
| `fast` | 0.875 | -0.0005 [-0.0015, +0.0005] | 1.000 / 0.977 |
|
| 52 |
+
| `center` | 0.875 | -0.0002 [-0.0018, +0.0014] | 1.000 / 0.955 |
|
| 53 |
+
Scan type (`"non-contrast"`, `"contrast"` or `"ldct"`): Pass the scan type; the model was trained with it.
|
| 54 |
+
|
| 55 |
+
## Usage
|
| 56 |
+
|
| 57 |
+
```python
|
| 58 |
+
import torch
|
| 59 |
+
from transformers import AutoModel
|
| 60 |
+
|
| 61 |
+
repo = "<user>/JECT_320_aligned"
|
| 62 |
+
model = AutoModel.from_pretrained(repo, trust_remote_code=True).cuda()
|
| 63 |
+
|
| 64 |
+
from huggingface_hub import snapshot_download # input preparation and the text tower ship as code in the repo
|
| 65 |
+
import sys; sys.path.insert(0, snapshot_download(repo))
|
| 66 |
+
from prepare import load_volume
|
| 67 |
+
from modeling_ject import load_text_model
|
| 68 |
+
|
| 69 |
+
volume = load_volume("scan.nii.gz", model.config).cuda() # mode="fast"; also "totalseg", "center", or an .npz path
|
| 70 |
+
with torch.no_grad(), torch.autocast("cuda", dtype=torch.bfloat16):
|
| 71 |
+
out = model(volume, ct_type=["non-contrast"])
|
| 72 |
+
text = load_text_model(repo, device="cuda") # needs chest2vec/chest2vec_0.6B
|
| 73 |
+
with torch.no_grad(), torch.autocast("cuda", dtype=torch.bfloat16):
|
| 74 |
+
texts = text(["[Lungs] - A 9 mm solid nodule in the right upper lobe.", "[Lungs] - The lungs are clear."])
|
| 75 |
+
print(model.similarity(out["embedding"].float(), texts.float()))
|
| 76 |
+
```
|
| 77 |
+
|
| 78 |
+
The model returns `embedding` (the 1,024-d retrieval vector), `latent_embeddings` (all 11 latents: 10 anatomy regions and
|
| 79 |
+
the global one), `pooled`, `s4_tokens` (10³ = 1,000 tokens × 1,024) and `s4_mean`. Scores follow SigLIP:
|
| 80 |
+
`sigmoid(exp(logit_scale_log) · cos + logit_bias)`.
|
| 81 |
+
|
| 82 |
+
## Results
|
| 83 |
+
|
| 84 |
+
CT-RATE validation, 1,564 scans and 1,503 distinct reports (identical reports count as matches), evaluated at the end of
|
| 85 |
+
alignment training:
|
| 86 |
+
|
| 87 |
+
| task | R@1 | R@5 | R@10 |
|
| 88 |
+
|---|---|---|---|
|
| 89 |
+
| image-to-report retrieval | 0.330 | 0.580 | 0.668 |
|
| 90 |
+
| report-to-image retrieval | 0.359 | 0.593 | 0.685 |
|
| 91 |
+
|
| 92 |
+
## Training data
|
| 93 |
+
|
| 94 |
+
Encoder: JECT pretraining at 224³ on about 173,000 chest CTs with reports (CT-RATE and a private institutional cohort),
|
| 95 |
+
then a cooldown at 320³ on 1 mm volumes. Alignment: CT-RATE training pairs (findings section), all encoder weights trained.
|
| 96 |
+
|
| 97 |
+
## Limitations
|
| 98 |
+
|
| 99 |
+
Trained on adult chest CT; reports in English. Not a medical device and not validated for clinical use. Retrieval scores
|
| 100 |
+
reflect CT-RATE-style findings text; other report styles may score lower. The model expects the input contract above;
|
| 101 |
+
other spacings, crops or orientations run without error but give wrong features.
|
| 102 |
+
|
| 103 |
+
## Files
|
| 104 |
+
|
| 105 |
+
| file | content |
|
| 106 |
+
|---|---|
|
| 107 |
+
| `config.json` | encoder architecture, readout and text settings, SigLIP scale and bias |
|
| 108 |
+
| `model.safetensors` | image encoder, latent-attention pooler and projection |
|
| 109 |
+
| `text_adapter.safetensors` | LoRA and projection for chest2vec 0.6B |
|
| 110 |
+
| `configuration_ject.py`, `modeling_ject.py`, `ject_encoder.py`, `preprocess.py` | model code (`AutoModel`, `trust_remote_code=True`) |
|
| 111 |
+
| `prepare.py` | NIfTI and npz input preparation |
|
| 112 |
+
|
| 113 |
+
## Citation
|
| 114 |
+
|
| 115 |
+
```bibtex
|
| 116 |
+
@misc{ject2026,
|
| 117 |
+
title = {{JECT}: Addressing the Context Shortcut in Joint-Embedding Predictive Learning for Chest {CT}},
|
| 118 |
+
author = {TBD},
|
| 119 |
+
year = {2026}
|
| 120 |
+
}
|
| 121 |
+
```
|
| 122 |
+
|
| 123 |
+
## License
|
| 124 |
+
|
| 125 |
+
Model weights: CC BY-NC-SA 4.0 (non-commercial use only; see `LICENSE`). Code: MIT (see `LICENSE-CODE`).
|
config.json
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model_type": "ject-ct",
|
| 3 |
+
"encoder": {
|
| 4 |
+
"input_size": [
|
| 5 |
+
320,
|
| 6 |
+
320,
|
| 7 |
+
320
|
| 8 |
+
],
|
| 9 |
+
"patch_size": [
|
| 10 |
+
4,
|
| 11 |
+
4,
|
| 12 |
+
4
|
| 13 |
+
],
|
| 14 |
+
"in_channels": 1,
|
| 15 |
+
"variant": "huge",
|
| 16 |
+
"local_block_type": "swin",
|
| 17 |
+
"stage3_block_type": "swin",
|
| 18 |
+
"stage4_block_type": "global",
|
| 19 |
+
"window": [
|
| 20 |
+
7,
|
| 21 |
+
7,
|
| 22 |
+
7
|
| 23 |
+
],
|
| 24 |
+
"mlp_ratio": 4.0,
|
| 25 |
+
"drop": 0.0,
|
| 26 |
+
"num_ct_types": 3,
|
| 27 |
+
"skip_fine": true,
|
| 28 |
+
"skip_pool_s3": "attention",
|
| 29 |
+
"skip_pool_s2s4": "attention",
|
| 30 |
+
"dims": [
|
| 31 |
+
128,
|
| 32 |
+
256,
|
| 33 |
+
512,
|
| 34 |
+
1024
|
| 35 |
+
],
|
| 36 |
+
"heads": [
|
| 37 |
+
8,
|
| 38 |
+
16,
|
| 39 |
+
32,
|
| 40 |
+
32
|
| 41 |
+
],
|
| 42 |
+
"depths": [
|
| 43 |
+
2,
|
| 44 |
+
4,
|
| 45 |
+
9,
|
| 46 |
+
9
|
| 47 |
+
],
|
| 48 |
+
"grid_s1": [
|
| 49 |
+
80,
|
| 50 |
+
80,
|
| 51 |
+
80
|
| 52 |
+
],
|
| 53 |
+
"grid_s2": [
|
| 54 |
+
40,
|
| 55 |
+
40,
|
| 56 |
+
40
|
| 57 |
+
],
|
| 58 |
+
"grid_s3": [
|
| 59 |
+
20,
|
| 60 |
+
20,
|
| 61 |
+
20
|
| 62 |
+
],
|
| 63 |
+
"grid_s4": [
|
| 64 |
+
10,
|
| 65 |
+
10,
|
| 66 |
+
10
|
| 67 |
+
]
|
| 68 |
+
},
|
| 69 |
+
"readout": {
|
| 70 |
+
"embed_dim": 1024,
|
| 71 |
+
"pool_latents": 11,
|
| 72 |
+
"pool_heads": 8,
|
| 73 |
+
"global_latent": 10
|
| 74 |
+
},
|
| 75 |
+
"input": {
|
| 76 |
+
"crop_size": 320,
|
| 77 |
+
"cube_size": 384,
|
| 78 |
+
"spacing_mm": 1.0,
|
| 79 |
+
"window_id": 2
|
| 80 |
+
},
|
| 81 |
+
"text": {
|
| 82 |
+
"base_model": "chest2vec/chest2vec_0.6B",
|
| 83 |
+
"lora_r": 16,
|
| 84 |
+
"lora_alpha": 32,
|
| 85 |
+
"lora_targets": [
|
| 86 |
+
"q_proj",
|
| 87 |
+
"k_proj",
|
| 88 |
+
"v_proj",
|
| 89 |
+
"o_proj",
|
| 90 |
+
"gate_proj",
|
| 91 |
+
"up_proj",
|
| 92 |
+
"down_proj"
|
| 93 |
+
],
|
| 94 |
+
"max_length": 1024
|
| 95 |
+
},
|
| 96 |
+
"logit_scale_log": 3.847844123840332,
|
| 97 |
+
"logit_bias": -10.863999366760254,
|
| 98 |
+
"architectures": [
|
| 99 |
+
"JECTImageEncoderModel"
|
| 100 |
+
],
|
| 101 |
+
"auto_map": {
|
| 102 |
+
"AutoConfig": "configuration_ject.JECTConfig",
|
| 103 |
+
"AutoModel": "modeling_ject.JECTImageEncoderModel"
|
| 104 |
+
}
|
| 105 |
+
}
|
configuration_ject.py
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Configuration for the JECT chest-CT encoder."""
|
| 2 |
+
from transformers import PretrainedConfig
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
class JECTConfig(PretrainedConfig):
|
| 6 |
+
model_type = "ject-ct"
|
| 7 |
+
|
| 8 |
+
def __init__(self, encoder=None, readout=None, input=None, text=None, logit_scale_log=0.0, logit_bias=0.0, **kwargs):
|
| 9 |
+
self.encoder = encoder or {}
|
| 10 |
+
self.readout = readout or {}
|
| 11 |
+
self.input = input or {}
|
| 12 |
+
self.text = text or {}
|
| 13 |
+
self.logit_scale_log = logit_scale_log
|
| 14 |
+
self.logit_bias = logit_bias
|
| 15 |
+
super().__init__(**kwargs)
|
ject_encoder.py
ADDED
|
@@ -0,0 +1,734 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""JECT image encoder: standalone architecture definition.
|
| 2 |
+
|
| 3 |
+
Depends only on ``torch``. Defines the 3D Swin-UNETR-style hierarchical encoder,
|
| 4 |
+
the dense latent decoder on top of it, and the config needed to rebuild both from
|
| 5 |
+
a training checkpoint. Training-only components (EMA, predictors, text heads,
|
| 6 |
+
losses) are not included.
|
| 7 |
+
|
| 8 |
+
Encoder layout (224 input, patch 4, `huge` variant)::
|
| 9 |
+
|
| 10 |
+
volume [B,1,224,224,224]
|
| 11 |
+
-> PatchEmbed3D (conv4x4x4) -> e0 56^3 x 128
|
| 12 |
+
-> stage1 (2x swin, window 7) -> e1 56^3 x 128
|
| 13 |
+
-> PatchMerge -> stage2 (4x swin) -> e2 28^3 x 256
|
| 14 |
+
-> PatchMerge -> +pos_embed -> stage3 (9x swin) -> e3 14^3 x 512
|
| 15 |
+
-> PatchMerge -> +pos_embed_s4 -> stage4 (9x global attn) -> e4 7^3 x 1024
|
| 16 |
+
|
| 17 |
+
At 336 input the grids are 84/42/21/11. The learned s3/s4 absolute position
|
| 18 |
+
embeddings are resolution-specific, so each input size has its own checkpoint.
|
| 19 |
+
"""
|
| 20 |
+
|
| 21 |
+
from __future__ import annotations
|
| 22 |
+
|
| 23 |
+
from dataclasses import dataclass, field
|
| 24 |
+
from typing import Dict, Optional, Tuple
|
| 25 |
+
|
| 26 |
+
import torch
|
| 27 |
+
import torch.nn as nn
|
| 28 |
+
import torch.nn.functional as F
|
| 29 |
+
|
| 30 |
+
Triple = Tuple[int, int, int]
|
| 31 |
+
|
| 32 |
+
# Per-variant (dims, heads, depths) for scales (s1, s2, s3, s4).
|
| 33 |
+
VARIANTS = {
|
| 34 |
+
"tiny": dict(dims=(32, 64, 128, 256), heads=(2, 4, 8, 8), depths=(2, 2, 2, 2)),
|
| 35 |
+
"small": dict(dims=(48, 96, 192, 384), heads=(3, 6, 12, 12), depths=(2, 2, 4, 2)),
|
| 36 |
+
"base": dict(dims=(64, 128, 256, 512), heads=(4, 8, 16, 16), depths=(2, 2, 4, 2)),
|
| 37 |
+
"large": dict(dims=(96, 192, 384, 768), heads=(6, 12, 24, 24), depths=(2, 2, 8, 4)),
|
| 38 |
+
"huge": dict(dims=(128, 256, 512, 1024), heads=(8, 16, 32, 32), depths=(2, 4, 9, 9)),
|
| 39 |
+
}
|
| 40 |
+
|
| 41 |
+
# HU windows the encoder was trained with. `window_id` is an additive embedding on the
|
| 42 |
+
# patch tokens, so the id passed at inference must match the window actually applied.
|
| 43 |
+
WINDOW_PARAMS = {
|
| 44 |
+
0: {"center": 450, "width": 2100}, # lung: -600 .. 1500
|
| 45 |
+
1: {"center": 195, "width": 310}, # mediastinal: 40 .. 350
|
| 46 |
+
2: {"center": 0, "width": 2000}, # general: -1000 .. 1000 (default)
|
| 47 |
+
3: {"center": 1500, "width": 5000}, # full: -1000 .. 4000
|
| 48 |
+
}
|
| 49 |
+
GENERAL_WINDOW_ID = 2
|
| 50 |
+
|
| 51 |
+
# ct_type to additive embedding index.
|
| 52 |
+
CT_TYPE_MAP = {"non-contrast": 0, "contrast": 1, "ldct": 2}
|
| 53 |
+
DEFAULT_CT_TYPE = 0
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def _triple(x) -> Triple:
|
| 57 |
+
if isinstance(x, int):
|
| 58 |
+
return (x, x, x)
|
| 59 |
+
vals = tuple(int(v) for v in x)
|
| 60 |
+
if len(vals) == 1:
|
| 61 |
+
return (vals[0],) * 3
|
| 62 |
+
if len(vals) != 3:
|
| 63 |
+
raise ValueError(f"expected an int or a 1/3-length iterable, got {x!r}")
|
| 64 |
+
return vals
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
# ---------------------------------------------------------------------------
|
| 68 |
+
# Config
|
| 69 |
+
# ---------------------------------------------------------------------------
|
| 70 |
+
@dataclass
|
| 71 |
+
class EncoderConfig:
|
| 72 |
+
"""Encoder geometry and width; normally built with ``from_training_config``."""
|
| 73 |
+
|
| 74 |
+
input_size: Triple = (224, 224, 224)
|
| 75 |
+
patch_size: Triple = (4, 4, 4)
|
| 76 |
+
in_channels: int = 1
|
| 77 |
+
variant: str = "huge"
|
| 78 |
+
|
| 79 |
+
local_block_type: str = "swin" # stages 1-2
|
| 80 |
+
stage3_block_type: str = "swin"
|
| 81 |
+
stage4_block_type: str = "global" # full attention over the 7^3 / 11^3 grid
|
| 82 |
+
window: Triple = (7, 7, 7)
|
| 83 |
+
mlp_ratio: float = 4.0
|
| 84 |
+
drop: float = 0.0
|
| 85 |
+
num_ct_types: int = 3
|
| 86 |
+
# Cumulative bottom-up skips of pooled finer features into the s2/s3/s4 stage entries.
|
| 87 |
+
skip_fine: bool = False
|
| 88 |
+
# Pooling for the s3 skip: "maxmean" ([max, mean] cell pools) or "attention" (LocalAttentionPool3D).
|
| 89 |
+
skip_pool_s3: str = "maxmean"
|
| 90 |
+
# Pooling for the s2 and s4 skips: "maxmean" or "attention".
|
| 91 |
+
skip_pool_s2s4: str = "maxmean"
|
| 92 |
+
|
| 93 |
+
# filled by __post_init__
|
| 94 |
+
dims: Tuple[int, ...] = field(default_factory=tuple)
|
| 95 |
+
heads: Tuple[int, ...] = field(default_factory=tuple)
|
| 96 |
+
depths: Tuple[int, ...] = field(default_factory=tuple)
|
| 97 |
+
grid_s1: Triple = (0, 0, 0)
|
| 98 |
+
grid_s2: Triple = (0, 0, 0)
|
| 99 |
+
grid_s3: Triple = (0, 0, 0)
|
| 100 |
+
grid_s4: Triple = (0, 0, 0)
|
| 101 |
+
|
| 102 |
+
def __post_init__(self):
|
| 103 |
+
self.input_size = _triple(self.input_size)
|
| 104 |
+
self.patch_size = _triple(self.patch_size)
|
| 105 |
+
self.window = _triple(self.window)
|
| 106 |
+
if self.variant not in VARIANTS:
|
| 107 |
+
raise ValueError(f"unknown variant {self.variant!r}; expected one of {sorted(VARIANTS)}")
|
| 108 |
+
spec = VARIANTS[self.variant]
|
| 109 |
+
if self.skip_pool_s3 not in ("maxmean", "attention"):
|
| 110 |
+
raise ValueError(f"unknown skip_pool_s3 {self.skip_pool_s3!r}; expected 'maxmean' or 'attention'")
|
| 111 |
+
if self.skip_pool_s3 != "maxmean" and not self.skip_fine:
|
| 112 |
+
raise ValueError("skip_pool_s3='attention' requires skip_fine (mirrors pretrain)")
|
| 113 |
+
if self.skip_pool_s2s4 not in ("maxmean", "attention"):
|
| 114 |
+
raise ValueError(f"unknown skip_pool_s2s4 {self.skip_pool_s2s4!r}; expected 'maxmean' or 'attention'")
|
| 115 |
+
if self.skip_pool_s2s4 != "maxmean" and not self.skip_fine:
|
| 116 |
+
raise ValueError("skip_pool_s2s4='attention' requires skip_fine (mirrors pretrain)")
|
| 117 |
+
self.dims = tuple(spec["dims"])
|
| 118 |
+
self.heads = tuple(spec["heads"])
|
| 119 |
+
self.depths = tuple(spec["depths"])
|
| 120 |
+
for name, v, p in zip("DHW", self.input_size, self.patch_size):
|
| 121 |
+
if v % p:
|
| 122 |
+
raise ValueError(f"input_size {name}={v} not divisible by patch_size {p}")
|
| 123 |
+
s1 = tuple(v // p for v, p in zip(self.input_size, self.patch_size))
|
| 124 |
+
s2 = tuple(g // 2 for g in s1)
|
| 125 |
+
s3 = tuple(g // 4 for g in s1)
|
| 126 |
+
# PatchMerge3D pads odd grids before halving, so s4 = ceil(s3 / 2) (e.g. s3 21 gives s4 11).
|
| 127 |
+
s4 = tuple((g + 1) // 2 for g in s3)
|
| 128 |
+
self.grid_s1, self.grid_s2, self.grid_s3, self.grid_s4 = s1, s2, s3, s4
|
| 129 |
+
|
| 130 |
+
# -- convenience ---------------------------------------------------------
|
| 131 |
+
@property
|
| 132 |
+
def num_tokens(self) -> Dict[str, int]:
|
| 133 |
+
return {f"s{i}": g[0] * g[1] * g[2]
|
| 134 |
+
for i, g in enumerate((self.grid_s1, self.grid_s2, self.grid_s3, self.grid_s4), 1)}
|
| 135 |
+
|
| 136 |
+
@property
|
| 137 |
+
def grids(self) -> Dict[str, Triple]:
|
| 138 |
+
return {"s1": self.grid_s1, "s2": self.grid_s2, "s3": self.grid_s3, "s4": self.grid_s4}
|
| 139 |
+
|
| 140 |
+
@property
|
| 141 |
+
def channels(self) -> Dict[str, int]:
|
| 142 |
+
return {f"s{i}": d for i, d in enumerate(self.dims, 1)}
|
| 143 |
+
|
| 144 |
+
@classmethod
|
| 145 |
+
def from_training_config(cls, saved: dict) -> "EncoderConfig":
|
| 146 |
+
"""Rebuild from the `config` dict stored inside a JECT training checkpoint.
|
| 147 |
+
|
| 148 |
+
The input size is derived from ``grid_size_s1 * patch_size`` rather than the ``volume``
|
| 149 |
+
block, and the derived grids and widths are checked against the saved ones.
|
| 150 |
+
"""
|
| 151 |
+
patch = saved.get("patch", {})
|
| 152 |
+
model = saved.get("model", {})
|
| 153 |
+
psize = _triple(patch.get("patch_size", (4, 4, 4)))
|
| 154 |
+
g1 = _triple(patch["grid_size_s1"])
|
| 155 |
+
cfg = cls(
|
| 156 |
+
input_size=tuple(g * p for g, p in zip(g1, psize)),
|
| 157 |
+
patch_size=psize,
|
| 158 |
+
in_channels=int(saved.get("volume", {}).get("channels", 1)),
|
| 159 |
+
variant=str(model.get("variant", "huge")),
|
| 160 |
+
local_block_type=str(model.get("local_block_type", "swin")),
|
| 161 |
+
stage3_block_type=str(model.get("stage3_block_type", "swin")),
|
| 162 |
+
stage4_block_type=str(model.get("stage4_block_type", "global")),
|
| 163 |
+
window=_triple(model.get("swin_window_size", (7, 7, 7))),
|
| 164 |
+
mlp_ratio=float(model.get("mlp_ratio", 4.0)),
|
| 165 |
+
drop=0.0, # no dropout at inference
|
| 166 |
+
num_ct_types=int(model.get("num_ct_types", 3)),
|
| 167 |
+
skip_fine=bool(model.get("skip_fine", False)),
|
| 168 |
+
skip_pool_s3=str(model.get("skip_pool_s3", "maxmean")),
|
| 169 |
+
skip_pool_s2s4=str(model.get("skip_pool_s2s4", "maxmean")),
|
| 170 |
+
)
|
| 171 |
+
for name, derived in (("grid_size_s2", cfg.grid_s2), ("grid_size_s3", cfg.grid_s3),
|
| 172 |
+
("grid_size_s4", cfg.grid_s4)):
|
| 173 |
+
if name in patch and _triple(patch[name]) != derived:
|
| 174 |
+
raise ValueError(f"geometry mismatch: checkpoint {name}={tuple(patch[name])} "
|
| 175 |
+
f"but input_size {cfg.input_size} implies {derived}")
|
| 176 |
+
for key, want in (("dim_s1", cfg.dims[0]), ("dim_s2", cfg.dims[1]),
|
| 177 |
+
("dim_s3", cfg.dims[2]), ("dim_s4", cfg.dims[3])):
|
| 178 |
+
if key in model and int(model[key]) != want:
|
| 179 |
+
raise ValueError(f"width mismatch: checkpoint {key}={model[key]} but "
|
| 180 |
+
f"variant {cfg.variant!r} implies {want}")
|
| 181 |
+
return cfg
|
| 182 |
+
|
| 183 |
+
|
| 184 |
+
# ---------------------------------------------------------------------------
|
| 185 |
+
# Shared pieces
|
| 186 |
+
# ---------------------------------------------------------------------------
|
| 187 |
+
class Mlp(nn.Module):
|
| 188 |
+
def __init__(self, dim: int, mlp_ratio: float = 4.0, drop: float = 0.0):
|
| 189 |
+
super().__init__()
|
| 190 |
+
hidden = int(dim * mlp_ratio)
|
| 191 |
+
self.fc1 = nn.Linear(dim, hidden)
|
| 192 |
+
self.act = nn.GELU()
|
| 193 |
+
self.fc2 = nn.Linear(hidden, dim)
|
| 194 |
+
self.drop = nn.Dropout(drop)
|
| 195 |
+
|
| 196 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 197 |
+
return self.drop(self.fc2(self.drop(self.act(self.fc1(x)))))
|
| 198 |
+
|
| 199 |
+
|
| 200 |
+
def to_grid(x: torch.Tensor, grid: Triple) -> torch.Tensor:
|
| 201 |
+
"""[B, N, C] -> [B, C, D, H, W]."""
|
| 202 |
+
B, N, C = x.shape
|
| 203 |
+
D, H, W = grid
|
| 204 |
+
if N != D * H * W:
|
| 205 |
+
raise ValueError(f"token count {N} does not match grid {grid} ({D * H * W})")
|
| 206 |
+
return x.transpose(1, 2).reshape(B, C, D, H, W)
|
| 207 |
+
|
| 208 |
+
|
| 209 |
+
def to_tokens(x: torch.Tensor) -> torch.Tensor:
|
| 210 |
+
"""[B, C, D, H, W] -> [B, N, C]."""
|
| 211 |
+
return x.flatten(2).transpose(1, 2)
|
| 212 |
+
|
| 213 |
+
|
| 214 |
+
class Attention(nn.Module):
|
| 215 |
+
"""Multi-head self-attention over all tokens (no positional bias)."""
|
| 216 |
+
|
| 217 |
+
def __init__(self, dim: int, num_heads: int, attn_drop: float = 0.0, proj_drop: float = 0.0):
|
| 218 |
+
super().__init__()
|
| 219 |
+
if dim % num_heads:
|
| 220 |
+
raise ValueError(f"dim={dim} not divisible by num_heads={num_heads}")
|
| 221 |
+
self.num_heads = num_heads
|
| 222 |
+
self.head_dim = dim // num_heads
|
| 223 |
+
self.qkv = nn.Linear(dim, dim * 3)
|
| 224 |
+
self.attn_drop_p = float(attn_drop)
|
| 225 |
+
self.proj = nn.Linear(dim, dim)
|
| 226 |
+
self.proj_drop = nn.Dropout(proj_drop)
|
| 227 |
+
|
| 228 |
+
def forward(self, x: torch.Tensor, bias: Optional[torch.Tensor] = None) -> torch.Tensor:
|
| 229 |
+
B, N, C = x.shape
|
| 230 |
+
qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, self.head_dim).permute(2, 0, 3, 1, 4)
|
| 231 |
+
q, k, v = qkv.unbind(0)
|
| 232 |
+
attn_mask = bias.unsqueeze(0).to(q.dtype) if bias is not None else None
|
| 233 |
+
out = F.scaled_dot_product_attention(
|
| 234 |
+
q, k, v, attn_mask=attn_mask, dropout_p=self.attn_drop_p if self.training else 0.0)
|
| 235 |
+
out = out.transpose(1, 2).reshape(B, N, C)
|
| 236 |
+
return self.proj_drop(self.proj(out))
|
| 237 |
+
|
| 238 |
+
|
| 239 |
+
class TransformerBlock3D(nn.Module):
|
| 240 |
+
"""Pre-norm global transformer block (stage 4)."""
|
| 241 |
+
|
| 242 |
+
def __init__(self, dim: int, num_heads: int, mlp_ratio: float = 4.0, drop: float = 0.0):
|
| 243 |
+
super().__init__()
|
| 244 |
+
self.norm1 = nn.LayerNorm(dim)
|
| 245 |
+
self.attn = Attention(dim, num_heads, attn_drop=drop, proj_drop=drop)
|
| 246 |
+
self.norm2 = nn.LayerNorm(dim)
|
| 247 |
+
self.mlp = Mlp(dim, mlp_ratio, drop)
|
| 248 |
+
|
| 249 |
+
def forward(self, x: torch.Tensor, grid: Triple) -> torch.Tensor:
|
| 250 |
+
x = x + self.attn(self.norm1(x))
|
| 251 |
+
x = x + self.mlp(self.norm2(x))
|
| 252 |
+
return x
|
| 253 |
+
|
| 254 |
+
|
| 255 |
+
def _pad_to_window(x: torch.Tensor, window: Triple):
|
| 256 |
+
"""Pad a [B, D, H, W, C] grid up to a multiple of the window on each spatial axis."""
|
| 257 |
+
_, D, H, W, _ = x.shape
|
| 258 |
+
wd, wh, ww = window
|
| 259 |
+
pd, ph, pw = (wd - D % wd) % wd, (wh - H % wh) % wh, (ww - W % ww) % ww
|
| 260 |
+
if pd or ph or pw:
|
| 261 |
+
x = F.pad(x, (0, 0, 0, pw, 0, ph, 0, pd)) # pads W, H, D (C last, untouched)
|
| 262 |
+
return x, (D, H, W)
|
| 263 |
+
|
| 264 |
+
|
| 265 |
+
def _win_partition(x: torch.Tensor, window: Triple) -> torch.Tensor:
|
| 266 |
+
B, D, H, W, C = x.shape
|
| 267 |
+
wd, wh, ww = window
|
| 268 |
+
x = x.view(B, D // wd, wd, H // wh, wh, W // ww, ww, C)
|
| 269 |
+
x = x.permute(0, 1, 3, 5, 2, 4, 6, 7).contiguous()
|
| 270 |
+
return x.view(-1, wd * wh * ww, C)
|
| 271 |
+
|
| 272 |
+
|
| 273 |
+
def _win_merge(windows: torch.Tensor, gridp: Triple, window: Triple, B: int) -> torch.Tensor:
|
| 274 |
+
Dp, Hp, Wp = gridp
|
| 275 |
+
wd, wh, ww = window
|
| 276 |
+
C = windows.shape[-1]
|
| 277 |
+
x = windows.view(B, Dp // wd, Hp // wh, Wp // ww, wd, wh, ww, C)
|
| 278 |
+
x = x.permute(0, 1, 4, 2, 5, 3, 6, 7).contiguous()
|
| 279 |
+
return x.view(B, Dp, Hp, Wp, C)
|
| 280 |
+
|
| 281 |
+
|
| 282 |
+
class WindowAttention3D(nn.Module):
|
| 283 |
+
"""Windowed multi-head attention with a 3D relative-position-bias table."""
|
| 284 |
+
|
| 285 |
+
def __init__(self, dim: int, num_heads: int, window: Triple,
|
| 286 |
+
attn_drop: float = 0.0, proj_drop: float = 0.0):
|
| 287 |
+
super().__init__()
|
| 288 |
+
if dim % num_heads:
|
| 289 |
+
raise ValueError(f"dim={dim} not divisible by num_heads={num_heads}")
|
| 290 |
+
self.num_heads = num_heads
|
| 291 |
+
self.head_dim = dim // num_heads
|
| 292 |
+
self.window = window
|
| 293 |
+
wd, wh, ww = window
|
| 294 |
+
self.relative_position_bias_table = nn.Parameter(
|
| 295 |
+
torch.zeros((2 * wd - 1) * (2 * wh - 1) * (2 * ww - 1), num_heads))
|
| 296 |
+
coords = torch.stack(torch.meshgrid(
|
| 297 |
+
torch.arange(wd), torch.arange(wh), torch.arange(ww), indexing="ij"))
|
| 298 |
+
coords_flat = coords.reshape(3, -1)
|
| 299 |
+
rel = (coords_flat[:, :, None] - coords_flat[:, None, :]).permute(1, 2, 0).contiguous()
|
| 300 |
+
rel[..., 0] += wd - 1
|
| 301 |
+
rel[..., 1] += wh - 1
|
| 302 |
+
rel[..., 2] += ww - 1
|
| 303 |
+
rel[..., 0] *= (2 * wh - 1) * (2 * ww - 1)
|
| 304 |
+
rel[..., 1] *= (2 * ww - 1)
|
| 305 |
+
self.register_buffer("relative_position_index", rel.sum(-1).view(-1), persistent=False)
|
| 306 |
+
self.qkv = nn.Linear(dim, dim * 3)
|
| 307 |
+
self.attn_drop_p = float(attn_drop)
|
| 308 |
+
self.proj = nn.Linear(dim, dim)
|
| 309 |
+
self.proj_drop = nn.Dropout(proj_drop)
|
| 310 |
+
|
| 311 |
+
def _bias(self) -> torch.Tensor:
|
| 312 |
+
ws = int(round(self.relative_position_index.numel() ** 0.5))
|
| 313 |
+
table = self.relative_position_bias_table[self.relative_position_index]
|
| 314 |
+
return table.view(ws, ws, -1).permute(2, 0, 1).contiguous() # [heads, ws, ws]
|
| 315 |
+
|
| 316 |
+
def forward(self, windows: torch.Tensor) -> torch.Tensor:
|
| 317 |
+
Bn, ws, C = windows.shape
|
| 318 |
+
qkv = self.qkv(windows).reshape(Bn, ws, 3, self.num_heads, self.head_dim).permute(2, 0, 3, 1, 4)
|
| 319 |
+
q, k, v = qkv.unbind(0)
|
| 320 |
+
attn_mask = self._bias().unsqueeze(0).to(q.dtype)
|
| 321 |
+
out = F.scaled_dot_product_attention(
|
| 322 |
+
q, k, v, attn_mask=attn_mask, dropout_p=self.attn_drop_p if self.training else 0.0)
|
| 323 |
+
out = out.transpose(1, 2).reshape(Bn, ws, C)
|
| 324 |
+
return self.proj_drop(self.proj(out))
|
| 325 |
+
|
| 326 |
+
|
| 327 |
+
class SwinBlock3D(nn.Module):
|
| 328 |
+
"""Shifted-window 3D attention block; pads internally so any grid size works."""
|
| 329 |
+
|
| 330 |
+
def __init__(self, dim: int, num_heads: int, window: Triple,
|
| 331 |
+
mlp_ratio: float = 4.0, drop: float = 0.0, shift: bool = False):
|
| 332 |
+
super().__init__()
|
| 333 |
+
self.window = window
|
| 334 |
+
self.shift_size = tuple(w // 2 for w in window) if shift else (0, 0, 0)
|
| 335 |
+
self.norm1 = nn.LayerNorm(dim)
|
| 336 |
+
self.attn = WindowAttention3D(dim, num_heads, window, attn_drop=drop, proj_drop=drop)
|
| 337 |
+
self.norm2 = nn.LayerNorm(dim)
|
| 338 |
+
self.mlp = Mlp(dim, mlp_ratio, drop)
|
| 339 |
+
|
| 340 |
+
def forward(self, x: torch.Tensor, grid: Triple) -> torch.Tensor:
|
| 341 |
+
B, N, C = x.shape
|
| 342 |
+
D, H, W = grid
|
| 343 |
+
h = self.norm1(x).view(B, D, H, W, C)
|
| 344 |
+
h, _ = _pad_to_window(h, self.window)
|
| 345 |
+
Dp, Hp, Wp = h.shape[1:4]
|
| 346 |
+
if any(s > 0 for s in self.shift_size):
|
| 347 |
+
h = torch.roll(h, shifts=tuple(-s for s in self.shift_size), dims=(1, 2, 3))
|
| 348 |
+
windows = self.attn(_win_partition(h, self.window))
|
| 349 |
+
h = _win_merge(windows, (Dp, Hp, Wp), self.window, B)
|
| 350 |
+
if any(s > 0 for s in self.shift_size):
|
| 351 |
+
h = torch.roll(h, shifts=self.shift_size, dims=(1, 2, 3))
|
| 352 |
+
h = h[:, :D, :H, :W].reshape(B, N, C) # crop the window padding
|
| 353 |
+
x = x + h
|
| 354 |
+
x = x + self.mlp(self.norm2(x))
|
| 355 |
+
return x
|
| 356 |
+
|
| 357 |
+
|
| 358 |
+
class ConvNeXtBlock3D(nn.Module):
|
| 359 |
+
"""Depthwise 3D ConvNeXt block."""
|
| 360 |
+
|
| 361 |
+
def __init__(self, dim: int, mlp_ratio: float = 4.0, drop: float = 0.0):
|
| 362 |
+
super().__init__()
|
| 363 |
+
self.dwconv = nn.Conv3d(dim, dim, kernel_size=7, padding=3, groups=dim)
|
| 364 |
+
self.norm = nn.LayerNorm(dim)
|
| 365 |
+
hidden = int(dim * mlp_ratio)
|
| 366 |
+
self.pw1 = nn.Linear(dim, hidden)
|
| 367 |
+
self.act = nn.GELU()
|
| 368 |
+
self.pw2 = nn.Linear(hidden, dim)
|
| 369 |
+
self.gamma = nn.Parameter(1e-6 * torch.ones(dim))
|
| 370 |
+
|
| 371 |
+
def forward(self, x: torch.Tensor, grid: Triple) -> torch.Tensor:
|
| 372 |
+
y = to_tokens(self.dwconv(to_grid(x, grid)))
|
| 373 |
+
y = self.pw2(self.act(self.pw1(self.norm(y))))
|
| 374 |
+
return x + self.gamma * y
|
| 375 |
+
|
| 376 |
+
|
| 377 |
+
def build_block(block_type: str, dim: int, num_heads: int, window: Triple,
|
| 378 |
+
mlp_ratio: float, drop: float, shift: bool = False) -> nn.Module:
|
| 379 |
+
if block_type == "swin":
|
| 380 |
+
return SwinBlock3D(dim, num_heads, window, mlp_ratio, drop, shift=shift)
|
| 381 |
+
if block_type in ("transformer", "global"):
|
| 382 |
+
return TransformerBlock3D(dim, num_heads, mlp_ratio, drop)
|
| 383 |
+
if block_type == "convnext":
|
| 384 |
+
return ConvNeXtBlock3D(dim, mlp_ratio, drop)
|
| 385 |
+
raise ValueError(f"unknown block type {block_type!r}")
|
| 386 |
+
|
| 387 |
+
|
| 388 |
+
class PatchEmbed3D(nn.Module):
|
| 389 |
+
def __init__(self, channels: int, dim: int, patch_size: Triple):
|
| 390 |
+
super().__init__()
|
| 391 |
+
self.proj = nn.Conv3d(channels, dim, kernel_size=patch_size, stride=patch_size)
|
| 392 |
+
self.norm = nn.LayerNorm(dim)
|
| 393 |
+
|
| 394 |
+
def forward(self, volume: torch.Tensor) -> torch.Tensor:
|
| 395 |
+
return self.norm(to_tokens(self.proj(volume)))
|
| 396 |
+
|
| 397 |
+
|
| 398 |
+
def _pool_fine(x: torch.Tensor, grid: Triple, grid_to: Triple) -> torch.Tensor:
|
| 399 |
+
"""Max- and mean-pool z-major tokens on `grid` into the cells of `grid_to`.
|
| 400 |
+
|
| 401 |
+
[B, N, C] -> [B, N_to, 2C], pooling the k^3 fine tokens of each coarse cell (k = grid / grid_to)."""
|
| 402 |
+
B, N, C = x.shape
|
| 403 |
+
D, H, W = grid
|
| 404 |
+
k = D // grid_to[0]
|
| 405 |
+
if (D, H, W) != (k * grid_to[0], k * grid_to[1], k * grid_to[2]):
|
| 406 |
+
raise ValueError(f"skip_fine needs grid {grid} to be an integer multiple of the coarser grid {grid_to}")
|
| 407 |
+
v = x.transpose(1, 2).reshape(B, C, D, H, W)
|
| 408 |
+
return torch.cat([F.max_pool3d(v, k), F.avg_pool3d(v, k)], dim=1).flatten(2).transpose(1, 2)
|
| 409 |
+
|
| 410 |
+
|
| 411 |
+
class LocalAttentionPool3D(nn.Module):
|
| 412 |
+
"""Coarse-query attention over one cell's fine tokens, with a learned 3D-offset bias.
|
| 413 |
+
|
| 414 |
+
Four heads, 128 internal channels, and the same 2C output width as max/mean pooling.
|
| 415 |
+
Token order within each cell is z/y/x, so each bias index is a distinct relative 3D
|
| 416 |
+
position. Attention never crosses cells.
|
| 417 |
+
"""
|
| 418 |
+
|
| 419 |
+
SDPA_CHUNK = 32768 # max cells per SDPA call
|
| 420 |
+
|
| 421 |
+
def __init__(self, fine_dim: int, coarse_dim: int, factor: int):
|
| 422 |
+
super().__init__()
|
| 423 |
+
self.factor = factor
|
| 424 |
+
self.heads, self.head_dim = 4, 32
|
| 425 |
+
self.fine_norm = nn.LayerNorm(fine_dim)
|
| 426 |
+
self.query_norm = nn.LayerNorm(coarse_dim)
|
| 427 |
+
self.query = nn.Linear(coarse_dim, 128)
|
| 428 |
+
self.key_value = nn.Linear(fine_dim, 256)
|
| 429 |
+
self.relative_bias = nn.Parameter(torch.zeros(4, factor ** 3))
|
| 430 |
+
self.output = nn.Linear(128, 2 * fine_dim)
|
| 431 |
+
|
| 432 |
+
def forward(self, fine: torch.Tensor, coarse: torch.Tensor, grid: Triple,
|
| 433 |
+
grid_to: Triple) -> torch.Tensor:
|
| 434 |
+
B, N, C = fine.shape
|
| 435 |
+
D, H, W = grid_to
|
| 436 |
+
k = self.factor
|
| 437 |
+
if tuple(grid) != (D * k, H * k, W * k) or N != D * H * W * k ** 3:
|
| 438 |
+
raise ValueError("attention skip requires exactly nested fine/coarse grids")
|
| 439 |
+
if coarse.shape[:2] != (B, D * H * W):
|
| 440 |
+
raise ValueError("attention skip query count does not match its coarse grid")
|
| 441 |
+
cells = fine.reshape(B, D, k, H, k, W, k, C)
|
| 442 |
+
cells = cells.permute(0, 1, 3, 5, 2, 4, 6, 7).reshape(B * D * H * W, k ** 3, C)
|
| 443 |
+
kv = self.key_value(self.fine_norm(cells))
|
| 444 |
+
kv = kv.reshape(-1, k ** 3, 2, self.heads, self.head_dim).permute(2, 0, 3, 1, 4)
|
| 445 |
+
q = self.query(self.query_norm(coarse)).reshape(-1, self.heads, 1, self.head_dim)
|
| 446 |
+
bias = self.relative_bias.to(q.dtype)[None, :, None, :]
|
| 447 |
+
# One SDPA batch entry per cell (B*D*H*W). Above ~65k entries the fused kernel exceeds the CUDA
|
| 448 |
+
# launch-grid limit, so large inputs are processed in chunks; the result is unchanged.
|
| 449 |
+
n = q.shape[0]
|
| 450 |
+
if n <= self.SDPA_CHUNK:
|
| 451 |
+
pooled = F.scaled_dot_product_attention(q, kv[0], kv[1], attn_mask=bias, dropout_p=0.0)
|
| 452 |
+
else:
|
| 453 |
+
pooled = torch.cat([F.scaled_dot_product_attention(q[i:i + self.SDPA_CHUNK], kv[0][i:i + self.SDPA_CHUNK],
|
| 454 |
+
kv[1][i:i + self.SDPA_CHUNK], attn_mask=bias, dropout_p=0.0)
|
| 455 |
+
for i in range(0, n, self.SDPA_CHUNK)], dim=0)
|
| 456 |
+
return self.output(pooled.reshape(B, D * H * W, 128))
|
| 457 |
+
|
| 458 |
+
|
| 459 |
+
class PatchMerge3D(nn.Module):
|
| 460 |
+
"""2x2x2 patch merging: [B, N, C] -> [B, N/8, out_dim] (odd grids are padded first)."""
|
| 461 |
+
|
| 462 |
+
def __init__(self, in_dim: int, out_dim: int):
|
| 463 |
+
super().__init__()
|
| 464 |
+
self.norm = nn.LayerNorm(8 * in_dim)
|
| 465 |
+
self.reduce = nn.Linear(8 * in_dim, out_dim, bias=False)
|
| 466 |
+
|
| 467 |
+
def forward(self, x: torch.Tensor, grid: Triple) -> Tuple[torch.Tensor, Triple]:
|
| 468 |
+
B, N, C = x.shape
|
| 469 |
+
D, H, W = grid
|
| 470 |
+
if D % 2 or H % 2 or W % 2:
|
| 471 |
+
x = x.view(B, D, H, W, C)
|
| 472 |
+
x = F.pad(x, (0, 0, 0, W % 2, 0, H % 2, 0, D % 2))
|
| 473 |
+
D, H, W = D + D % 2, H + H % 2, W + W % 2
|
| 474 |
+
x = x.reshape(B, D * H * W, C)
|
| 475 |
+
x = x.view(B, D // 2, 2, H // 2, 2, W // 2, 2, C)
|
| 476 |
+
x = x.permute(0, 1, 3, 5, 2, 4, 6, 7).contiguous()
|
| 477 |
+
x = x.view(B, (D // 2) * (H // 2) * (W // 2), 8 * C)
|
| 478 |
+
return self.reduce(self.norm(x)), (D // 2, H // 2, W // 2)
|
| 479 |
+
|
| 480 |
+
|
| 481 |
+
# ---------------------------------------------------------------------------
|
| 482 |
+
# Encoder / dense decoder
|
| 483 |
+
# ---------------------------------------------------------------------------
|
| 484 |
+
class JECTEncoder(nn.Module):
|
| 485 |
+
"""Hierarchical 3D encoder producing token features at the s1/s2/s3/s4 grids.
|
| 486 |
+
|
| 487 |
+
Module and parameter names match the pretraining checkpoint exactly, so
|
| 488 |
+
``state_dict`` keys load without remapping (strip the ``target_encoder.``
|
| 489 |
+
or ``student_encoder.`` prefix first).
|
| 490 |
+
"""
|
| 491 |
+
|
| 492 |
+
def __init__(self, config: EncoderConfig):
|
| 493 |
+
super().__init__()
|
| 494 |
+
self.config = config
|
| 495 |
+
d1, d2, d3, d4 = config.dims
|
| 496 |
+
h1, h2, h3, h4 = config.heads
|
| 497 |
+
n1, n2, n3, n4 = config.depths
|
| 498 |
+
win = config.window
|
| 499 |
+
self.grid_s1, self.grid_s2 = config.grid_s1, config.grid_s2
|
| 500 |
+
self.grid_s3, self.grid_s4 = config.grid_s3, config.grid_s4
|
| 501 |
+
|
| 502 |
+
self.patch_embed = PatchEmbed3D(config.in_channels, d1, config.patch_size)
|
| 503 |
+
self.window_embed = nn.Embedding(4, d1) # HU-window id (see WINDOW_PARAMS)
|
| 504 |
+
self.ct_type_embed = nn.Embedding(config.num_ct_types, d1)
|
| 505 |
+
self.mask_token = nn.Parameter(torch.zeros(1, 1, d1)) # pretraining only; unused here
|
| 506 |
+
|
| 507 |
+
self.stage1 = nn.ModuleList([
|
| 508 |
+
build_block(config.local_block_type, d1, h1, win, config.mlp_ratio, config.drop,
|
| 509 |
+
shift=(i % 2 == 1)) for i in range(n1)])
|
| 510 |
+
self.merge1 = PatchMerge3D(d1, d2)
|
| 511 |
+
self.stage2 = nn.ModuleList([
|
| 512 |
+
build_block(config.local_block_type, d2, h2, win, config.mlp_ratio, config.drop,
|
| 513 |
+
shift=(i % 2 == 1)) for i in range(n2)])
|
| 514 |
+
self.merge2 = PatchMerge3D(d2, d3)
|
| 515 |
+
# skip_fine: at each stage entry, a projection of pooled features from every finer representation
|
| 516 |
+
# (patch embeddings, e1, e2, e3).
|
| 517 |
+
self.skip_fine = bool(config.skip_fine)
|
| 518 |
+
if self.skip_fine:
|
| 519 |
+
self.skip2_norm = nn.LayerNorm(2 * (d1 + d1)); self.skip2_proj = nn.Linear(2 * (d1 + d1), d2)
|
| 520 |
+
self.skip3_norm = nn.LayerNorm(2 * (d1 + d1 + d2)); self.skip3_proj = nn.Linear(2 * (d1 + d1 + d2), d3)
|
| 521 |
+
self.skip4_norm = nn.LayerNorm(2 * (d1 + d1 + d2 + d3)); self.skip4_proj = nn.Linear(2 * (d1 + d1 + d2 + d3), d4)
|
| 522 |
+
# skip_pool_s3="attention": one LocalAttentionPool3D per s3 skip source instead of max/mean pooling.
|
| 523 |
+
self.skip_pool_s3 = str(config.skip_pool_s3)
|
| 524 |
+
if self.skip_fine and self.skip_pool_s3 == "attention":
|
| 525 |
+
self.skip3_attention = nn.ModuleList([
|
| 526 |
+
LocalAttentionPool3D(d1, d3, 4),
|
| 527 |
+
LocalAttentionPool3D(d1, d3, 4),
|
| 528 |
+
LocalAttentionPool3D(d2, d3, 2),
|
| 529 |
+
])
|
| 530 |
+
# skip_pool_s2s4="attention": the same per-cell attention pooling at the s2 and s4 entries.
|
| 531 |
+
self.skip_pool_s2s4 = str(getattr(config, "skip_pool_s2s4", "maxmean"))
|
| 532 |
+
if self.skip_fine and self.skip_pool_s2s4 == "attention":
|
| 533 |
+
self.skip2_attention = nn.ModuleList([
|
| 534 |
+
LocalAttentionPool3D(d1, d2, 2),
|
| 535 |
+
LocalAttentionPool3D(d1, d2, 2),
|
| 536 |
+
])
|
| 537 |
+
self.skip4_attention = nn.ModuleList([
|
| 538 |
+
LocalAttentionPool3D(d1, d4, 8),
|
| 539 |
+
LocalAttentionPool3D(d1, d4, 8),
|
| 540 |
+
LocalAttentionPool3D(d2, d4, 4),
|
| 541 |
+
LocalAttentionPool3D(d3, d4, 2),
|
| 542 |
+
])
|
| 543 |
+
self.pos_embed = nn.Parameter(torch.zeros(1, config.num_tokens["s3"], d3))
|
| 544 |
+
self.stage3 = nn.ModuleList([
|
| 545 |
+
build_block(config.stage3_block_type, d3, h3, win, config.mlp_ratio, config.drop,
|
| 546 |
+
shift=(i % 2 == 1)) for i in range(n3)])
|
| 547 |
+
self.merge3 = PatchMerge3D(d3, d4)
|
| 548 |
+
self.pos_embed_s4 = nn.Parameter(torch.zeros(1, config.num_tokens["s4"], d4))
|
| 549 |
+
self.stage4 = nn.ModuleList([
|
| 550 |
+
build_block(config.stage4_block_type, d4, h4, win, config.mlp_ratio, config.drop,
|
| 551 |
+
shift=(i % 2 == 1)) for i in range(n4)])
|
| 552 |
+
|
| 553 |
+
def trunk(self, volume: torch.Tensor,
|
| 554 |
+
window_id: Optional[torch.Tensor] = None,
|
| 555 |
+
ct_type_id: Optional[torch.Tensor] = None,
|
| 556 |
+
mask_s1: Optional[torch.Tensor] = None) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
| 557 |
+
"""Run stages 1-3 and merge3 (with skips), stopping before pos_embed_s4 and stage 4.
|
| 558 |
+
|
| 559 |
+
Useful for fine-tuning only stage 4. Returns ``(x_s4_in, e1, e2, e3)``.
|
| 560 |
+
"""
|
| 561 |
+
expected = self.config.input_size
|
| 562 |
+
if tuple(volume.shape[-3:]) != tuple(expected):
|
| 563 |
+
raise ValueError(f"expected volume {expected}, got {tuple(volume.shape[-3:])}. "
|
| 564 |
+
f"This checkpoint is fixed-resolution (learned s3/s4 position "
|
| 565 |
+
f"embeddings); use the checkpoint matching your input size.")
|
| 566 |
+
x = self.patch_embed(volume)
|
| 567 |
+
if window_id is not None:
|
| 568 |
+
x = x + self.window_embed(window_id).unsqueeze(1)
|
| 569 |
+
if ct_type_id is not None:
|
| 570 |
+
x = x + self.ct_type_embed(ct_type_id).unsqueeze(1)
|
| 571 |
+
if mask_s1 is not None:
|
| 572 |
+
x = torch.where(mask_s1.unsqueeze(-1), self.mask_token.to(x.dtype), x)
|
| 573 |
+
p0 = x if self.skip_fine else None
|
| 574 |
+
|
| 575 |
+
for blk in self.stage1:
|
| 576 |
+
x = blk(x, self.grid_s1)
|
| 577 |
+
e1 = x
|
| 578 |
+
x, _ = self.merge1(x, self.grid_s1)
|
| 579 |
+
if self.skip_fine:
|
| 580 |
+
if self.skip_pool_s2s4 == "attention":
|
| 581 |
+
sources = ((p0, self.grid_s1), (e1, self.grid_s1))
|
| 582 |
+
fine = torch.cat([pool(f, x, grid, self.grid_s2)
|
| 583 |
+
for pool, (f, grid) in zip(self.skip2_attention, sources)], dim=-1)
|
| 584 |
+
else:
|
| 585 |
+
fine = torch.cat([_pool_fine(p0, self.grid_s1, self.grid_s2), _pool_fine(e1, self.grid_s1, self.grid_s2)], dim=-1)
|
| 586 |
+
x = x + self.skip2_proj(self.skip2_norm(fine)).to(x.dtype)
|
| 587 |
+
for blk in self.stage2:
|
| 588 |
+
x = blk(x, self.grid_s2)
|
| 589 |
+
e2 = x
|
| 590 |
+
x, _ = self.merge2(x, self.grid_s2)
|
| 591 |
+
if self.skip_fine:
|
| 592 |
+
if self.skip_pool_s3 == "attention":
|
| 593 |
+
sources = ((p0, self.grid_s1), (e1, self.grid_s1), (e2, self.grid_s2))
|
| 594 |
+
fine = torch.cat([pool(f, x, grid, self.grid_s3)
|
| 595 |
+
for pool, (f, grid) in zip(self.skip3_attention, sources)], dim=-1)
|
| 596 |
+
else:
|
| 597 |
+
fine = torch.cat([_pool_fine(p0, self.grid_s1, self.grid_s3), _pool_fine(e1, self.grid_s1, self.grid_s3),
|
| 598 |
+
_pool_fine(e2, self.grid_s2, self.grid_s3)], dim=-1)
|
| 599 |
+
x = x + self.skip3_proj(self.skip3_norm(fine)).to(x.dtype)
|
| 600 |
+
x = x + self.pos_embed
|
| 601 |
+
for blk in self.stage3:
|
| 602 |
+
x = blk(x, self.grid_s3)
|
| 603 |
+
e3 = x
|
| 604 |
+
x, _ = self.merge3(x, self.grid_s3)
|
| 605 |
+
if self.skip_fine:
|
| 606 |
+
if self.skip_pool_s2s4 == "attention":
|
| 607 |
+
sources = ((p0, self.grid_s1), (e1, self.grid_s1), (e2, self.grid_s2), (e3, self.grid_s3))
|
| 608 |
+
fine = torch.cat([pool(f, x, grid, self.grid_s4)
|
| 609 |
+
for pool, (f, grid) in zip(self.skip4_attention, sources)], dim=-1)
|
| 610 |
+
else:
|
| 611 |
+
fine = torch.cat([_pool_fine(p0, self.grid_s1, self.grid_s4), _pool_fine(e1, self.grid_s1, self.grid_s4),
|
| 612 |
+
_pool_fine(e2, self.grid_s2, self.grid_s4), _pool_fine(e3, self.grid_s3, self.grid_s4)], dim=-1)
|
| 613 |
+
x = x + self.skip4_proj(self.skip4_norm(fine)).to(x.dtype)
|
| 614 |
+
return x, e1, e2, e3
|
| 615 |
+
|
| 616 |
+
def forward(self, volume: torch.Tensor,
|
| 617 |
+
window_id: Optional[torch.Tensor] = None,
|
| 618 |
+
ct_type_id: Optional[torch.Tensor] = None,
|
| 619 |
+
mask_s1: Optional[torch.Tensor] = None) -> Dict[str, torch.Tensor]:
|
| 620 |
+
"""Encode a volume.
|
| 621 |
+
|
| 622 |
+
Args:
|
| 623 |
+
volume: [B, 1, D, H, W], HU-windowed to [0, 1] (see ``preprocess``).
|
| 624 |
+
window_id: [B] long, the HU window applied (see WINDOW_PARAMS).
|
| 625 |
+
ct_type_id: [B] long, optional (see CT_TYPE_MAP).
|
| 626 |
+
mask_s1: [B, N_s1] bool, optional; replaces tokens with the mask token.
|
| 627 |
+
|
| 628 |
+
Returns:
|
| 629 |
+
dict of token features ``{e1, e2, e3, e4}``, each [B, N_s, C_s].
|
| 630 |
+
"""
|
| 631 |
+
x, e1, e2, e3 = self.trunk(volume, window_id, ct_type_id, mask_s1)
|
| 632 |
+
x = x + self.pos_embed_s4
|
| 633 |
+
for blk in self.stage4:
|
| 634 |
+
x = blk(x, self.grid_s4)
|
| 635 |
+
e4 = x
|
| 636 |
+
return {"e1": e1, "e2": e2, "e3": e3, "e4": e4}
|
| 637 |
+
|
| 638 |
+
|
| 639 |
+
class DenseLatentDecoder(nn.Module):
|
| 640 |
+
"""Upsampling decoder producing dense latents at the s3/s2/s1 grids.
|
| 641 |
+
|
| 642 |
+
These latents are the JEPA targets during pretraining. Optional at inference; the
|
| 643 |
+
encoder features are the recommended readout for probing and retrieval.
|
| 644 |
+
"""
|
| 645 |
+
|
| 646 |
+
def __init__(self, config: EncoderConfig):
|
| 647 |
+
super().__init__()
|
| 648 |
+
d1, d2, d3, _ = config.dims
|
| 649 |
+
self.grid_s1, self.grid_s2, self.grid_s3 = config.grid_s1, config.grid_s2, config.grid_s3
|
| 650 |
+
self.block3 = Mlp(d3, config.mlp_ratio, config.drop)
|
| 651 |
+
self.norm3 = nn.LayerNorm(d3)
|
| 652 |
+
self.up3to2 = nn.Linear(d3, d2)
|
| 653 |
+
self.fuse2 = nn.LayerNorm(d2)
|
| 654 |
+
self.block2 = Mlp(d2, config.mlp_ratio, config.drop)
|
| 655 |
+
self.up2to1 = nn.Linear(d2, d1)
|
| 656 |
+
self.fuse1 = nn.LayerNorm(d1)
|
| 657 |
+
self.block1 = Mlp(d1, config.mlp_ratio, config.drop)
|
| 658 |
+
self.out1 = nn.LayerNorm(d1)
|
| 659 |
+
self.out2 = nn.LayerNorm(d2)
|
| 660 |
+
self.out3 = nn.LayerNorm(d3)
|
| 661 |
+
|
| 662 |
+
@staticmethod
|
| 663 |
+
def _upsample(x: torch.Tensor, src_grid: Triple, dst_grid: Triple) -> torch.Tensor:
|
| 664 |
+
y = F.interpolate(to_grid(x, src_grid), size=tuple(dst_grid),
|
| 665 |
+
mode="trilinear", align_corners=False)
|
| 666 |
+
return to_tokens(y)
|
| 667 |
+
|
| 668 |
+
def forward(self, enc: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]:
|
| 669 |
+
e1, e2, e3 = enc["e1"], enc["e2"], enc["e3"]
|
| 670 |
+
d3 = e3 + self.block3(self.norm3(e3))
|
| 671 |
+
u2 = self.up3to2(self._upsample(d3, self.grid_s3, self.grid_s2))
|
| 672 |
+
d2 = u2 + e2
|
| 673 |
+
d2 = d2 + self.block2(self.fuse2(d2))
|
| 674 |
+
u1 = self.up2to1(self._upsample(d2, self.grid_s2, self.grid_s1))
|
| 675 |
+
d1 = u1 + e1
|
| 676 |
+
d1 = d1 + self.block1(self.fuse1(d1))
|
| 677 |
+
return {"s1": self.out1(d1), "s2": self.out2(d2), "s3": self.out3(d3)}
|
| 678 |
+
|
| 679 |
+
|
| 680 |
+
# ---------------------------------------------------------------------------
|
| 681 |
+
# Checkpoint loading
|
| 682 |
+
# ---------------------------------------------------------------------------
|
| 683 |
+
def _substate(state: Dict[str, torch.Tensor], prefix: str) -> Dict[str, torch.Tensor]:
|
| 684 |
+
p = prefix if prefix.endswith(".") else prefix + "."
|
| 685 |
+
return {k[len(p):]: v for k, v in state.items() if k.startswith(p)}
|
| 686 |
+
|
| 687 |
+
|
| 688 |
+
def load_pretrained(checkpoint_path, stream: str = "target", with_decoder: bool = False,
|
| 689 |
+
map_location: str = "cpu"):
|
| 690 |
+
"""Rebuild the encoder (and optionally the dense decoder) from a training checkpoint.
|
| 691 |
+
|
| 692 |
+
Args:
|
| 693 |
+
checkpoint_path: path to ``checkpoint_epoch_*.pt``.
|
| 694 |
+
stream: ``"target"`` (EMA teacher, recommended) or ``"student"`` (online weights).
|
| 695 |
+
with_decoder: also build and load the dense latent decoder.
|
| 696 |
+
map_location: device passed to ``torch.load``.
|
| 697 |
+
|
| 698 |
+
Returns:
|
| 699 |
+
``(encoder, config, info)``, or ``(encoder, decoder, config, info)`` when
|
| 700 |
+
``with_decoder``. Modules are in eval mode. ``info`` holds the epoch, step,
|
| 701 |
+
stream, checkpoint path and the raw training config.
|
| 702 |
+
"""
|
| 703 |
+
if stream not in ("target", "student"):
|
| 704 |
+
raise ValueError(f"stream must be 'target' or 'student', got {stream!r}")
|
| 705 |
+
ck = torch.load(str(checkpoint_path), map_location=map_location, weights_only=False)
|
| 706 |
+
if "model" not in ck or "config" not in ck:
|
| 707 |
+
raise ValueError(f"{checkpoint_path} is not an JECT training checkpoint "
|
| 708 |
+
f"(missing 'model'/'config'; found {sorted(ck)[:6]})")
|
| 709 |
+
cfg = EncoderConfig.from_training_config(ck["config"])
|
| 710 |
+
|
| 711 |
+
encoder = JECTEncoder(cfg)
|
| 712 |
+
enc_state = _substate(ck["model"], f"{stream}_encoder")
|
| 713 |
+
if not enc_state:
|
| 714 |
+
raise ValueError(f"checkpoint has no '{stream}_encoder.*' weights")
|
| 715 |
+
missing, unexpected = encoder.load_state_dict(enc_state, strict=False)
|
| 716 |
+
# `relative_position_index` is a non-persistent buffer rebuilt in __init__.
|
| 717 |
+
missing = [k for k in missing if not k.endswith("relative_position_index")]
|
| 718 |
+
if missing or unexpected:
|
| 719 |
+
raise RuntimeError(f"encoder state mismatch: missing={missing[:5]} "
|
| 720 |
+
f"unexpected={unexpected[:5]}")
|
| 721 |
+
encoder.eval()
|
| 722 |
+
|
| 723 |
+
info = {"epoch": ck.get("epoch"), "step": ck.get("step"), "stream": stream,
|
| 724 |
+
"checkpoint": str(checkpoint_path), "training_config": ck["config"]}
|
| 725 |
+
if not with_decoder:
|
| 726 |
+
return encoder, cfg, info
|
| 727 |
+
|
| 728 |
+
decoder = DenseLatentDecoder(cfg)
|
| 729 |
+
dec_state = _substate(ck["model"], f"{stream}_decoder")
|
| 730 |
+
if not dec_state:
|
| 731 |
+
raise ValueError(f"checkpoint has no '{stream}_decoder.*' weights")
|
| 732 |
+
missing, unexpected = decoder.load_state_dict(dec_state, strict=True)
|
| 733 |
+
decoder.eval()
|
| 734 |
+
return encoder, decoder, cfg, info
|
model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:3d675ae5eb24f0c689f351de3df32f6d3d138cad2599267185cd0376f15f63fb
|
| 3 |
+
size 699660744
|
modeling_ject.py
ADDED
|
@@ -0,0 +1,269 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""JECT chest-CT encoder with a report-aligned readout.
|
| 2 |
+
|
| 3 |
+
The image encoder is frozen JECT pretraining. A latent-attention pooler reads its stage-4 tokens into 11 latents
|
| 4 |
+
(10 anatomy regions and one global latent); the global latent, projected and L2-normalised, is the image embedding.
|
| 5 |
+
Report text is embedded with chest2vec plus a LoRA adapter and a projection into the same space.
|
| 6 |
+
"""
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
import dataclasses
|
| 10 |
+
import json
|
| 11 |
+
import os
|
| 12 |
+
from typing import List, Optional, Sequence
|
| 13 |
+
|
| 14 |
+
import numpy as np
|
| 15 |
+
import torch
|
| 16 |
+
import torch.nn as nn
|
| 17 |
+
import torch.nn.functional as F
|
| 18 |
+
from safetensors.torch import load_file
|
| 19 |
+
|
| 20 |
+
from transformers import PreTrainedModel
|
| 21 |
+
|
| 22 |
+
try: # loaded as remote code by transformers (package-relative) or from a local folder on sys.path
|
| 23 |
+
from .configuration_ject import JECTConfig
|
| 24 |
+
from .ject_encoder import CT_TYPE_MAP, WINDOW_PARAMS, EncoderConfig, JECTEncoder
|
| 25 |
+
except ImportError:
|
| 26 |
+
from configuration_ject import JECTConfig
|
| 27 |
+
from ject_encoder import CT_TYPE_MAP, WINDOW_PARAMS, EncoderConfig, JECTEncoder
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def window_hu(volume: torch.Tensor, window_id: int) -> torch.Tensor:
|
| 31 |
+
"""Raw HU to [0, 1] with the given window."""
|
| 32 |
+
p = WINDOW_PARAMS[window_id]
|
| 33 |
+
low, high = p["center"] - p["width"] / 2.0, p["center"] + p["width"] / 2.0
|
| 34 |
+
return ((volume.float() - low) / (high - low)).clamp_(0.0, 1.0)
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def center_crop(volume: np.ndarray, size: int) -> np.ndarray:
|
| 38 |
+
"""Centre crop the last three axes of a cube to size^3."""
|
| 39 |
+
off = [(s - size) // 2 for s in volume.shape[-3:]]
|
| 40 |
+
if min(off) < 0:
|
| 41 |
+
raise ValueError(f"volume {volume.shape[-3:]} is smaller than {size}^3")
|
| 42 |
+
return volume[..., off[0]:off[0] + size, off[1]:off[1] + size, off[2]:off[2] + size]
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
class LatentAttentionPooling(nn.Module):
|
| 46 |
+
"""Learned queries cross-attend to the token grid; returns one vector per query."""
|
| 47 |
+
|
| 48 |
+
def __init__(self, dim: int, num_latents: int, num_heads: int, mlp_ratio: float = 4.0):
|
| 49 |
+
super().__init__()
|
| 50 |
+
self.latents = nn.Parameter(torch.zeros(1, num_latents, dim))
|
| 51 |
+
self.norm_latents = nn.LayerNorm(dim)
|
| 52 |
+
self.norm_tokens = nn.LayerNorm(dim)
|
| 53 |
+
self.attn = nn.MultiheadAttention(dim, num_heads, batch_first=True)
|
| 54 |
+
self.norm_mlp = nn.LayerNorm(dim)
|
| 55 |
+
hidden = int(dim * mlp_ratio)
|
| 56 |
+
self.mlp = nn.Sequential(nn.Linear(dim, hidden), nn.GELU(), nn.Linear(hidden, dim))
|
| 57 |
+
|
| 58 |
+
def forward(self, tokens: torch.Tensor) -> torch.Tensor:
|
| 59 |
+
q = self.norm_latents(self.latents).expand(tokens.shape[0], -1, -1)
|
| 60 |
+
kv = self.norm_tokens(tokens)
|
| 61 |
+
attended, _ = self.attn(q, kv, kv, need_weights=False)
|
| 62 |
+
attended = attended + q
|
| 63 |
+
return attended + self.mlp(self.norm_mlp(attended))
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
class JECTImageModel(nn.Module):
|
| 67 |
+
def __init__(self, config: dict):
|
| 68 |
+
super().__init__()
|
| 69 |
+
fields = {f.name: f for f in dataclasses.fields(EncoderConfig)}
|
| 70 |
+
enc = {k: tuple(v) if isinstance(v, list) else v for k, v in config["encoder"].items() if k in fields}
|
| 71 |
+
self.encoder = JECTEncoder(EncoderConfig(**enc))
|
| 72 |
+
r = config["readout"]
|
| 73 |
+
dim = self.encoder.pos_embed_s4.shape[-1]
|
| 74 |
+
self.pool = LatentAttentionPooling(dim, r["pool_latents"], r["pool_heads"])
|
| 75 |
+
self.proj = nn.Linear(dim, r["embed_dim"])
|
| 76 |
+
self.global_latent = r["global_latent"]
|
| 77 |
+
self.window_id = config["input"]["window_id"]
|
| 78 |
+
self.crop_size = config["input"]["crop_size"]
|
| 79 |
+
|
| 80 |
+
@torch.no_grad()
|
| 81 |
+
def forward(self, volume_hu: torch.Tensor, ct_type: Optional[Sequence[str]] = None) -> dict:
|
| 82 |
+
"""volume_hu: [B, 1, 224, 224, 224] raw HU; ct_type: optional per-scan 'non-contrast', 'contrast' or 'ldct'.
|
| 83 |
+
|
| 84 |
+
Returns the image embedding and intermediate features.
|
| 85 |
+
"""
|
| 86 |
+
e = self.encoder
|
| 87 |
+
if tuple(volume_hu.shape[-3:]) != (self.crop_size,) * 3:
|
| 88 |
+
raise ValueError(f"expected {self.crop_size}^3 input, got {tuple(volume_hu.shape[-3:])}")
|
| 89 |
+
x = window_hu(volume_hu, self.window_id)
|
| 90 |
+
wid = torch.full((x.shape[0],), self.window_id, dtype=torch.long, device=x.device)
|
| 91 |
+
cid = None
|
| 92 |
+
if ct_type is not None:
|
| 93 |
+
cid = torch.tensor([CT_TYPE_MAP[str(t).strip().lower()] for t in ct_type], dtype=torch.long, device=x.device)
|
| 94 |
+
x, _, _, _ = e.trunk(x, wid, cid)
|
| 95 |
+
x = x + e.pos_embed_s4
|
| 96 |
+
for blk in e.stage4:
|
| 97 |
+
x = blk(x, e.grid_s4)
|
| 98 |
+
latents = self.pool(x)
|
| 99 |
+
z = F.normalize(self.proj(latents).float(), dim=-1)
|
| 100 |
+
return {"embedding": z[:, self.global_latent], "latent_embeddings": z,
|
| 101 |
+
"pooled": latents[:, self.global_latent].float(), "s4_tokens": x, "s4_mean": x.mean(1).float()}
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
class JECTTextModel(nn.Module):
|
| 105 |
+
def __init__(self, config: dict, base_model: Optional[str] = None):
|
| 106 |
+
super().__init__()
|
| 107 |
+
from peft import LoraConfig, get_peft_model
|
| 108 |
+
from transformers import AutoModel, AutoTokenizer
|
| 109 |
+
|
| 110 |
+
t = config["text"]
|
| 111 |
+
name = base_model or t["base_model"]
|
| 112 |
+
base = AutoModel.from_pretrained(name, trust_remote_code=True, attn_implementation="eager")
|
| 113 |
+
self.tokenizer = AutoTokenizer.from_pretrained(name, padding_side="left", trust_remote_code=True)
|
| 114 |
+
if self.tokenizer.pad_token_id is None:
|
| 115 |
+
self.tokenizer.pad_token = self.tokenizer.eos_token
|
| 116 |
+
self.model = get_peft_model(base, LoraConfig(r=t["lora_r"], lora_alpha=t["lora_alpha"], lora_dropout=t.get("lora_dropout", 0.0),
|
| 117 |
+
target_modules=list(t["lora_targets"]), bias="none",
|
| 118 |
+
task_type="FEATURE_EXTRACTION"))
|
| 119 |
+
dim = config["readout"]["embed_dim"]
|
| 120 |
+
self.proj = nn.Linear(dim, dim)
|
| 121 |
+
self.max_length = t["max_length"]
|
| 122 |
+
self.pad_id = self.tokenizer.pad_token_id
|
| 123 |
+
eod = self.tokenizer.convert_tokens_to_ids("<|endoftext|>")
|
| 124 |
+
self.eod_id = eod if eod is not None and eod >= 0 else self.pad_id
|
| 125 |
+
|
| 126 |
+
def _ids(self, texts: Sequence[str]) -> List[List[int]]:
|
| 127 |
+
enc = self.tokenizer([str(t) for t in texts], add_special_tokens=False, truncation=True,
|
| 128 |
+
max_length=self.max_length - 1, padding=False, return_attention_mask=False)
|
| 129 |
+
return [x + [self.eod_id] for x in enc["input_ids"]]
|
| 130 |
+
|
| 131 |
+
@torch.no_grad()
|
| 132 |
+
def forward(self, texts: Sequence[str], batch_size: int = 16) -> torch.Tensor:
|
| 133 |
+
"""Texts to L2-normalised embeddings [N, D]. Texts are length-sorted and left-padded per batch."""
|
| 134 |
+
device = self.proj.weight.device
|
| 135 |
+
ids = self._ids(texts)
|
| 136 |
+
order = sorted(range(len(ids)), key=lambda i: len(ids[i]))
|
| 137 |
+
out: List[Optional[torch.Tensor]] = [None] * len(ids)
|
| 138 |
+
for start in range(0, len(order), batch_size):
|
| 139 |
+
idx = order[start:start + batch_size]
|
| 140 |
+
width = max(len(ids[i]) for i in idx)
|
| 141 |
+
input_ids = torch.tensor([[self.pad_id] * (width - len(ids[i])) + ids[i] for i in idx], device=device)
|
| 142 |
+
attn = torch.tensor([[0] * (width - len(ids[i])) + [1] * len(ids[i]) for i in idx], device=device)
|
| 143 |
+
pooled = self.model.base_model.model.encode(input_ids, attn, normalize=False).pooler_output
|
| 144 |
+
emb = F.normalize(self.proj(pooled.float()), dim=-1)
|
| 145 |
+
for j, i in enumerate(idx):
|
| 146 |
+
out[i] = emb[j]
|
| 147 |
+
return torch.stack(out)
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
class JECTModel(nn.Module):
|
| 151 |
+
"""Image and text towers with the SigLIP scale and bias used for scoring."""
|
| 152 |
+
|
| 153 |
+
def __init__(self, config: dict, with_text: bool = True, text_base_model: Optional[str] = None):
|
| 154 |
+
super().__init__()
|
| 155 |
+
self.config = config
|
| 156 |
+
self.image = JECTImageModel(config)
|
| 157 |
+
self.text = JECTTextModel(config, text_base_model) if with_text else None
|
| 158 |
+
self.logit_scale = float(np.exp(config["logit_scale_log"]))
|
| 159 |
+
self.logit_bias = float(config["logit_bias"])
|
| 160 |
+
|
| 161 |
+
@classmethod
|
| 162 |
+
def from_pretrained(cls, path: str, with_text: bool = True, text_base_model: Optional[str] = None,
|
| 163 |
+
device: str = "cpu") -> "JECTModel":
|
| 164 |
+
"""Load from a Hugging Face repo id or a local folder (config.json, model.safetensors, text_adapter.safetensors)."""
|
| 165 |
+
path = _local(path)
|
| 166 |
+
config = json.load(open(os.path.join(path, "config.json")))
|
| 167 |
+
model = cls(config, with_text, text_base_model)
|
| 168 |
+
_load_image_weights(model.image, os.path.join(path, "model.safetensors"))
|
| 169 |
+
if model.text is not None:
|
| 170 |
+
_load_text_adapter(model.text, os.path.join(path, "text_adapter.safetensors"))
|
| 171 |
+
return model.to(device).eval()
|
| 172 |
+
|
| 173 |
+
def similarity(self, image_embedding: torch.Tensor, text_embedding: torch.Tensor) -> torch.Tensor:
|
| 174 |
+
"""Pairwise SigLIP probabilities that each image matches each text."""
|
| 175 |
+
return torch.sigmoid(self.logit_scale * image_embedding @ text_embedding.T + self.logit_bias)
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
def _local(path: str) -> str:
|
| 179 |
+
if os.path.isdir(path):
|
| 180 |
+
return path
|
| 181 |
+
from huggingface_hub import snapshot_download
|
| 182 |
+
return snapshot_download(path)
|
| 183 |
+
|
| 184 |
+
|
| 185 |
+
def _relative_position_index(window) -> torch.Tensor:
|
| 186 |
+
wd, wh, ww = window
|
| 187 |
+
coords = torch.stack(torch.meshgrid(torch.arange(wd), torch.arange(wh), torch.arange(ww), indexing="ij")).reshape(3, -1)
|
| 188 |
+
rel = (coords[:, :, None] - coords[:, None, :]).permute(1, 2, 0).contiguous()
|
| 189 |
+
rel[..., 0] += wd - 1
|
| 190 |
+
rel[..., 1] += wh - 1
|
| 191 |
+
rel[..., 2] += ww - 1
|
| 192 |
+
rel[..., 0] *= (2 * wh - 1) * (2 * ww - 1)
|
| 193 |
+
rel[..., 1] *= 2 * ww - 1
|
| 194 |
+
return rel.sum(-1).view(-1)
|
| 195 |
+
|
| 196 |
+
|
| 197 |
+
def _rebuild_buffers(module: nn.Module) -> None:
|
| 198 |
+
"""Recompute the windowed-attention index buffers, which are not stored in the weights."""
|
| 199 |
+
for m in module.modules():
|
| 200 |
+
if "relative_position_index" in m._buffers and hasattr(m, "window"):
|
| 201 |
+
dev = m.relative_position_bias_table.device
|
| 202 |
+
m._buffers["relative_position_index"] = _relative_position_index(m.window).to(dev)
|
| 203 |
+
|
| 204 |
+
|
| 205 |
+
def _load_image_weights(image: nn.Module, file: str) -> None:
|
| 206 |
+
sd = {k[len("image."):]: v for k, v in load_file(file).items() if k.startswith("image.")}
|
| 207 |
+
missing, unexpected = image.load_state_dict(sd, strict=False)
|
| 208 |
+
missing = [k for k in missing if not k.endswith("relative_position_index")]
|
| 209 |
+
if missing or unexpected:
|
| 210 |
+
raise RuntimeError(f"image weights do not match: missing={missing[:5]} unexpected={unexpected[:5]}")
|
| 211 |
+
|
| 212 |
+
|
| 213 |
+
def _load_text_adapter(text: nn.Module, file: str) -> None:
|
| 214 |
+
sd = load_file(file)
|
| 215 |
+
own = dict(text.named_parameters())
|
| 216 |
+
bad = [k for k in sd if k not in own]
|
| 217 |
+
if bad:
|
| 218 |
+
raise RuntimeError(f"text adapter weights do not match: {bad[:5]}")
|
| 219 |
+
with torch.no_grad():
|
| 220 |
+
for k, v in sd.items():
|
| 221 |
+
own[k].copy_(v)
|
| 222 |
+
|
| 223 |
+
|
| 224 |
+
class JECTImageEncoderModel(PreTrainedModel):
|
| 225 |
+
"""The JECT image encoder and readout, loadable with ``AutoModel.from_pretrained(..., trust_remote_code=True)``."""
|
| 226 |
+
|
| 227 |
+
config_class = JECTConfig
|
| 228 |
+
base_model_prefix = "image"
|
| 229 |
+
main_input_name = "pixel_values"
|
| 230 |
+
|
| 231 |
+
def __init__(self, config: JECTConfig):
|
| 232 |
+
super().__init__(config)
|
| 233 |
+
self.image = JECTImageModel({"encoder": config.encoder, "readout": config.readout, "input": config.input})
|
| 234 |
+
self.post_init()
|
| 235 |
+
|
| 236 |
+
def _init_weights(self, module):
|
| 237 |
+
pass # all weights come from model.safetensors
|
| 238 |
+
|
| 239 |
+
@classmethod
|
| 240 |
+
def from_pretrained(cls, *args, **kwargs):
|
| 241 |
+
model = super().from_pretrained(*args, **kwargs)
|
| 242 |
+
_rebuild_buffers(model)
|
| 243 |
+
return model.eval()
|
| 244 |
+
|
| 245 |
+
def forward(self, pixel_values: torch.Tensor, ct_type: Optional[Sequence[str]] = None) -> dict:
|
| 246 |
+
"""pixel_values: [B, 1, S, S, S] raw HU (see prepare.load_volume). Returns the same dict as JECTImageModel."""
|
| 247 |
+
return self.image(pixel_values, ct_type)
|
| 248 |
+
|
| 249 |
+
def similarity(self, image_embedding: torch.Tensor, text_embedding: torch.Tensor) -> torch.Tensor:
|
| 250 |
+
scale = float(np.exp(self.config.logit_scale_log))
|
| 251 |
+
return torch.sigmoid(scale * image_embedding @ text_embedding.T + self.config.logit_bias)
|
| 252 |
+
|
| 253 |
+
|
| 254 |
+
def load_text_model(repo: str, device: str = "cpu", text_base_model: Optional[str] = None) -> "JECTTextModel":
|
| 255 |
+
"""chest2vec 0.6B with this model's LoRA adapter and projection, for report and prompt embeddings."""
|
| 256 |
+
path = _local(repo)
|
| 257 |
+
config = json.load(open(os.path.join(path, "config.json")))
|
| 258 |
+
text = JECTTextModel(config, text_base_model)
|
| 259 |
+
_load_text_adapter(text, os.path.join(path, "text_adapter.safetensors"))
|
| 260 |
+
return text.to(device).eval()
|
| 261 |
+
|
| 262 |
+
|
| 263 |
+
def load_cube(path: str, crop_size: int = 224) -> torch.Tensor:
|
| 264 |
+
"""Read a preprocessed npz cube (key ``ct``, raw HU) and return a [1, 1, 224, 224, 224] tensor."""
|
| 265 |
+
with np.load(path) as d:
|
| 266 |
+
v = d["ct"]
|
| 267 |
+
if v.ndim == 4:
|
| 268 |
+
v = v[0]
|
| 269 |
+
return torch.from_numpy(np.ascontiguousarray(center_crop(v, crop_size))).unsqueeze(0).unsqueeze(0)
|
prepare.py
ADDED
|
@@ -0,0 +1,160 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Turn a chest CT into model input.
|
| 2 |
+
|
| 3 |
+
Modes:
|
| 4 |
+
npz a cube already preprocessed for this model (key ``ct``); centre-cropped to the input size
|
| 5 |
+
totalseg NIfTI; lungs segmented with TotalSegmentator, then the exact preprocessing used in training
|
| 6 |
+
fast NIfTI; same preprocessing, but the lungs are found by HU threshold instead of TotalSegmentator
|
| 7 |
+
center NIfTI; same preprocessing, but the cube is centred on the middle of the volume
|
| 8 |
+
|
| 9 |
+
NIfTI preprocessing: reorient to LPS, resample to the model's spacing (trilinear), pad with air, crop a cube
|
| 10 |
+
around the lungs, store depth-first as raw HU int16, then centre-crop to the model's input size.
|
| 11 |
+
``totalseg`` needs the ``TotalSegmentator`` package and a GPU for reasonable speed; the other modes need
|
| 12 |
+
``monai`` and ``nibabel`` only.
|
| 13 |
+
"""
|
| 14 |
+
from __future__ import annotations
|
| 15 |
+
|
| 16 |
+
import os
|
| 17 |
+
import tempfile
|
| 18 |
+
from typing import Optional, Sequence, Tuple
|
| 19 |
+
|
| 20 |
+
import numpy as np
|
| 21 |
+
import torch
|
| 22 |
+
|
| 23 |
+
AIR_HU = -1024.0
|
| 24 |
+
LUNG_LABELS = (10, 11, 12, 13, 14) # TotalSegmentator lung lobes
|
| 25 |
+
MODES = ("npz", "totalseg", "fast", "center")
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def crop_starts(shape: Sequence[int], roi: Sequence[int], lo: Sequence[int], hi: Sequence[int]) -> Tuple[int, ...]:
|
| 29 |
+
"""Start per axis of a crop of size ``roi`` around the box ``lo``..``hi``, kept inside the volume."""
|
| 30 |
+
starts = []
|
| 31 |
+
for size, r, a, b in zip(shape, roi, lo, hi):
|
| 32 |
+
if size <= r:
|
| 33 |
+
starts.append(0)
|
| 34 |
+
continue
|
| 35 |
+
a, b = int(a), int(b)
|
| 36 |
+
extent = b - a + 1
|
| 37 |
+
start = (a + b) // 2 - r // 2 if extent >= r else a - (r - extent) // 2
|
| 38 |
+
starts.append(int(np.clip(start, 0, size - r)))
|
| 39 |
+
return tuple(starts)
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def hu_lung_box(vol: np.ndarray) -> Optional[Tuple[np.ndarray, np.ndarray]]:
|
| 43 |
+
"""Bounding box of the lungs found by HU threshold, or None.
|
| 44 |
+
|
| 45 |
+
The body is the largest connected region above -500 HU (this drops the table), filled slice by slice
|
| 46 |
+
along the last (I-S) axis; the lungs are air below -400 HU strictly inside it, keeping the two
|
| 47 |
+
largest connected regions.
|
| 48 |
+
"""
|
| 49 |
+
from scipy import ndimage
|
| 50 |
+
lab, n = ndimage.label(vol > -500)
|
| 51 |
+
if n == 0:
|
| 52 |
+
return None
|
| 53 |
+
sizes = ndimage.sum_labels(np.ones_like(lab, dtype=np.int8), lab, index=np.arange(1, n + 1))
|
| 54 |
+
body = lab == (int(np.argmax(sizes)) + 1)
|
| 55 |
+
filled = np.zeros_like(body)
|
| 56 |
+
for z in range(body.shape[2]):
|
| 57 |
+
filled[:, :, z] = ndimage.binary_fill_holes(body[:, :, z])
|
| 58 |
+
air = filled & ~body & (vol < -400)
|
| 59 |
+
lab, n = ndimage.label(air)
|
| 60 |
+
if n == 0:
|
| 61 |
+
return None
|
| 62 |
+
sizes = ndimage.sum_labels(np.ones_like(lab, dtype=np.int8), lab, index=np.arange(1, n + 1))
|
| 63 |
+
keep = np.argsort(sizes)[::-1][:2] + 1
|
| 64 |
+
lungs = np.isin(lab, keep)
|
| 65 |
+
if lungs.sum() < 10000:
|
| 66 |
+
return None
|
| 67 |
+
coords = np.argwhere(lungs)
|
| 68 |
+
return coords.min(0), coords.max(0)
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def run_totalsegmentator(nifti_path: str, out_path: str, device: str = "gpu") -> str:
|
| 72 |
+
"""Segment the scan with TotalSegmentator (multi-label output) and return the mask path."""
|
| 73 |
+
from totalsegmentator.python_api import totalsegmentator
|
| 74 |
+
totalsegmentator(nifti_path, out_path, ml=True, device=device, quiet=True)
|
| 75 |
+
return out_path
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
def preprocess_nifti(nifti_path: str, spacing_mm: float, cube_size: int, mode: str = "fast",
|
| 79 |
+
totalseg_mask: Optional[str] = None, device: str = "gpu", return_volume: bool = False):
|
| 80 |
+
"""NIfTI to a raw-HU int16 cube [cube, cube, cube] in the model's layout (LPS, depth-first).
|
| 81 |
+
|
| 82 |
+
With return_volume, also returns the resampled volume (depth-first) and the crop start per axis.
|
| 83 |
+
"""
|
| 84 |
+
from monai.transforms import (Compose, EnsureChannelFirstd, LoadImaged, Orientationd, Spacingd,
|
| 85 |
+
SpatialPadd)
|
| 86 |
+
if mode not in ("totalseg", "fast", "center"):
|
| 87 |
+
raise ValueError(f"mode must be totalseg, fast or center for NIfTI input, got {mode!r}")
|
| 88 |
+
tmp = None
|
| 89 |
+
if mode == "totalseg" and totalseg_mask is None:
|
| 90 |
+
tmp = tempfile.mkdtemp()
|
| 91 |
+
totalseg_mask = run_totalsegmentator(nifti_path, os.path.join(tmp, "seg.nii.gz"), device)
|
| 92 |
+
keys = ("ct", "seg") if mode == "totalseg" else ("ct",)
|
| 93 |
+
roi = (cube_size,) * 3
|
| 94 |
+
steps = [LoadImaged(keys=keys, reader="NibabelReader"), EnsureChannelFirstd(keys=keys),
|
| 95 |
+
Orientationd(keys=keys, axcodes="LPS", labels=(("L", "R"), ("P", "A"), ("I", "S"))),
|
| 96 |
+
Spacingd(keys=("ct",), pixdim=(spacing_mm,) * 3, mode="trilinear", align_corners=True)]
|
| 97 |
+
if mode == "totalseg":
|
| 98 |
+
steps.append(Spacingd(keys=("seg",), pixdim=(spacing_mm,) * 3, mode="nearest", align_corners=True))
|
| 99 |
+
steps.append(SpatialPadd(keys=("ct",), spatial_size=roi, constant_values=AIR_HU))
|
| 100 |
+
if mode == "totalseg":
|
| 101 |
+
steps.append(SpatialPadd(keys=("seg",), spatial_size=roi, constant_values=0.0))
|
| 102 |
+
paths = {"ct": nifti_path, **({"seg": totalseg_mask} if mode == "totalseg" else {})}
|
| 103 |
+
d = Compose(steps)(paths)
|
| 104 |
+
ct = np.asarray(d["ct"])[0]
|
| 105 |
+
if mode == "totalseg":
|
| 106 |
+
lungs = np.isin(np.asarray(d["seg"])[0], LUNG_LABELS)
|
| 107 |
+
if not lungs.any():
|
| 108 |
+
raise ValueError(f"{nifti_path}: TotalSegmentator found no lungs")
|
| 109 |
+
coords = np.argwhere(lungs)
|
| 110 |
+
starts = crop_starts(ct.shape, roi, coords.min(0), coords.max(0))
|
| 111 |
+
elif mode == "fast":
|
| 112 |
+
box = hu_lung_box(ct)
|
| 113 |
+
if box is None:
|
| 114 |
+
c = tuple(x // 2 for x in ct.shape)
|
| 115 |
+
box = (c, c)
|
| 116 |
+
starts = crop_starts(ct.shape, roi, box[0], box[1])
|
| 117 |
+
else:
|
| 118 |
+
c = tuple(s // 2 for s in ct.shape)
|
| 119 |
+
starts = crop_starts(ct.shape, roi, c, c)
|
| 120 |
+
full = ct
|
| 121 |
+
ct = ct[tuple(slice(s, s + r) for s, r in zip(starts, roi))]
|
| 122 |
+
ct = np.clip(np.round(ct), np.iinfo(np.int16).min, np.iinfo(np.int16).max)
|
| 123 |
+
if tmp is not None:
|
| 124 |
+
for f in os.listdir(tmp):
|
| 125 |
+
os.remove(os.path.join(tmp, f))
|
| 126 |
+
os.rmdir(tmp)
|
| 127 |
+
cube = np.ascontiguousarray(np.transpose(ct, (2, 0, 1))).astype(np.int16)
|
| 128 |
+
if return_volume:
|
| 129 |
+
return cube, np.transpose(full, (2, 0, 1)), (starts[2], starts[0], starts[1])
|
| 130 |
+
return cube
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
def center_crop(volume: np.ndarray, size: int) -> np.ndarray:
|
| 134 |
+
off = [(s - size) // 2 for s in volume.shape[-3:]]
|
| 135 |
+
if min(off) < 0:
|
| 136 |
+
raise ValueError(f"volume {volume.shape[-3:]} is smaller than {size}^3")
|
| 137 |
+
return volume[..., off[0]:off[0] + size, off[1]:off[1] + size, off[2]:off[2] + size]
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
def load_volume(path: str, config: dict, mode: Optional[str] = None, totalseg_mask: Optional[str] = None,
|
| 141 |
+
device: str = "gpu") -> torch.Tensor:
|
| 142 |
+
"""Read an npz cube or a NIfTI scan and return a [1, 1, S, S, S] raw-HU tensor for the model.
|
| 143 |
+
|
| 144 |
+
``config`` is the model config (a dict or the loaded config object; its ``input`` block sets spacing, cube and
|
| 145 |
+
crop size). ``mode``
|
| 146 |
+
defaults to ``npz`` for .npz files and ``fast`` for NIfTI.
|
| 147 |
+
"""
|
| 148 |
+
inp = config["input"] if isinstance(config, dict) else config.input
|
| 149 |
+
if mode is None:
|
| 150 |
+
mode = "npz" if path.endswith(".npz") else "fast"
|
| 151 |
+
if mode not in MODES:
|
| 152 |
+
raise ValueError(f"mode must be one of {MODES}, got {mode!r}")
|
| 153 |
+
if mode == "npz":
|
| 154 |
+
with np.load(path) as d:
|
| 155 |
+
cube = d["ct"]
|
| 156 |
+
cube = cube[0] if cube.ndim == 4 else cube
|
| 157 |
+
else:
|
| 158 |
+
cube = preprocess_nifti(path, inp["spacing_mm"], inp["cube_size"], mode, totalseg_mask, device)
|
| 159 |
+
cube = center_crop(cube, inp["crop_size"])
|
| 160 |
+
return torch.from_numpy(np.ascontiguousarray(cube)).unsqueeze(0).unsqueeze(0)
|
preprocess.py
ADDED
|
@@ -0,0 +1,155 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Input preprocessing for the JECT encoder.
|
| 2 |
+
|
| 3 |
+
Input contract used during pretraining (features degrade without warning otherwise):
|
| 4 |
+
|
| 5 |
+
1. Orientation LPS, axes ordered (D, H, W) = (R-L, A-P, I-S), as produced by MONAI's
|
| 6 |
+
``Orientationd(axcodes="LPS")``.
|
| 7 |
+
2. Isotropic resampling, then a lung-centred crop/pad to a fixed cube of raw HU
|
| 8 |
+
(int16, padded with -1024 HU):
|
| 9 |
+
* 1.5 mm spacing, 256^3 cube: input for the 224 checkpoint
|
| 10 |
+
* 1.0 mm spacing, 384^3 cube: input for the 336 checkpoint
|
| 11 |
+
Both give a ~336 mm physical field of view at the model input.
|
| 12 |
+
3. Centre crop / pad (with -1000 HU) from that cube to the model's input size.
|
| 13 |
+
4. HU window, then linear scale to [0, 1]. The default is the general window
|
| 14 |
+
(id 2, centre 0, width 2000, i.e. [-1000, 1000]). Pass the same id to the encoder
|
| 15 |
+
as ``window_id``; it selects an additive token embedding.
|
| 16 |
+
|
| 17 |
+
``preprocess_hu_volume`` performs steps 3-4 on an already-resampled cube and is the
|
| 18 |
+
usual entry point. ``resample_to_spacing`` covers the resampling in step 2 for raw
|
| 19 |
+
NIfTI input; it is a plain trilinear resize without the lung-bounding-box crop, so
|
| 20 |
+
features differ slightly from those of the original pipeline.
|
| 21 |
+
"""
|
| 22 |
+
|
| 23 |
+
from __future__ import annotations
|
| 24 |
+
|
| 25 |
+
import os
|
| 26 |
+
from typing import Optional, Sequence, Tuple
|
| 27 |
+
|
| 28 |
+
import numpy as np
|
| 29 |
+
import torch
|
| 30 |
+
|
| 31 |
+
try: # importable as a package module or as a plain script in this directory
|
| 32 |
+
from .ject_encoder import CT_TYPE_MAP, DEFAULT_CT_TYPE, GENERAL_WINDOW_ID, WINDOW_PARAMS
|
| 33 |
+
except ImportError:
|
| 34 |
+
from ject_encoder import CT_TYPE_MAP, DEFAULT_CT_TYPE, GENERAL_WINDOW_ID, WINDOW_PARAMS
|
| 35 |
+
|
| 36 |
+
Triple = Tuple[int, int, int]
|
| 37 |
+
|
| 38 |
+
AIR_HU = -1000.0 # pad value for the centre crop/pad
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
# ---------------------------------------------------------------------------
|
| 42 |
+
# Windowing
|
| 43 |
+
# ---------------------------------------------------------------------------
|
| 44 |
+
def apply_window(volume_hu: np.ndarray, window_id: int = GENERAL_WINDOW_ID) -> np.ndarray:
|
| 45 |
+
"""Clamp raw HU to a window and rescale to [0, 1]."""
|
| 46 |
+
if window_id not in WINDOW_PARAMS:
|
| 47 |
+
raise ValueError(f"window_id must be one of {sorted(WINDOW_PARAMS)}, got {window_id}")
|
| 48 |
+
p = WINDOW_PARAMS[window_id]
|
| 49 |
+
low = p["center"] - p["width"] / 2.0
|
| 50 |
+
high = p["center"] + p["width"] / 2.0
|
| 51 |
+
out = (volume_hu.astype(np.float32) - low) / (high - low)
|
| 52 |
+
return np.clip(out, 0.0, 1.0)
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def ct_type_to_id(ct_type) -> int:
|
| 56 |
+
"""Map a ct_type string ('non-contrast' | 'contrast' | 'ldct') to its embedding index."""
|
| 57 |
+
if ct_type is None:
|
| 58 |
+
return DEFAULT_CT_TYPE
|
| 59 |
+
if isinstance(ct_type, (int, np.integer)):
|
| 60 |
+
return int(ct_type)
|
| 61 |
+
return CT_TYPE_MAP.get(str(ct_type).strip().lower(), DEFAULT_CT_TYPE)
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
# ---------------------------------------------------------------------------
|
| 65 |
+
# Crop / pad
|
| 66 |
+
# ---------------------------------------------------------------------------
|
| 67 |
+
def make_crop_pad_transform(src_shape: Triple, dst_shape: Triple, random_crop: bool = False):
|
| 68 |
+
"""Slices + pads that take ``src_shape`` to exactly ``dst_shape`` (centre crop by default)."""
|
| 69 |
+
slices, pads = [], []
|
| 70 |
+
for src, dst in zip(src_shape, dst_shape):
|
| 71 |
+
if src >= dst:
|
| 72 |
+
start = (src - dst) // 2
|
| 73 |
+
if random_crop and src > dst:
|
| 74 |
+
start = int(np.random.randint(0, src - dst + 1))
|
| 75 |
+
slices.append(slice(start, start + dst))
|
| 76 |
+
pads.append((0, 0))
|
| 77 |
+
else:
|
| 78 |
+
slices.append(slice(0, src))
|
| 79 |
+
before = (dst - src) // 2
|
| 80 |
+
pads.append((before, dst - src - before))
|
| 81 |
+
return tuple(slices), tuple(pads)
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
def apply_crop_pad_3d(arr: np.ndarray, slices, pads, constant: float = AIR_HU) -> np.ndarray:
|
| 85 |
+
"""Apply crop/pad to the last three dims of ``arr``."""
|
| 86 |
+
cropped = arr[(...,) + tuple(slices)]
|
| 87 |
+
if any(a > 0 or b > 0 for a, b in pads):
|
| 88 |
+
pad_width = [(0, 0)] * (cropped.ndim - 3) + list(pads)
|
| 89 |
+
cropped = np.pad(cropped, pad_width=pad_width, mode="constant", constant_values=constant)
|
| 90 |
+
return cropped
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def center_crop_pad(volume_hu: np.ndarray, target: Triple) -> np.ndarray:
|
| 94 |
+
"""Centre crop/pad a raw-HU volume (last three dims) to ``target``, padding with air."""
|
| 95 |
+
slices, pads = make_crop_pad_transform(tuple(volume_hu.shape[-3:]), tuple(target))
|
| 96 |
+
return apply_crop_pad_3d(volume_hu, slices, pads, constant=AIR_HU)
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
# ---------------------------------------------------------------------------
|
| 100 |
+
# Entry points
|
| 101 |
+
# ---------------------------------------------------------------------------
|
| 102 |
+
def preprocess_hu_volume(volume_hu: np.ndarray, input_size: Triple,
|
| 103 |
+
window_id: int = GENERAL_WINDOW_ID) -> torch.Tensor:
|
| 104 |
+
"""Convert a raw-HU cube to a model input tensor ``[1, D, H, W]`` in [0, 1].
|
| 105 |
+
|
| 106 |
+
Crop/pad happens in HU space (padding with air) before windowing, so padded
|
| 107 |
+
borders map to 0.0 in the general window.
|
| 108 |
+
"""
|
| 109 |
+
vol = np.asarray(volume_hu)
|
| 110 |
+
if vol.ndim == 4:
|
| 111 |
+
if vol.shape[0] != 1:
|
| 112 |
+
raise ValueError(f"expected a single-channel volume, got shape {vol.shape}")
|
| 113 |
+
vol = vol[0]
|
| 114 |
+
if vol.ndim != 3:
|
| 115 |
+
raise ValueError(f"expected a 3D HU volume, got shape {vol.shape}")
|
| 116 |
+
vol = center_crop_pad(vol, tuple(input_size))
|
| 117 |
+
return torch.from_numpy(apply_window(vol, window_id)).unsqueeze(0)
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def load_npz_volume(path: str, input_size: Triple, key: str = "ct",
|
| 121 |
+
window_id: int = GENERAL_WINDOW_ID) -> torch.Tensor:
|
| 122 |
+
"""Load a preprocessed ``.npz`` (raw int16 HU under ``key``) into a model input tensor."""
|
| 123 |
+
if not os.path.exists(path) and os.path.exists(path + ".npz"):
|
| 124 |
+
path = path + ".npz"
|
| 125 |
+
with np.load(path) as d:
|
| 126 |
+
if key not in d:
|
| 127 |
+
raise KeyError(f"{path} has no '{key}' array (found {list(d.files)})")
|
| 128 |
+
ct = d[key]
|
| 129 |
+
return preprocess_hu_volume(ct, input_size, window_id=window_id)
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
def resample_to_spacing(volume_hu: np.ndarray, src_spacing: Sequence[float],
|
| 133 |
+
dst_spacing: float = 1.5) -> np.ndarray:
|
| 134 |
+
"""Trilinear-resample a raw-HU volume to isotropic ``dst_spacing`` mm.
|
| 135 |
+
|
| 136 |
+
Convenience for raw NIfTI input. It does not apply the lung-bounding-box crop
|
| 137 |
+
used in pretraining; see the module docstring.
|
| 138 |
+
"""
|
| 139 |
+
vol = np.asarray(volume_hu, dtype=np.float32)
|
| 140 |
+
if vol.ndim != 3:
|
| 141 |
+
raise ValueError(f"expected a 3D HU volume, got shape {vol.shape}")
|
| 142 |
+
if len(src_spacing) != 3:
|
| 143 |
+
raise ValueError(f"src_spacing must have 3 entries, got {src_spacing!r}")
|
| 144 |
+
scale = [float(s) / float(dst_spacing) for s in src_spacing]
|
| 145 |
+
out_shape = tuple(max(1, int(round(n * s))) for n, s in zip(vol.shape, scale))
|
| 146 |
+
t = torch.from_numpy(vol)[None, None]
|
| 147 |
+
t = torch.nn.functional.interpolate(t, size=out_shape, mode="trilinear", align_corners=True)
|
| 148 |
+
return t[0, 0].numpy()
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
def stack_batch(volumes: Sequence[torch.Tensor]) -> torch.Tensor:
|
| 152 |
+
"""Stack [1, D, H, W] tensors into a [B, 1, D, H, W] batch."""
|
| 153 |
+
if not volumes:
|
| 154 |
+
raise ValueError("stack_batch got an empty list")
|
| 155 |
+
return torch.stack([v if v.ndim == 4 else v.unsqueeze(0) for v in volumes], dim=0)
|
requirements.txt
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
torch>=2.4
|
| 2 |
+
numpy>=1.24
|
| 3 |
+
safetensors>=0.4
|
| 4 |
+
huggingface_hub>=0.30
|
| 5 |
+
transformers>=4.57
|
| 6 |
+
peft>=0.18
|
| 7 |
+
# NIfTI input (prepare.py)
|
| 8 |
+
monai>=1.5
|
| 9 |
+
nibabel>=5.0
|
| 10 |
+
scipy>=1.10
|
| 11 |
+
# optional: prepare.py mode "totalseg"
|
| 12 |
+
# TotalSegmentator>=2.0
|
text_adapter.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e0069b975fbdd32278e2aa6dab00e3daed5db45786ec869bf0a8faf15814392c
|
| 3 |
+
size 44626192
|