Upload MIMIC test evaluation results

README.md

@@ -34,7 +34,7 @@ metrics:
 - Project status: `Training in progress`
 - Release status: `Research preview checkpoint`
 - Current checkpoint status: `Not final`
-- Training completion toward planned run: `35.77%` (`1.073` / `3` epochs)
+- Training completion toward planned run: `39.96%` (`1.199` / `3` epochs)
 - Current published metrics are intermediate and will change as training continues.
 
 ## Overview
@@ -43,6 +43,10 @@ LAnA is a medical report-generation project for chest X-ray images. The complete
 
 The architecture combines a DINOv3 vision encoder, lung and heart segmentation heads, and a GPT-2 decoder modified so each transformer layer receives a different anatomical attention bias derived from the segmentation mask.
 
+## How to Run
+
+For local inference instructions, go to the [Inference](#inference) section.
+
 ## Intended Use
 
 - Input: a chest X-ray image resized to `512x512` and normalized with ImageNet mean/std.
@@ -59,7 +63,7 @@ The architecture combines a DINOv3 vision encoder, lung and heart segmentation h
 ## Evaluation
 
 - Text-generation metrics used in this project include BLEU, METEOR, ROUGE, and CIDEr.
-- Medical report metrics implemented in the repository include RadGraph F1 and CheXpert F1.
+- Medical report metrics implemented in the repository include RadGraph F1 and CheXpert F1 (`14-micro`, `5-micro`, `14-macro`, `5-macro`).
 
 ## Training Snapshot
 
@@ -75,78 +79,95 @@ The architecture combines a DINOv3 vision encoder, lung and heart segmentation h
 - Scheduler: `cosine`
 - Warmup steps: `5114`
 - Weight decay: `0.01`
-- Steps completed: `36576`
+- Steps completed: `40864`
 - Planned total steps: `102276`
-- Images seen: `292640`
-- Total training time: `8.6892` hours
+- Images seen: `326946`
+- Total training time: `9.6893` hours
 - Hardware: `NVIDIA GeForce RTX 5070`
-- Final train loss: `1.4422`
-- Validation loss: `1.4824`
+- Final train loss: `2.2784`
+- Validation loss: `1.4888`
 
 ## MIMIC Test Results
 
 Frontal-only evaluation using `PA/AP` studies only.
 
+### Current Checkpoint Results
+
+| Metric | Value |
+| --- | --- |
+| Number of studies | `3041` |
+| RadGraph F1 | `0.0964` |
+| RadGraph entity F1 | `0.1603` |
+| RadGraph relation F1 | `0.1412` |
+| CheXpert F1 micro | `0.1898` |
+| CheXpert F1 macro | `0.1006` |
+
+### Final Completed Training Results
+
+The final table will be populated when the planned training run is completed. Until then, final-report metrics remain `TBD`.
+
 | Metric | Value |
 | --- | --- |
 | Number of studies | TBD |
 | RadGraph F1 | TBD |
+| RadGraph entity F1 | TBD |
+| RadGraph relation F1 | TBD |
 | CheXpert F1 micro | TBD |
 | CheXpert F1 macro | TBD |
 
-## Inference
-
-Standard `AutoModel.from_pretrained(..., trust_remote_code=True)` loading is currently blocked for this repo because the custom model constructor performs nested pretrained submodel loads.
-Use the verified manual load path below instead: download the HF repo snapshot, import the downloaded package, and load the exported `model.safetensors` directly.
-
-```python
-from pathlib import Path
-import sys
-
-import numpy as np
-import torch
-from PIL import Image
-from huggingface_hub import snapshot_download
-from safetensors.torch import load_file
-from transformers import AutoTokenizer
-
-repo_dir = Path(snapshot_download("manu02/LAnA"))
-sys.path.insert(0, str(repo_dir))
-
-from lana_radgen import LanaConfig, LanaForConditionalGeneration
-
-config = LanaConfig.from_pretrained(repo_dir)
-config.lung_segmenter_checkpoint = str(repo_dir / "segmenters" / "lung_segmenter_dinounet_finetuned.pth")
-config.heart_segmenter_checkpoint = str(repo_dir / "segmenters" / "heart_segmenter_dinounet_best.pth")
-
-device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-
-model = LanaForConditionalGeneration(config)
-state_dict = load_file(str(repo_dir / "model.safetensors"))
-missing, unexpected = model.load_state_dict(state_dict, strict=True)
-assert not missing and not unexpected
-
-model.tokenizer = AutoTokenizer.from_pretrained(repo_dir, trust_remote_code=True)
-model.move_non_quantized_modules(device)
-model.eval()
-
-image_path = Path("example.png")
-image = Image.open(image_path).convert("RGB")
-image = image.resize((512, 512), resample=Image.BICUBIC)
+## Inference
+
+Standard `AutoModel.from_pretrained(..., trust_remote_code=True)` loading is currently blocked for this repo because the custom model constructor performs nested pretrained submodel loads.
+Use the verified manual load path below instead: download the HF repo snapshot, import the downloaded package, and load the exported `model.safetensors` directly.
+
+```python
+from pathlib import Path
+import sys
+
+import numpy as np
+import torch
+from PIL import Image
+from huggingface_hub import snapshot_download
+from safetensors.torch import load_file
+from transformers import AutoTokenizer
+
+repo_dir = Path(snapshot_download("manu02/LAnA"))
+sys.path.insert(0, str(repo_dir))
+
+from lana_radgen import LanaConfig, LanaForConditionalGeneration
+
+config = LanaConfig.from_pretrained(repo_dir)
+config.lung_segmenter_checkpoint = str(repo_dir / "segmenters" / "lung_segmenter_dinounet_finetuned.pth")
+config.heart_segmenter_checkpoint = str(repo_dir / "segmenters" / "heart_segmenter_dinounet_best.pth")
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+model = LanaForConditionalGeneration(config)
+state_dict = load_file(str(repo_dir / "model.safetensors"))
+missing, unexpected = model.load_state_dict(state_dict, strict=True)
+assert not missing and not unexpected
+
+model.tokenizer = AutoTokenizer.from_pretrained(repo_dir, trust_remote_code=True)
+model.move_non_quantized_modules(device)
+model.eval()
+
+image_path = Path("example.png")
+image = Image.open(image_path).convert("RGB")
+image = image.resize((512, 512), resample=Image.BICUBIC)
 array = np.asarray(image, dtype=np.float32) / 255.0
 pixel_values = torch.from_numpy(array).permute(2, 0, 1)
 mean = torch.tensor([0.485, 0.456, 0.406]).view(3, 1, 1)
 std = torch.tensor([0.229, 0.224, 0.225]).view(3, 1, 1)
 pixel_values = ((pixel_values - mean) / std).unsqueeze(0).to(device)
 
-with torch.no_grad():
-    generated = model.generate(pixel_values=pixel_values, max_new_tokens=128)
-
-report = model.tokenizer.batch_decode(generated, skip_special_tokens=True)[0]
-print(report)
-```
-
-## Notes
+with torch.no_grad():
+    generated = model.generate(pixel_values=pixel_values, max_new_tokens=128)
+
+report = model.tokenizer.batch_decode(generated, skip_special_tokens=True)[0]
+print(report)
+```
+
+## Notes
 
 - `segmenters/` contains the lung and heart segmentation checkpoints used to build anatomical attention masks.
 - `evaluations/mimic_test_metrics.json` contains the latest saved MIMIC test metrics.
@@ -157,26 +178,14 @@ print(report)
 - Dataset: `MIMIC-CXR test`
 - View filter: `frontal-only (PA/AP)`
 - Number of examples: `3041`
-- CheXpert F1 micro: `0.1610`
-- CheXpert F1 macro: `0.1124`
-- RadGraph F1: `0.0956`
-- RadGraph entity F1: `0.1582`
-- RadGraph relation F1: `0.1381`
+- CheXpert F1 micro: `0.1898`
+- CheXpert F1 macro: `0.1006`
+- RadGraph F1: `0.0964`
+- RadGraph entity F1: `0.1603`
+- RadGraph relation F1: `0.1412`
 - RadGraph available: `True`
 - RadGraph error: `None`
 
 - Evaluation file: `evaluations/mimic_test_metrics.json`
 - Predictions file: `evaluations/mimic_test_predictions.csv`
 <!-- EVAL_RESULTS_END -->
-
-<!-- MIMIC_TEST_RESULTS_START -->
-## MIMIC Test Results
-
-Frontal-only evaluation using `PA/AP` studies only. Number of evaluated studies: `3041`.
-
-| Metric | Value |
-| --- | --- |
-| RadGraph F1 | `0.0956` |
-| CheXpert F1 micro | `0.1610` |
-| CheXpert F1 macro | `0.1124` |
-<!-- MIMIC_TEST_RESULTS_END -->

benchmark_results.json

@@ -1,3 +1,391 @@
 {
-  "results": []
+  "results": [
+    {
+      "method": "qlora_paged_adamw8bit",
+      "local_batch_size": 1,
+      "global_batch_size_requested": 1,
+      "status": "failed",
+      "error": "element 0 of tensors does not require grad and does not have a grad_fn"
+    },
+    {
+      "method": "qlora_paged_adamw8bit",
+      "local_batch_size": 1,
+      "global_batch_size_requested": 8,
+      "status": "failed",
+      "error": "element 0 of tensors does not require grad and does not have a grad_fn"
+    },
+    {
+      "method": "qlora_paged_adamw8bit",
+      "local_batch_size": 1,
+      "global_batch_size_requested": 16,
+      "status": "failed",
+      "error": "element 0 of tensors does not require grad and does not have a grad_fn"
+    },
+    {
+      "method": "qlora_paged_adamw8bit",
+      "local_batch_size": 2,
+      "global_batch_size_requested": 2,
+      "status": "failed",
+      "error": "element 0 of tensors does not require grad and does not have a grad_fn"
+    },
+    {
+      "method": "qlora_paged_adamw8bit",
+      "local_batch_size": 2,
+      "global_batch_size_requested": 8,
+      "status": "failed",
+      "error": "element 0 of tensors does not require grad and does not have a grad_fn"
+    },
+    {
+      "method": "qlora_paged_adamw8bit",
+      "local_batch_size": 2,
+      "global_batch_size_requested": 16,
+      "status": "failed",
+      "error": "element 0 of tensors does not require grad and does not have a grad_fn"
+    },
+    {
+      "method": "qlora_paged_adamw8bit",
+      "local_batch_size": 4,
+      "global_batch_size_requested": 4,
+      "status": "failed",
+      "error": "element 0 of tensors does not require grad and does not have a grad_fn"
+    },
+    {
+      "method": "qlora_paged_adamw8bit",
+      "local_batch_size": 4,
+      "global_batch_size_requested": 8,
+      "status": "failed",
+      "error": "element 0 of tensors does not require grad and does not have a grad_fn"
+    },
+    {
+      "method": "qlora_paged_adamw8bit",
+      "local_batch_size": 4,
+      "global_batch_size_requested": 16,
+      "status": "failed",
+      "error": "element 0 of tensors does not require grad and does not have a grad_fn"
+    },
+    {
+      "method": "lora_adamw",
+      "local_batch_size": 1,
+      "global_batch_size_requested": 1,
+      "status": "ok",
+      "effective_global_batch_size": 1,
+      "gradient_accumulation_steps": 1,
+      "optimizer_step_time_sec": 0.12944729999981064,
+      "images_per_sec": 7.7251514709187665,
+      "mean_loss": 9.920842170715332,
+      "trainable_params": 1106688
+    },
+    {
+      "method": "lora_adamw",
+      "local_batch_size": 1,
+      "global_batch_size_requested": 8,
+      "status": "ok",
+      "effective_global_batch_size": 8,
+      "gradient_accumulation_steps": 8,
+      "optimizer_step_time_sec": 0.792737899999338,
+      "images_per_sec": 10.091607831550228,
+      "mean_loss": 8.131502032279968,
+      "trainable_params": 1106688
+    },
+    {
+      "method": "lora_adamw",
+      "local_batch_size": 1,
+      "global_batch_size_requested": 16,
+      "status": "ok",
+      "effective_global_batch_size": 16,
+      "gradient_accumulation_steps": 16,
+      "optimizer_step_time_sec": 1.6773667999987083,
+      "images_per_sec": 9.538760395169572,
+      "mean_loss": 8.80642619729042,
+      "trainable_params": 1106688
+    },
+    {
+      "method": "lora_adamw",
+      "local_batch_size": 2,
+      "global_batch_size_requested": 2,
+      "status": "ok",
+      "effective_global_batch_size": 2,
+      "gradient_accumulation_steps": 1,
+      "optimizer_step_time_sec": 0.20009290000052715,
+      "images_per_sec": 9.995357156574427,
+      "mean_loss": 9.088608741760254,
+      "trainable_params": 1106688
+    },
+    {
+      "method": "lora_adamw",
+      "local_batch_size": 2,
+      "global_batch_size_requested": 8,
+      "status": "ok",
+      "effective_global_batch_size": 8,
+      "gradient_accumulation_steps": 4,
+      "optimizer_step_time_sec": 0.8304937000011705,
+      "images_per_sec": 9.63282442719159,
+      "mean_loss": 8.245712995529175,
+      "trainable_params": 1106688
+    },
+    {
+      "method": "lora_adamw",
+      "local_batch_size": 2,
+      "global_batch_size_requested": 16,
+      "status": "ok",
+      "effective_global_batch_size": 16,
+      "gradient_accumulation_steps": 8,
+      "optimizer_step_time_sec": 1.6668036999981268,
+      "images_per_sec": 9.599210752902685,
+      "mean_loss": 9.106984257698059,
+      "trainable_params": 1106688
+    },
+    {
+      "method": "lora_adamw",
+      "local_batch_size": 4,
+      "global_batch_size_requested": 4,
+      "status": "ok",
+      "effective_global_batch_size": 4,
+      "gradient_accumulation_steps": 1,
+      "optimizer_step_time_sec": 0.4656030999994982,
+      "images_per_sec": 8.591008092524106,
+      "mean_loss": 8.862140655517578,
+      "trainable_params": 1106688
+    },
+    {
+      "method": "lora_adamw",
+      "local_batch_size": 4,
+      "global_batch_size_requested": 8,
+      "status": "ok",
+      "effective_global_batch_size": 8,
+      "gradient_accumulation_steps": 2,
+      "optimizer_step_time_sec": 2.6093234999989363,
+      "images_per_sec": 3.0659287742601715,
+      "mean_loss": 8.241507053375244,
+      "trainable_params": 1106688
+    },
+    {
+      "method": "lora_adamw",
+      "local_batch_size": 4,
+      "global_batch_size_requested": 16,
+      "status": "ok",
+      "effective_global_batch_size": 16,
+      "gradient_accumulation_steps": 4,
+      "optimizer_step_time_sec": 18.058491499999946,
+      "images_per_sec": 0.8860097755119827,
+      "mean_loss": 8.916554927825928,
+      "trainable_params": 1106688
+    },
+    {
+      "method": "full_adam",
+      "local_batch_size": 1,
+      "global_batch_size_requested": 1,
+      "status": "ok",
+      "effective_global_batch_size": 1,
+      "gradient_accumulation_steps": 1,
+      "optimizer_step_time_sec": 1.4309436000003188,
+      "images_per_sec": 0.6988395629288094,
+      "mean_loss": 8.042855262756348,
+      "trainable_params": 125521920
+    },
+    {
+      "method": "full_adam",
+      "local_batch_size": 1,
+      "global_batch_size_requested": 8,
+      "status": "ok",
+      "effective_global_batch_size": 8,
+      "gradient_accumulation_steps": 8,
+      "optimizer_step_time_sec": 2.7121656999988772,
+      "images_per_sec": 2.9496722858796245,
+      "mean_loss": 7.829526960849762,
+      "trainable_params": 125521920
+    },
+    {
+      "method": "full_adam",
+      "local_batch_size": 1,
+      "global_batch_size_requested": 16,
+      "status": "ok",
+      "effective_global_batch_size": 16,
+      "gradient_accumulation_steps": 16,
+      "optimizer_step_time_sec": 1.8378386999993381,
+      "images_per_sec": 8.705878268863183,
+      "mean_loss": 9.189274996519089,
+      "trainable_params": 125521920
+    },
+    {
+      "method": "full_adam",
+      "local_batch_size": 2,
+      "global_batch_size_requested": 2,
+      "status": "ok",
+      "effective_global_batch_size": 2,
+      "gradient_accumulation_steps": 1,
+      "optimizer_step_time_sec": 0.23647629999868514,
+      "images_per_sec": 8.457507158269646,
+      "mean_loss": 9.128178596496582,
+      "trainable_params": 125521920
+    },
+    {
+      "method": "full_adam",
+      "local_batch_size": 2,
+      "global_batch_size_requested": 8,
+      "status": "ok",
+      "effective_global_batch_size": 8,
+      "gradient_accumulation_steps": 4,
+      "optimizer_step_time_sec": 0.8083188999989943,
+      "images_per_sec": 9.897083935572896,
+      "mean_loss": 8.64337944984436,
+      "trainable_params": 125521920
+    },
+    {
+      "method": "full_adam",
+      "local_batch_size": 2,
+      "global_batch_size_requested": 16,
+      "status": "ok",
+      "effective_global_batch_size": 16,
+      "gradient_accumulation_steps": 8,
+      "optimizer_step_time_sec": 1.8274533999974665,
+      "images_per_sec": 8.755353214490823,
+      "mean_loss": 8.331470370292664,
+      "trainable_params": 125521920
+    },
+    {
+      "method": "full_adam",
+      "local_batch_size": 4,
+      "global_batch_size_requested": 4,
+      "status": "ok",
+      "effective_global_batch_size": 4,
+      "gradient_accumulation_steps": 1,
+      "optimizer_step_time_sec": 0.511095199999545,
+      "images_per_sec": 7.826330593602838,
+      "mean_loss": 8.954268455505371,
+      "trainable_params": 125521920
+    },
+    {
+      "method": "full_adam",
+      "local_batch_size": 4,
+      "global_batch_size_requested": 8,
+      "status": "ok",
+      "effective_global_batch_size": 8,
+      "gradient_accumulation_steps": 2,
+      "optimizer_step_time_sec": 2.2738564999981463,
+      "images_per_sec": 3.518251921353226,
+      "mean_loss": 9.192809581756592,
+      "trainable_params": 125521920
+    },
+    {
+      "method": "full_adam",
+      "local_batch_size": 4,
+      "global_batch_size_requested": 16,
+      "status": "ok",
+      "effective_global_batch_size": 16,
+      "gradient_accumulation_steps": 4,
+      "optimizer_step_time_sec": 18.631701800000883,
+      "images_per_sec": 0.8587513997244869,
+      "mean_loss": 8.159156560897827,
+      "trainable_params": 125521920
+    },
+    {
+      "method": "full_adam8bit",
+      "local_batch_size": 1,
+      "global_batch_size_requested": 1,
+      "status": "ok",
+      "effective_global_batch_size": 1,
+      "gradient_accumulation_steps": 1,
+      "optimizer_step_time_sec": 0.13992360000156623,
+      "images_per_sec": 7.146757230294293,
+      "mean_loss": 9.259998321533203,
+      "trainable_params": 125521920
+    },
+    {
+      "method": "full_adam8bit",
+      "local_batch_size": 1,
+      "global_batch_size_requested": 8,
+      "status": "ok",
+      "effective_global_batch_size": 8,
+      "gradient_accumulation_steps": 8,
+      "optimizer_step_time_sec": 0.8451360999988538,
+      "images_per_sec": 9.465930990299492,
+      "mean_loss": 8.10985803604126,
+      "trainable_params": 125521920
+    },
+    {
+      "method": "full_adam8bit",
+      "local_batch_size": 1,
+      "global_batch_size_requested": 16,
+      "status": "ok",
+      "effective_global_batch_size": 16,
+      "gradient_accumulation_steps": 16,
+      "optimizer_step_time_sec": 1.8945816999930685,
+      "images_per_sec": 8.445135936897595,
+      "mean_loss": 8.591163873672485,
+      "trainable_params": 125521920
+    },
+    {
+      "method": "full_adam8bit",
+      "local_batch_size": 2,
+      "global_batch_size_requested": 2,
+      "status": "ok",
+      "effective_global_batch_size": 2,
+      "gradient_accumulation_steps": 1,
+      "optimizer_step_time_sec": 0.23971350000101666,
+      "images_per_sec": 8.343293139483249,
+      "mean_loss": 9.75894832611084,
+      "trainable_params": 125521920
+    },
+    {
+      "method": "full_adam8bit",
+      "local_batch_size": 2,
+      "global_batch_size_requested": 8,
+      "status": "ok",
+      "effective_global_batch_size": 8,
+      "gradient_accumulation_steps": 4,
+      "optimizer_step_time_sec": 0.9259438999997656,
+      "images_per_sec": 8.6398322835779,
+      "mean_loss": 8.462790489196777,
+      "trainable_params": 125521920
+    },
+    {
+      "method": "full_adam8bit",
+      "local_batch_size": 2,
+      "global_batch_size_requested": 16,
+      "status": "ok",
+      "effective_global_batch_size": 16,
+      "gradient_accumulation_steps": 8,
+      "optimizer_step_time_sec": 1.8237968999983423,
+      "images_per_sec": 8.772906676184471,
+      "mean_loss": 10.191668510437012,
+      "trainable_params": 125521920
+    },
+    {
+      "method": "full_adam8bit",
+      "local_batch_size": 4,
+      "global_batch_size_requested": 4,
+      "status": "ok",
+      "effective_global_batch_size": 4,
+      "gradient_accumulation_steps": 1,
+      "optimizer_step_time_sec": 0.5224713000006886,
+      "images_per_sec": 7.655922918626779,
+      "mean_loss": 8.14057445526123,
+      "trainable_params": 125521920
+    },
+    {
+      "method": "full_adam8bit",
+      "local_batch_size": 4,
+      "global_batch_size_requested": 8,
+      "status": "ok",
+      "effective_global_batch_size": 8,
+      "gradient_accumulation_steps": 2,
+      "optimizer_step_time_sec": 3.7809107000011863,
+      "images_per_sec": 2.1158923430795364,
+      "mean_loss": 8.521550178527832,
+      "trainable_params": 125521920
+    },
+    {
+      "method": "full_adam8bit",
+      "local_batch_size": 4,
+      "global_batch_size_requested": 16,
+      "status": "ok",
+      "effective_global_batch_size": 16,
+      "gradient_accumulation_steps": 4,
+      "optimizer_step_time_sec": 27.688971800002037,
+      "images_per_sec": 0.5778473868790903,
+      "mean_loss": 9.247632026672363,
+      "trainable_params": 125521920
+    }
+  ]
 }

evaluations/mimic_test_metrics.json

@@ -3,21 +3,27 @@
   "dataset": "mimic-cxr",
   "view_filter": "frontal-only (PA/AP)",
   "num_examples": 3041,
-  "chexpert_f1_micro": 0.16103319543941078,
-  "chexpert_f1_macro": 0.1124493965203456,
+  "chexpert_f1_micro": 0.1897964951950254,
+  "chexpert_f1_macro": 0.1005536332270045,
   "chexpert_per_label_f1": {
-    "Atelectasis": 0.0825,
-    "Cardiomegaly": 0.00737100737100737,
-    "Consolidation": 0.04561824729891957,
-    "Edema": 0.2830349531116795,
-    "Pleural Effusion": 0.33264462809917356,
-    "Pneumonia": 0.08787346221441125,
-    "Pneumothorax": 0.0605528740675735,
+    "Enlarged Cardiomediastinum": 0.0,
+    "Cardiomegaly": 0.0,
+    "Lung Opacity": 0.0,
+    "Lung Lesion": 0.0,
+    "Edema": 0.37237569060773473,
+    "Consolidation": 0.08431952662721894,
+    "Pneumonia": 0.12570145903479238,
+    "Atelectasis": 0.0456989247311828,
+    "Pneumothorax": 0.045261669024045256,
+    "Pleural Effusion": 0.3687114573190523,
+    "Pleural Other": 0.0,
+    "Fracture": 0.0,
+    "Support Devices": 0.36568213783403664,
     "No Finding": 0.0
   },
-  "radgraph_f1": 0.09555911275393814,
-  "radgraph_f1_entity": 0.15815788437570172,
-  "radgraph_f1_relation": 0.13806964215526496,
+  "radgraph_f1": 0.09636633496704333,
+  "radgraph_f1_entity": 0.16033252587414393,
+  "radgraph_f1_relation": 0.14117679892881935,
   "radgraph_available": true,
   "radgraph_error": null
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b30676d31ff967d3ff33af89e65dcee6d59ca0c8fe2f348abdb6f9d9546c88e1
+oid sha256:85d0600d4706b6aae446467d50481d2bfde0cae44576e88a994b6f574c33bb63
 size 1152540320

run_summary.json

@@ -1,18 +1,18 @@
 {
   "method": "lora_adamw",
   "run_name": "full_3_epoch_mask_run",
-  "steps": 36576,
+  "steps": 40864,
   "epochs_completed": 1,
   "epoch_index": 1,
   "target_epochs": 3,
-  "progress_epochs": 1.072994734757931,
-  "training_completion_percent": 35.766491158597695,
-  "elapsed_seconds": 31281.024234599987,
-  "images_seen": 292640,
-  "train_loss_last": 1.442158818244934,
-  "train_loss_mean": 2.1974657088950895,
-  "val_loss": 1.482397198677063,
-  "images_per_second": 9.355192394125972,
+  "progress_epochs": 1.1987812211255005,
+  "training_completion_percent": 39.95937403751668,
+  "elapsed_seconds": 34881.30608979998,
+  "images_seen": 326946,
+  "train_loss_last": 2.278425455093384,
+  "train_loss_mean": 2.149527354583105,
+  "val_loss": 1.4887660503387452,
+  "images_per_second": 9.373100856897265,
   "trainable_params": 1106688,
   "vision_model_name": "facebook/dinov3-vits16-pretrain-lvd1689m",
   "text_model_name": "gpt2",
@@ -33,9 +33,40 @@
   "seed": 42,
   "resume_supported": true,
   "checkpoint_every_n_steps": 1000,
-  "cumulative_loss_sum": 643066.365051059,
-  "cumulative_loss_count": 292640,
+  "cumulative_loss_sum": 702779.3704715278,
+  "cumulative_loss_count": 326946,
   "completed": false,
   "target_duration_seconds": 3600,
-  "target_duration_mode": "per_invocation"
+  "target_duration_mode": "per_invocation",
+  "train_datasets": "CheXpert, MIMIC-CXR",
+  "validation_datasets": "CheXpert, MIMIC-CXR",
+  "latest_evaluation": {
+    "split": "test",
+    "dataset": "mimic-cxr",
+    "view_filter": "frontal-only (PA/AP)",
+    "num_examples": 3041,
+    "chexpert_f1_micro": 0.1897964951950254,
+    "chexpert_f1_macro": 0.1005536332270045,
+    "chexpert_per_label_f1": {
+      "Enlarged Cardiomediastinum": 0.0,
+      "Cardiomegaly": 0.0,
+      "Lung Opacity": 0.0,
+      "Lung Lesion": 0.0,
+      "Edema": 0.37237569060773473,
+      "Consolidation": 0.08431952662721894,
+      "Pneumonia": 0.12570145903479238,
+      "Atelectasis": 0.0456989247311828,
+      "Pneumothorax": 0.045261669024045256,
+      "Pleural Effusion": 0.3687114573190523,
+      "Pleural Other": 0.0,
+      "Fracture": 0.0,
+      "Support Devices": 0.36568213783403664,
+      "No Finding": 0.0
+    },
+    "radgraph_f1": 0.09636633496704333,
+    "radgraph_f1_entity": 0.16033252587414393,
+    "radgraph_f1_relation": 0.14117679892881935,
+    "radgraph_available": true,
+    "radgraph_error": null
+  }
 }

tokenizer_config.json

@@ -4,13 +4,9 @@
   "bos_token": "<|endoftext|>",
   "eos_token": "<|endoftext|>",
   "errors": "replace",
-  "is_local": true,
-  "max_length": 1022,
+  "is_local": false,
   "model_max_length": 1024,
   "pad_token": "<|endoftext|>",
-  "stride": 0,
   "tokenizer_class": "GPT2Tokenizer",
-  "truncation_side": "right",
-  "truncation_strategy": "longest_first",
   "unk_token": "<|endoftext|>"
 }