| { |
| "selected_artifact": "/data/lfm2.5_encoder_lab/outputs/monet-100k/final-gptq-int4-clean", |
| "selection_role": "publication-oriented clean image-only encoder branch", |
| "training_correction": "image branch never receives candidate caption during contrastive training or retrieval evaluation", |
| "dataset": "/data/datasets/monet-100k", |
| "external_eval": "/data/datasets/deep-eval-cc3m-2k", |
| "compression": "GPTQ INT4 W4A16 with fused calibration", |
| "size_MB": 370.4, |
| "clean_monet_bf16": { |
| "pair_count": 12500, |
| "image_to_text": { |
| "recall_at_1": 0.11935999989509583, |
| "recall_at_5": 0.3088800013065338, |
| "recall_at_10": 0.4241600036621094, |
| "mean_rank": 58.65480041503906, |
| "median_rank": 16.0, |
| "bootstrap_95ci": { |
| "recall_at_1": [ |
| 0.11336000263690948, |
| 0.12439999729394913 |
| ], |
| "recall_at_5": [ |
| 0.2990399897098541, |
| 0.3176800012588501 |
| ], |
| "recall_at_10": [ |
| 0.4156000018119812, |
| 0.432559996843338 |
| ], |
| "mean_rank": [ |
| 56.50775909423828, |
| 61.287200927734375 |
| ], |
| "resamples": 200 |
| } |
| }, |
| "text_to_image": { |
| "recall_at_1": 0.15783999860286713, |
| "recall_at_5": 0.3598400056362152, |
| "recall_at_10": 0.4731200039386749, |
| "mean_rank": 50.04256057739258, |
| "median_rank": 12.0, |
| "bootstrap_95ci": { |
| "recall_at_1": [ |
| 0.1515199989080429, |
| 0.1632000058889389 |
| ], |
| "recall_at_5": [ |
| 0.3503200113773346, |
| 0.36711999773979187 |
| ], |
| "recall_at_10": [ |
| 0.4652799963951111, |
| 0.4806399941444397 |
| ], |
| "mean_rank": [ |
| 47.80712127685547, |
| 52.608638763427734 |
| ], |
| "resamples": 200 |
| } |
| }, |
| "finite_hidden": true |
| }, |
| "clean_monet_gptq": { |
| "pair_count": 12500, |
| "image_to_text": { |
| "recall_at_1": 0.10927999764680862, |
| "recall_at_5": 0.2959200143814087, |
| "recall_at_10": 0.4089600145816803, |
| "mean_rank": 63.38008117675781, |
| "median_rank": 17.0, |
| "bootstrap_95ci": { |
| "recall_at_1": [ |
| 0.10367999970912933, |
| 0.11488000303506851 |
| ], |
| "recall_at_5": [ |
| 0.28672000765800476, |
| 0.30616000294685364 |
| ], |
| "recall_at_10": [ |
| 0.39952000975608826, |
| 0.4176799952983856 |
| ], |
| "mean_rank": [ |
| 61.031761169433594, |
| 65.88712310791016 |
| ], |
| "resamples": 200 |
| } |
| }, |
| "text_to_image": { |
| "recall_at_1": 0.1459999978542328, |
| "recall_at_5": 0.341839998960495, |
| "recall_at_10": 0.45903998613357544, |
| "mean_rank": 53.78031921386719, |
| "median_rank": 13.0, |
| "bootstrap_95ci": { |
| "recall_at_1": [ |
| 0.13991999626159668, |
| 0.15112000703811646 |
| ], |
| "recall_at_5": [ |
| 0.3343999981880188, |
| 0.34887999296188354 |
| ], |
| "recall_at_10": [ |
| 0.44944000244140625, |
| 0.46775999665260315 |
| ], |
| "mean_rank": [ |
| 51.349281311035156, |
| 56.50944137573242 |
| ], |
| "resamples": 200 |
| } |
| }, |
| "finite_hidden": true |
| }, |
| "clean_cc3m_gptq": { |
| "pair_count": 2000, |
| "image_to_text": { |
| "recall_at_1": 0.08649999648332596, |
| "recall_at_5": 0.21899999678134918, |
| "recall_at_10": 0.3059999942779541, |
| "mean_rank": 137.33999633789062, |
| "median_rank": 35.0, |
| "bootstrap_95ci": { |
| "recall_at_1": [ |
| 0.07349999994039536, |
| 0.09950000047683716 |
| ], |
| "recall_at_5": [ |
| 0.20100000500679016, |
| 0.2409999966621399 |
| ], |
| "recall_at_10": [ |
| 0.2854999899864197, |
| 0.3269999921321869 |
| ], |
| "mean_rank": [ |
| 127.39800262451172, |
| 148.1855010986328 |
| ], |
| "resamples": 200 |
| } |
| }, |
| "text_to_image": { |
| "recall_at_1": 0.1145000010728836, |
| "recall_at_5": 0.28349998593330383, |
| "recall_at_10": 0.3865000009536743, |
| "mean_rank": 112.76300048828125, |
| "median_rank": 19.0, |
| "bootstrap_95ci": { |
| "recall_at_1": [ |
| 0.10100000351667404, |
| 0.13099999725818634 |
| ], |
| "recall_at_5": [ |
| 0.26499998569488525, |
| 0.30300000309944153 |
| ], |
| "recall_at_10": [ |
| 0.367000013589859, |
| 0.4115000069141388 |
| ], |
| "mean_rank": [ |
| 102.36699676513672, |
| 124.84700012207031 |
| ], |
| "resamples": 200 |
| } |
| }, |
| "finite_hidden": true |
| }, |
| "matching": { |
| "monet_bf16": { |
| "cyclic_negative": { |
| "pairs": 12500, |
| "accuracy_at_zero": 0.9342, |
| "positive_beats_negative": 0.981440007686615, |
| "positive_mean": 2.9899163246154785, |
| "negative_mean": -6.390422344207764, |
| "margin_mean": 9.380338668823242, |
| "auroc": 0.9746800661087036, |
| "auprc": 0.9634296875 |
| }, |
| "hard_negative": { |
| "pairs": 12500, |
| "accuracy_at_zero": 0.50116, |
| "positive_beats_negative": 0.48607999086380005, |
| "positive_mean": 2.9899163246154785, |
| "negative_mean": 2.9905307292938232, |
| "margin_mean": -0.0006143388454802334, |
| "auroc": 0.49737921357154846, |
| "auprc": 0.497440703125 |
| }, |
| "zero_visual_positive": { |
| "positive_mean": -0.28617942333221436, |
| "positive_minus_trained_mean": 3.2760958671569824 |
| } |
| }, |
| "monet_gptq": { |
| "cyclic_negative": { |
| "pairs": 12500, |
| "accuracy_at_zero": 0.93128, |
| "positive_beats_negative": 0.9807999730110168, |
| "positive_mean": 2.9612369537353516, |
| "negative_mean": -6.149127960205078, |
| "margin_mean": 9.110363960266113, |
| "auroc": 0.9733310341835022, |
| "auprc": 0.96191859375 |
| }, |
| "hard_negative": { |
| "pairs": 12500, |
| "accuracy_at_zero": 0.50064, |
| "positive_beats_negative": 0.4914399981498718, |
| "positive_mean": 2.9612369537353516, |
| "negative_mean": 2.967132568359375, |
| "margin_mean": -0.005896187853068113, |
| "auroc": 0.49611741304397583, |
| "auprc": 0.49736484375 |
| }, |
| "zero_visual_positive": { |
| "positive_mean": -0.13991403579711914, |
| "positive_minus_trained_mean": 3.1011509895324707 |
| } |
| }, |
| "cc3m_gptq": { |
| "cyclic_negative": { |
| "pairs": 2000, |
| "accuracy_at_zero": 0.7265, |
| "positive_beats_negative": 0.8690000176429749, |
| "positive_mean": 1.1929634809494019, |
| "negative_mean": -1.334358811378479, |
| "margin_mean": 2.527322292327881, |
| "auroc": 0.8065452575683594, |
| "auprc": 0.7932015380859375 |
| }, |
| "hard_negative": { |
| "pairs": 2000, |
| "accuracy_at_zero": 0.5185, |
| "positive_beats_negative": 0.5665000081062317, |
| "positive_mean": 1.1929634809494019, |
| "negative_mean": 1.0014755725860596, |
| "margin_mean": 0.19148792326450348, |
| "auroc": 0.525624692440033, |
| "auprc": 0.5216671752929688 |
| }, |
| "zero_visual_positive": { |
| "positive_mean": 0.3554673492908478, |
| "positive_minus_trained_mean": 0.8374961614608765 |
| } |
| } |
| }, |
| "fidelity": { |
| "hidden_cosine_mean": 0.9427031874656677, |
| "logits_cosine_mean": 0.965385913848877, |
| "logits_top1_agreement": 0.8484848737716675 |
| }, |
| "reports": "/data/lfm2.5_encoder_lab/reports/deep_eval", |
| "caveats": [ |
| "hard-negative AUROC is near chance", |
| "masked-token prediction does not improve over zero-image controls", |
| "external subset is bounded CC3M and source terms require review", |
| "no generation/decoder is included" |
| ] |
| } |
|
|
