Pooled five-fold consistency: foundation models pooled like CNN/CLIP; 9-model McNemar (36 pairs); per-class pooled; pooled ensemble report

analysis/ensemble_pooled.json

@@ -0,0 +1,33 @@
+{
+  "protocol": "pooled five-fold soft vote, weight = pooled macro-F1, all nine models",
+  "members": [
+    "inception_v3",
+    "clip_openai",
+    "vgg19",
+    "resnet101",
+    "dinov2_l",
+    "densenet121",
+    "resnet50",
+    "swin_b",
+    "retfound"
+  ],
+  "ensemble": {
+    "acc": 90.43,
+    "f1": 92.71,
+    "roc_auc": 0.9945,
+    "ece": 0.0237,
+    "kappa": 0.887
+  },
+  "best_single": {
+    "model": "inception_v3",
+    "acc": 90.18,
+    "f1": 92.54
+  },
+  "ensemble_vs_best_mcnemar": {
+    "b": 54,
+    "c": 46,
+    "p": 0.484,
+    "significant_005": false
+  },
+  "n_test": 3208
+}

analysis/per_class_metrics_pooled.csv

@@ -0,0 +1,91 @@
+model,class,precision,recall,f1,support
+inception_v3,Central Serous Chorioretinopathy,0.8947368421052632,0.9444444444444444,0.918918918918919,108
+inception_v3,Diabetic Retinopathy,0.9957386363636364,0.956343792633015,0.9756437021572721,733
+inception_v3,Disc Edema,0.9072847682119205,1.0,0.9513888888888888,137
+inception_v3,Glaucoma,0.840630472854641,0.7881773399014779,0.8135593220338984,609
+inception_v3,Healthy,0.8610108303249098,0.8704379562043796,0.8656987295825771,548
+inception_v3,Macular Scar,0.8640226628895185,0.9131736526946108,0.8879184861717613,334
+inception_v3,Myopia,0.8713968957871396,0.8931818181818182,0.8821548821548821,440
+inception_v3,Pterygium,1.0,1.0,1.0,17
+inception_v3,Retinal Detachment,0.9921875,1.0,0.996078431372549,127
+inception_v3,Retinitis Pigmentosa,0.9333333333333333,0.9935483870967742,0.9625,155
+clip_openai,Central Serous Chorioretinopathy,0.907563025210084,1.0,0.9515418502202643,108
+clip_openai,Diabetic Retinopathy,0.9914772727272727,0.9522510231923602,0.9714683368128044,733
+clip_openai,Disc Edema,0.9072847682119205,1.0,0.9513888888888888,137
+clip_openai,Glaucoma,0.8493647912885662,0.7684729064039408,0.8068965517241379,609
+clip_openai,Healthy,0.8397212543554007,0.8795620437956204,0.8591800356506238,548
+clip_openai,Macular Scar,0.8731988472622478,0.907185628742515,0.8898678414096917,334
+clip_openai,Myopia,0.8802660753880266,0.9022727272727272,0.8911335578002245,440
+clip_openai,Pterygium,1.0,1.0,1.0,17
+clip_openai,Retinal Detachment,0.9921875,1.0,0.996078431372549,127
+clip_openai,Retinitis Pigmentosa,0.9337349397590361,1.0,0.9657320872274143,155
+vgg19,Central Serous Chorioretinopathy,0.8983050847457628,0.9814814814814815,0.9380530973451328,108
+vgg19,Diabetic Retinopathy,0.9831460674157303,0.9549795361527967,0.9688581314878892,733
+vgg19,Disc Edema,0.9066666666666666,0.9927007299270073,0.9477351916376306,137
+vgg19,Glaucoma,0.8584392014519057,0.7766830870279147,0.8155172413793104,609
+vgg19,Healthy,0.8566308243727598,0.8722627737226277,0.864376130198915,548
+vgg19,Macular Scar,0.8567335243553008,0.8952095808383234,0.8755490483162518,334
+vgg19,Myopia,0.8733624454148472,0.9090909090909091,0.89086859688196,440
+vgg19,Pterygium,1.0,1.0,1.0,17
+vgg19,Retinal Detachment,0.9844961240310077,1.0,0.9921875,127
+vgg19,Retinitis Pigmentosa,0.9337349397590361,1.0,0.9657320872274143,155
+resnet101,Central Serous Chorioretinopathy,0.9043478260869565,0.9629629629629629,0.9327354260089686,108
+resnet101,Diabetic Retinopathy,0.983026874115983,0.9481582537517054,0.9652777777777778,733
+resnet101,Disc Edema,0.9066666666666666,0.9927007299270073,0.9477351916376306,137
+resnet101,Glaucoma,0.8527272727272728,0.7701149425287356,0.8093183779119931,609
+resnet101,Healthy,0.864376130198915,0.8722627737226277,0.8683015440508629,548
+resnet101,Macular Scar,0.8689458689458689,0.9131736526946108,0.8905109489051095,334
+resnet101,Myopia,0.8577494692144374,0.9181818181818182,0.8869374313940724,440
+resnet101,Pterygium,1.0,1.0,1.0,17
+resnet101,Retinal Detachment,0.9921875,1.0,0.996078431372549,127
+resnet101,Retinitis Pigmentosa,0.9337349397590361,1.0,0.9657320872274143,155
+dinov2_l,Central Serous Chorioretinopathy,0.8956521739130435,0.9537037037037037,0.9237668161434978,108
+dinov2_l,Diabetic Retinopathy,0.9846582984658299,0.9631650750341064,0.9737931034482759,733
+dinov2_l,Disc Edema,0.9379310344827586,0.9927007299270073,0.9645390070921985,137
+dinov2_l,Glaucoma,0.8339100346020761,0.7914614121510674,0.8121314237573716,609
+dinov2_l,Healthy,0.8597122302158273,0.8722627737226277,0.8659420289855072,548
+dinov2_l,Macular Scar,0.8604651162790697,0.8862275449101796,0.8731563421828908,334
+dinov2_l,Myopia,0.8800904977375565,0.884090909090909,0.8820861678004536,440
+dinov2_l,Pterygium,1.0,1.0,1.0,17
+dinov2_l,Retinal Detachment,0.9921875,1.0,0.996078431372549,127
+dinov2_l,Retinitis Pigmentosa,0.9337349397590361,1.0,0.9657320872274143,155
+densenet121,Central Serous Chorioretinopathy,0.8947368421052632,0.9444444444444444,0.918918918918919,108
+densenet121,Diabetic Retinopathy,0.9818688981868898,0.9604365620736699,0.9710344827586207,733
+densenet121,Disc Edema,0.9315068493150684,0.9927007299270073,0.9611307420494699,137
+densenet121,Glaucoma,0.8324514991181657,0.7750410509031199,0.8027210884353742,609
+densenet121,Healthy,0.8472468916518651,0.8704379562043796,0.8586858685868587,548
+densenet121,Macular Scar,0.86,0.9011976047904192,0.8801169590643275,334
+densenet121,Myopia,0.875,0.875,0.875,440
+densenet121,Pterygium,1.0,1.0,1.0,17
+densenet121,Retinal Detachment,0.9921875,1.0,0.996078431372549,127
+densenet121,Retinitis Pigmentosa,0.9337349397590361,1.0,0.9657320872274143,155
+resnet50,Central Serous Chorioretinopathy,0.8947368421052632,0.9444444444444444,0.918918918918919,108
+resnet50,Diabetic Retinopathy,0.9777777777777777,0.9604365620736699,0.969029593943565,733
+resnet50,Disc Edema,0.9194630872483222,1.0,0.958041958041958,137
+resnet50,Glaucoma,0.830122591943958,0.7783251231527094,0.8033898305084746,609
+resnet50,Healthy,0.850828729281768,0.843065693430657,0.846929422548121,548
+resnet50,Macular Scar,0.8587570621468926,0.9101796407185628,0.8837209302325582,334
+resnet50,Myopia,0.8721973094170403,0.884090909090909,0.8781038374717833,440
+resnet50,Pterygium,1.0,1.0,1.0,17
+resnet50,Retinal Detachment,0.9921875,1.0,0.996078431372549,127
+resnet50,Retinitis Pigmentosa,0.9337349397590361,1.0,0.9657320872274143,155
+swin_b,Central Serous Chorioretinopathy,0.8899082568807339,0.8981481481481481,0.8940092165898618,108
+swin_b,Diabetic Retinopathy,0.9775910364145658,0.9522510231923602,0.9647546648237734,733
+swin_b,Disc Edema,0.9072847682119205,1.0,0.9513888888888888,137
+swin_b,Glaucoma,0.7552674230145867,0.7651888341543513,0.7601957585644372,609
+swin_b,Healthy,0.8506616257088847,0.8211678832116789,0.8356545961002786,548
+swin_b,Macular Scar,0.8676470588235294,0.8832335329341318,0.8753709198813057,334
+swin_b,Myopia,0.8279816513761468,0.8204545454545454,0.8242009132420092,440
+swin_b,Pterygium,1.0,1.0,1.0,17
+swin_b,Retinal Detachment,0.9844961240310077,1.0,0.9921875,127
+swin_b,Retinitis Pigmentosa,0.9337349397590361,1.0,0.9657320872274143,155
+retfound,Central Serous Chorioretinopathy,0.7727272727272727,0.9444444444444444,0.85,108
+retfound,Diabetic Retinopathy,0.9691011235955056,0.9413369713506139,0.9550173010380623,733
+retfound,Disc Edema,0.8851351351351351,0.9562043795620438,0.9192982456140351,137
+retfound,Glaucoma,0.7291666666666666,0.632183908045977,0.6772207563764292,609
+retfound,Healthy,0.7688356164383562,0.8193430656934306,0.7932862190812721,548
+retfound,Macular Scar,0.839344262295082,0.7664670658682635,0.8012519561815337,334
+retfound,Myopia,0.7468879668049793,0.8181818181818182,0.7809110629067245,440
+retfound,Pterygium,1.0,1.0,1.0,17
+retfound,Retinal Detachment,0.9621212121212122,1.0,0.9806949806949807,127
+retfound,Retinitis Pigmentosa,0.9226190476190477,1.0,0.9597523219814241,155

kfold/cnn_clip/mcnemar.json

@@ -1,77 +1,198 @@
 {
-  "vgg19_vs_resnet50": {
-    "b": 85,
-    "c": 65,
-    "p": 0.1205291697094882
-  },
-  "vgg19_vs_resnet101": {
-    "b": 79,
-    "c": 78,
-    "p": 1.0
-  },
-  "vgg19_vs_densenet121": {
-    "b": 77,
-    "c": 62,
-    "p": 0.23493150329059748
-  },
-  "vgg19_vs_inception_v3": {
-    "b": 69,
-    "c": 71,
-    "p": 0.9326867554513667
-  },
-  "vgg19_vs_clip_openai": {
-    "b": 69,
-    "c": 70,
-    "p": 1.0
-  },
-  "resnet50_vs_resnet101": {
-    "b": 65,
-    "c": 84,
-    "p": 0.14005540740614844
-  },
-  "resnet50_vs_densenet121": {
-    "b": 52,
-    "c": 57,
-    "p": 0.7018111820646833
-  },
-  "resnet50_vs_inception_v3": {
-    "b": 66,
-    "c": 88,
-    "p": 0.09027672732911116
-  },
-  "resnet50_vs_clip_openai": {
-    "b": 72,
-    "c": 93,
-    "p": 0.1192051902607031
-  },
-  "resnet101_vs_densenet121": {
-    "b": 80,
-    "c": 66,
-    "p": 0.2819365341298733
-  },
-  "resnet101_vs_inception_v3": {
-    "b": 74,
-    "c": 77,
-    "p": 0.8707787409560847
-  },
-  "resnet101_vs_clip_openai": {
-    "b": 73,
-    "c": 75,
-    "p": 0.934524985417101
-  },
-  "densenet121_vs_inception_v3": {
-    "b": 51,
-    "c": 68,
-    "p": 0.14213202807505837
-  },
-  "densenet121_vs_clip_openai": {
-    "b": 67,
-    "c": 83,
-    "p": 0.22054483769441566
-  },
-  "inception_v3_vs_clip_openai": {
-    "b": 79,
-    "c": 78,
-    "p": 1.0
-  }
+ "inception_v3_vs_clip_openai": {
+  "b": 79,
+  "c": 78,
+  "p": 1.0
+ },
+ "inception_v3_vs_vgg19": {
+  "b": 71,
+  "c": 69,
+  "p": 0.9326867554513667
+ },
+ "inception_v3_vs_resnet101": {
+  "b": 77,
+  "c": 74,
+  "p": 0.8707787409560847
+ },
+ "inception_v3_vs_dinov2_l": {
+  "b": 69,
+  "c": 65,
+  "p": 0.7956287149703885
+ },
+ "inception_v3_vs_densenet121": {
+  "b": 68,
+  "c": 51,
+  "p": 0.14213202807505837
+ },
+ "inception_v3_vs_resnet50": {
+  "b": 88,
+  "c": 66,
+  "p": 0.09027672732911116
+ },
+ "inception_v3_vs_swin_b": {
+  "b": 180,
+  "c": 90,
+  "p": 4.610619902983377e-08
+ },
+ "inception_v3_vs_retfound": {
+  "b": 315,
+  "c": 94,
+  "p": 5.697404677187621e-29
+ },
+ "clip_openai_vs_vgg19": {
+  "b": 70,
+  "c": 69,
+  "p": 1.0
+ },
+ "clip_openai_vs_resnet101": {
+  "b": 75,
+  "c": 73,
+  "p": 0.934524985417101
+ },
+ "clip_openai_vs_dinov2_l": {
+  "b": 66,
+  "c": 63,
+  "p": 0.8603106786164316
+ },
+ "clip_openai_vs_densenet121": {
+  "b": 83,
+  "c": 67,
+  "p": 0.22054483769441566
+ },
+ "clip_openai_vs_resnet50": {
+  "b": 93,
+  "c": 72,
+  "p": 0.1192051902607031
+ },
+ "clip_openai_vs_swin_b": {
+  "b": 181,
+  "c": 92,
+  "p": 7.77569309248485e-08
+ },
+ "clip_openai_vs_retfound": {
+  "b": 314,
+  "c": 94,
+  "p": 8.787560694760801e-29
+ },
+ "vgg19_vs_resnet101": {
+  "b": 79,
+  "c": 78,
+  "p": 1.0
+ },
+ "vgg19_vs_dinov2_l": {
+  "b": 64,
+  "c": 62,
+  "p": 0.9290596866317961
+ },
+ "vgg19_vs_densenet121": {
+  "b": 77,
+  "c": 62,
+  "p": 0.23493150329059748
+ },
+ "vgg19_vs_resnet50": {
+  "b": 85,
+  "c": 65,
+  "p": 0.1205291697094882
+ },
+ "vgg19_vs_swin_b": {
+  "b": 172,
+  "c": 84,
+  "p": 4.012384715025164e-08
+ },
+ "vgg19_vs_retfound": {
+  "b": 302,
+  "c": 83,
+  "p": 2.471793518151419e-30
+ },
+ "resnet101_vs_dinov2_l": {
+  "b": 74,
+  "c": 73,
+  "p": 1.0
+ },
+ "resnet101_vs_densenet121": {
+  "b": 80,
+  "c": 66,
+  "p": 0.2819365341298733
+ },
+ "resnet101_vs_resnet50": {
+  "b": 84,
+  "c": 65,
+  "p": 0.14005540740614844
+ },
+ "resnet101_vs_swin_b": {
+  "b": 185,
+  "c": 98,
+  "p": 2.5864512510148e-07
+ },
+ "resnet101_vs_retfound": {
+  "b": 320,
+  "c": 102,
+  "p": 2.780978964366266e-27
+ },
+ "dinov2_l_vs_densenet121": {
+  "b": 50,
+  "c": 37,
+  "p": 0.1979791406725175
+ },
+ "dinov2_l_vs_resnet50": {
+  "b": 72,
+  "c": 54,
+  "p": 0.12957613928638612
+ },
+ "dinov2_l_vs_swin_b": {
+  "b": 165,
+  "c": 79,
+  "p": 3.858135530104346e-08
+ },
+ "dinov2_l_vs_retfound": {
+  "b": 313,
+  "c": 96,
+  "p": 6.251629311872708e-28
+ },
+ "densenet121_vs_resnet50": {
+  "b": 57,
+  "c": 52,
+  "p": 0.7018111820646833
+ },
+ "densenet121_vs_swin_b": {
+  "b": 175,
+  "c": 102,
+  "p": 1.363248210508541e-05
+ },
+ "densenet121_vs_retfound": {
+  "b": 320,
+  "c": 116,
+  "p": 3.729180955951273e-23
+ },
+ "resnet50_vs_swin_b": {
+  "b": 163,
+  "c": 95,
+  "p": 2.7421155370550847e-05
+ },
+ "resnet50_vs_retfound": {
+  "b": 304,
+  "c": 105,
+  "p": 1.5570182304471996e-23
+ },
+ "swin_b_vs_retfound": {
+  "b": 207,
+  "c": 76,
+  "p": 3.469840032407836e-15
+ },
+ "_meta": {
+  "models": [
+   "inception_v3",
+   "clip_openai",
+   "vgg19",
+   "resnet101",
+   "dinov2_l",
+   "densenet121",
+   "resnet50",
+   "swin_b",
+   "retfound"
+  ],
+  "n_pairs": 36,
+  "bonferroni_factor": 36,
+  "test": "exact binomial two-sided McNemar"
+ }
 }

kfold/kfold_v2_summary.csv

@@ -1,10 +1,10 @@
 model,family,test_acc,test_acc_ci_lo,test_acc_ci_hi,test_f1,test_f1_ci_lo,test_f1_ci_hi,test_precision,test_recall,roc_auc,pr_auc,ece,kappa,brier,n_folds,kappa_note
-inception_v3,CNN/CLIP,90.18,89.24,91.24,92.54,91.68,93.4,91.6,93.59,0.993,0.9676,0.0194,0.884,0.015,5,5-fold pooled preds
-clip_openai,CNN/CLIP,90.15,89.18,91.24,92.83,92.0,93.61,91.75,94.1,0.9944,0.9701,0.0217,0.884,0.014,5,5-fold pooled preds
-vgg19,CNN/CLIP,90.12,89.09,91.12,92.59,91.77,93.41,91.52,93.82,0.9933,0.9644,0.0228,0.884,0.015,5,5-fold pooled preds
-resnet101,CNN/CLIP,90.09,89.15,91.12,92.63,91.77,93.47,91.64,93.78,0.9941,0.9688,0.0243,0.883,0.014,5,5-fold pooled preds
-densenet121,CNN/CLIP,89.65,88.62,90.71,92.29,91.37,93.07,91.49,93.19,0.9937,0.9658,0.0272,0.878,0.015,5,5-fold pooled preds
-dinov2_l,Foundation,89.61,88.57,90.64,92.27,91.38,93.08,91.48,93.15,0.9934,0.9672,0.0299,0.876,0.016,5,single-run holdout (fold preds not stored)
-resnet50,CNN/CLIP,89.5,88.4,90.59,92.2,91.34,93.03,91.3,93.21,0.9945,0.9699,0.0339,0.876,0.014,5,5-fold pooled preds
-swin_b,Foundation,87.0,85.92,88.15,90.26,89.31,91.19,89.7,90.96,0.9896,0.945,0.0294,0.845,0.019,5,single-run holdout (fold preds not stored)
-retfound,Foundation,83.35,82.16,84.65,87.27,86.22,88.35,86.11,88.86,0.9834,0.9144,0.0242,0.81,0.024,5,single-run holdout (fold preds not stored)
+inception_v3,CNN/CLIP,90.18,89.24,91.24,92.54,91.68,93.4,91.6,93.59,0.993,0.9676,0.0194,0.884,0.015,5,five-fold pooled preds
+clip_openai,CNN/CLIP,90.15,89.18,91.24,92.83,92.0,93.61,91.75,94.1,0.9944,0.9701,0.0217,0.884,0.014,5,five-fold pooled preds
+vgg19,CNN/CLIP,90.12,89.09,91.12,92.59,91.77,93.41,91.52,93.82,0.9933,0.9644,0.0228,0.884,0.015,5,five-fold pooled preds
+resnet101,CNN/CLIP,90.09,89.15,91.12,92.63,91.77,93.47,91.64,93.78,0.9941,0.9688,0.0243,0.883,0.014,5,five-fold pooled preds
+dinov2_l,Foundation,90.06,89.06,91.05,92.57,91.71,93.39,91.78,93.44,0.9941,0.9698,0.018,0.883,0.014,5,five-fold pooled preds
+densenet121,CNN/CLIP,89.65,88.62,90.71,92.29,91.37,93.07,91.49,93.19,0.9937,0.9658,0.0272,0.878,0.015,5,five-fold pooled preds
+resnet50,CNN/CLIP,89.5,88.4,90.59,92.2,91.34,93.03,91.3,93.21,0.9945,0.9699,0.0339,0.876,0.014,5,five-fold pooled preds
+swin_b,Foundation,87.38,86.22,88.53,90.63,89.68,91.56,89.95,91.4,0.9905,0.9482,0.033,0.851,0.018,5,five-fold pooled preds
+retfound,Foundation,83.29,82.04,84.51,87.17,86.07,88.23,85.96,88.78,0.9837,0.9155,0.02,0.803,0.024,5,five-fold pooled preds