Leakage-free retrain + evaluation

README.md

@@ -17,13 +17,13 @@ NeuroGSD group project, Faculty of Information Technology, University of Moratuw
 Research and academic evaluation only. **Not a medical device.** Not clinically
 validated, no regulatory clearance, must not inform decisions about real patients.
 
-## Results (leakage-free held-out test set, n=63)
+## Results (leakage-free held-out test set, n=59)
 
 | Metric | Value | 95% CI |
 |---|---|---|
-| Accuracy | 0.937 | 0.873–0.984 |
-| Balanced accuracy | 0.902 | 0.798–0.984 |
-| Quadratic weighted kappa | 0.957 | 0.899–0.991 |
+| Accuracy | 0.983 | 0.949–1.000 |
+| Balanced accuracy | 0.964 | 0.875–1.000 |
+| Quadratic weighted kappa | 0.988 | 0.956–1.000 |
 | Macro AUC | nan | — |
 | Within-one-grade agreement | 1.000 | — |
 

best_finetune.keras

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e99a32cb8de6590d1212a15adad35205a5676ffc209c9511d5f901e6482160db
+oid sha256:41372ea19e760c46bce697010b9085d86d749ede844bd4e949ed05f69ecf0ffd
 size 70281268

best_head.keras

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ffef206a66f62fff6e0fc9bb3dbf94cb54b7869a435b73dd8622e4cc51eb0740
+oid sha256:014b894b0e7fd831ef56430674850eadc8a07adfcffabc7cea975ab5064652ff
 size 27852980

cataract_final.keras

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9eaf2b3cd4397353c1a89ea0725afa75649d5c239afa7843098e4bdd8b294a85
+oid sha256:2409fa2c040a4f03bcdedebc56a27f02bfa09c7a92390d3d86105e0d883be144
 size 70281269

evaluation.json

@@ -1,30 +1,30 @@
 {
-  "n_test": 63,
+  "n_test": 59,
   "tta": true,
   "leakage_free": true,
-  "accuracy": 0.9365079365079365,
-  "balanced_accuracy": 0.9017857142857143,
-  "qwk": 0.9574324324324325,
-  "mcc": 0.9008461919069264,
+  "accuracy": 0.9830508474576272,
+  "balanced_accuracy": 0.9642857142857143,
+  "qwk": 0.9882727092029417,
+  "mcc": 0.971513726759843,
   "macro_auc": NaN,
-  "mae_grades": 0.06349206349206349,
+  "mae_grades": 0.01694915254237288,
   "within_one_grade": 1.0,
-  "log_loss": 0.257312461587281,
+  "log_loss": 0.1904372375299754,
   "bootstrap_ci": {
     "accuracy": {
-      "mean": 0.9368492063492065,
-      "lo": 0.873015873015873,
-      "hi": 0.9841269841269841
+      "mean": 0.9830169491525423,
+      "lo": 0.9491525423728814,
+      "hi": 1.0
     },
     "balanced_accuracy": {
-      "mean": 0.9018161446820331,
-      "lo": 0.7980654761904762,
-      "hi": 0.984375
+      "mean": 0.9636371017360907,
+      "lo": 0.875,
+      "hi": 1.0
     },
     "qwk": {
-      "mean": 0.95502895660941,
-      "lo": 0.8991192641947278,
-      "hi": 0.9906408313042285
+      "mean": 0.9874861564441398,
+      "lo": 0.9562298281312692,
+      "hi": 1.0
     }
   },
   "confusion_matrix": [
@@ -37,20 +37,20 @@
     [
       0,
       6,
-      2,
+      1,
       0
     ],
     [
       0,
       0,
-      34,
+      35,
       0
     ],
     [
       0,
       0,
-      2,
-      12
+      0,
+      10
     ]
   ],
   "per_class": {
@@ -62,34 +62,34 @@
     },
     "1": {
       "precision": 1.0,
-      "recall": 0.75,
-      "f1-score": 0.8571428571428571,
-      "support": 8.0
+      "recall": 0.8571428571428571,
+      "f1-score": 0.9230769230769231,
+      "support": 7.0
     },
     "2": {
-      "precision": 0.8947368421052632,
+      "precision": 0.9722222222222222,
       "recall": 1.0,
-      "f1-score": 0.9444444444444444,
-      "support": 34.0
+      "f1-score": 0.9859154929577465,
+      "support": 35.0
     },
     "3": {
       "precision": 1.0,
-      "recall": 0.8571428571428571,
-      "f1-score": 0.9230769230769231,
-      "support": 14.0
+      "recall": 1.0,
+      "f1-score": 1.0,
+      "support": 10.0
     },
-    "accuracy": 0.9365079365079365,
+    "accuracy": 0.9830508474576272,
     "macro avg": {
-      "precision": 0.9736842105263158,
-      "recall": 0.9017857142857143,
-      "f1-score": 0.9311660561660562,
-      "support": 63.0
+      "precision": 0.9930555555555556,
+      "recall": 0.9642857142857143,
+      "f1-score": 0.9772481040086674,
+      "support": 59.0
     },
     "weighted avg": {
-      "precision": 0.9431913116123642,
-      "recall": 0.9365079365079365,
-      "f1-score": 0.9347830300211251,
-      "support": 63.0
+      "precision": 0.9835216572504708,
+      "recall": 0.9830508474576272,
+      "f1-score": 0.9825183172043997,
+      "support": 59.0
     }
   }
 }

log_finetune.csv

@@ -1,12 +1,12 @@
 epoch,acc,learning_rate,loss,val_acc,val_loss,val_qwk
-0,0.9561527371406555,9.999999747378752e-06,0.33054685592651367,0.9523809552192688,0.39878472685813904,0.8811524609843937
-1,0.9689440727233887,9.999999747378752e-06,0.3150775730609894,0.9523809552192688,0.397876501083374,0.8811524609843937
-2,0.9736024737358093,9.999999747378752e-06,0.3095778226852417,0.9523809552192688,0.3983605206012726,0.8811524609843937
-3,0.9829192757606506,9.999999747378752e-06,0.2992762327194214,0.9365079402923584,0.40606778860092163,0.8387096774193549
-4,0.9813664555549622,9.999999747378752e-06,0.3065238296985626,0.9365079402923584,0.4098004698753357,0.8387096774193549
-5,0.9736024737358093,9.999999747378752e-06,0.3139995336532593,0.9365079402923584,0.40549176931381226,0.8387096774193549
-6,0.97826087474823,9.999999747378752e-06,0.3070058524608612,0.9365079402923584,0.40747979283332825,0.8387096774193549
-7,0.9720497131347656,2.9999998787388904e-06,0.30631914734840393,0.9365079402923584,0.41367360949516296,0.8387096774193549
-8,0.9704968929290771,2.9999998787388904e-06,0.3152365982532501,0.9365079402923584,0.42263123393058777,0.8387096774193549
-9,0.9767080545425415,2.9999998787388904e-06,0.32062479853630066,0.9365079402923584,0.42704808712005615,0.8387096774193549
-10,0.9798136353492737,2.9999998787388904e-06,0.29803115129470825,0.9365079402923584,0.431363970041275,0.8387096774193549
+0,0.9881129264831543,9.999999747378752e-06,0.27213847637176514,0.9152542352676392,0.3951820731163025,0.8440105755541998
+1,0.9885993599891663,9.999999747378752e-06,0.27813324332237244,0.9152542352676392,0.39619433879852295,0.7578659370725034
+2,0.9869706630706787,9.999999747378752e-06,0.2833753228187561,0.9152542352676392,0.3945697844028473,0.7578659370725034
+3,0.9869706630706787,9.999999747378752e-06,0.27418798208236694,0.9152542352676392,0.39326488971710205,0.7578659370725034
+4,0.9837133288383484,9.999999747378752e-06,0.27106282114982605,0.9152542352676392,0.3924816846847534,0.7578659370725034
+5,0.9885993599891663,9.999999747378752e-06,0.2840113043785095,0.9152542352676392,0.392253577709198,0.7578659370725034
+6,0.9804560542106628,9.999999747378752e-06,0.2774450182914734,0.9152542352676392,0.3948898911476135,0.7578659370725034
+7,0.9918566942214966,9.999999747378752e-06,0.2633183002471924,0.9152542352676392,0.39582300186157227,0.7578659370725034
+8,0.990227997303009,9.999999747378752e-06,0.27457863092422485,0.9152542352676392,0.3983452320098877,0.7578659370725034
+9,0.990227997303009,9.999999747378752e-06,0.267590194940567,0.9152542352676392,0.40089669823646545,0.7578659370725034
+10,0.9951140284538269,9.999999747378752e-06,0.26098212599754333,0.9152542352676392,0.39580008387565613,0.7578659370725034

log_head.csv

@@ -1,26 +1,26 @@
 epoch,acc,learning_rate,loss,val_acc,val_loss,val_qwk
-0,0.6506211161613464,0.0010000000474974513,1.1065553426742554,0.6507936716079712,0.9912384748458862,0.6700610997963341
-1,0.8850931525230408,0.0010000000474974513,0.621780514717102,0.841269850730896,0.8068436980247498,0.7825086306098965
-2,0.9068322777748108,0.0010000000474974513,0.5296788811683655,0.8730158805847168,0.7578881978988647,0.816793893129771
-3,0.9208074808120728,0.0010000000474974513,0.4751219153404236,0.841269850730896,0.7114213109016418,0.7247058823529412
-4,0.9223602414131165,0.0010000000474974513,0.4478212296962738,0.9047619104385376,0.6748998761177063,0.7670588235294118
-5,0.9347826242446899,0.0010000000474974513,0.41521695256233215,0.8571428656578064,0.6612588763237,0.5826397146254458
-6,0.9409937858581543,0.0010000000474974513,0.4177846908569336,0.9047619104385376,0.6103928685188293,0.7620192307692307
-7,0.9487577676773071,0.0010000000474974513,0.40420666337013245,0.9047619104385376,0.6055206656455994,0.807061790668348
-8,0.9503105878829956,0.0010000000474974513,0.39463260769844055,0.8888888955116272,0.6145486235618591,0.7584597432905484
-9,0.9472049474716187,0.0010000000474974513,0.38087889552116394,0.8571428656578064,0.5753405690193176,0.7416762342135477
-10,0.967391312122345,0.0010000000474974513,0.3514486849308014,0.9047619104385376,0.5245923399925232,0.7318235995232419
-11,0.9440993666648865,0.0010000000474974513,0.38267603516578674,0.8730158805847168,0.4733535051345825,0.6800947867298578
-12,0.9534161686897278,0.0010000000474974513,0.35804077982902527,0.920634925365448,0.4525250494480133,0.8588661037394452
-13,0.9596273303031921,0.0010000000474974513,0.3549429476261139,0.920634925365448,0.4646533131599426,0.7758007117437722
-14,0.9549689292907715,0.0010000000474974513,0.3534187376499176,0.9047619104385376,0.4401915669441223,0.7318235995232419
-15,0.9596273303031921,0.0010000000474974513,0.35917574167251587,0.8730158805847168,0.4354936480522156,0.75
-16,0.9658384919166565,0.0010000000474974513,0.34323573112487793,0.8888888955116272,0.451433390378952,0.7291666666666667
-17,0.9580745100975037,0.0010000000474974513,0.34321698546409607,0.9047619104385376,0.4298260807991028,0.8131868131868132
-18,0.9704968929290771,0.0010000000474974513,0.326946884393692,0.9365079402923584,0.4077427089214325,0.8714285714285714
-19,0.9611801505088806,0.0010000000474974513,0.33465710282325745,0.9047619104385376,0.4530092477798462,0.8131868131868132
-20,0.9767080545425415,0.0010000000474974513,0.3187553286552429,0.920634925365448,0.4457206130027771,0.8618654073199528
-21,0.9642857313156128,0.0010000000474974513,0.3217926621437073,0.9365079402923584,0.40628862380981445,0.8714285714285714
-22,0.9720497131347656,0.0010000000474974513,0.31698986887931824,0.920634925365448,0.42284706234931946,0.8293838862559242
-23,0.9611801505088806,0.0010000000474974513,0.32016733288764954,0.9047619104385376,0.436846524477005,0.8185053380782918
-24,0.9798136353492737,0.0010000000474974513,0.30709758400917053,0.9047619104385376,0.45204219222068787,0.7877358490566038
+0,0.6612377762794495,0.0010000000474974513,1.0813597440719604,0.4237288236618042,1.1277486085891724,0.2985772753849152
+1,0.8973941206932068,0.0010000000474974513,0.6030792593955994,0.7457627058029175,0.8226550221443176,0.6426790167438546
+2,0.9348534345626831,0.0010000000474974513,0.504050612449646,0.8644067645072937,0.6954477429389954,0.719047619047619
+3,0.9446253776550293,0.0010000000474974513,0.39159250259399414,0.8813559412956238,0.6843019723892212,0.7285457091418284
+4,0.9495114088058472,0.0010000000474974513,0.4188724458217621,0.8644067645072937,0.7304669618606567,0.723545490042952
+5,0.9511400461196899,0.0010000000474974513,0.36688876152038574,0.8135592937469482,0.692980945110321,0.7009573868969401
+6,0.9723126888275146,0.0010000000474974513,0.3414730727672577,0.8813559412956238,0.6467596888542175,0.7329265892540838
+7,0.9706840515136719,0.0010000000474974513,0.335051566362381,0.9152542352676392,0.6519331932067871,0.7578659370725034
+8,0.9706840515136719,0.0010000000474974513,0.32793745398521423,0.9322034120559692,0.5852716565132141,0.7675334909377463
+9,0.9576547145843506,0.0010000000474974513,0.35312023758888245,0.9152542352676392,0.5942530035972595,0.7578659370725034
+10,0.9739413857460022,0.0010000000474974513,0.327210396528244,0.9152542352676392,0.534002959728241,0.763323782234957
+11,0.9641693830490112,0.0010000000474974513,0.32755348086357117,0.9152542352676392,0.5403637886047363,0.8501074848544069
+12,0.9804560542106628,0.0010000000474974513,0.3122323453426361,0.9322034120559692,0.47280600666999817,0.7675334909377463
+13,0.975570023059845,0.0010000000474974513,0.30920541286468506,0.8983050584793091,0.48284080624580383,0.7723765432098766
+14,0.9706840515136719,0.0010000000474974513,0.32432639598846436,0.8813559412956238,0.5428078770637512,0.7052267486548809
+15,0.9739413857460022,0.0010000000474974513,0.32023704051971436,0.9152542352676392,0.4765341877937317,0.8501074848544069
+16,0.9739413857460022,0.0010000000474974513,0.3043287396430969,0.8983050584793091,0.5218567848205566,0.9302600472813238
+17,0.9804560542106628,0.0010000000474974513,0.30412283539772034,0.8983050584793091,0.43766137957572937,0.7483520744474603
+18,0.975570023059845,0.0010000000474974513,0.29872557520866394,0.9322034120559692,0.4098648130893707,0.9131075110456554
+19,0.9853420257568359,0.0010000000474974513,0.2886912226676941,0.8983050584793091,0.43425893783569336,0.7522900763358779
+20,0.9804560542106628,0.0010000000474974513,0.2837938964366913,0.9491525292396545,0.3998212516307831,0.9627603618767094
+21,0.9788273572921753,0.0010000000474974513,0.2839686870574951,0.9152542352676392,0.42340466380119324,0.809215844785772
+22,0.9885993599891663,0.0010000000474974513,0.2704813480377197,0.9322034120559692,0.3931242823600769,0.7712291585886002
+23,0.9837133288383484,0.0010000000474974513,0.27949148416519165,0.9661017060279846,0.37841087579727173,0.881858229875851
+24,0.9869706630706787,0.0010000000474974513,0.2755202353000641,0.9661017060279846,0.37384745478630066,0.881858229875851