| { |
| "model": "/Users/adarsh/Desktop/Projects/OpenForensics/runs/v2/final.keras", |
| "threshold_sweep": [ |
| { |
| "threshold": 0.05, |
| "accuracy": 0.874, |
| "real_called_fake": 4, |
| "fake_called_real": 248 |
| }, |
| { |
| "threshold": 0.1, |
| "accuracy": 0.9085, |
| "real_called_fake": 7, |
| "fake_called_real": 176 |
| }, |
| { |
| "threshold": 0.15, |
| "accuracy": 0.922, |
| "real_called_fake": 12, |
| "fake_called_real": 144 |
| }, |
| { |
| "threshold": 0.2, |
| "accuracy": 0.9325, |
| "real_called_fake": 17, |
| "fake_called_real": 118 |
| }, |
| { |
| "threshold": 0.25, |
| "accuracy": 0.9405, |
| "real_called_fake": 20, |
| "fake_called_real": 99 |
| }, |
| { |
| "threshold": 0.3, |
| "accuracy": 0.9435, |
| "real_called_fake": 23, |
| "fake_called_real": 90 |
| }, |
| { |
| "threshold": 0.35, |
| "accuracy": 0.9465, |
| "real_called_fake": 25, |
| "fake_called_real": 82 |
| }, |
| { |
| "threshold": 0.4, |
| "accuracy": 0.95, |
| "real_called_fake": 26, |
| "fake_called_real": 74 |
| }, |
| { |
| "threshold": 0.45, |
| "accuracy": 0.9515, |
| "real_called_fake": 32, |
| "fake_called_real": 65 |
| }, |
| { |
| "threshold": 0.5, |
| "accuracy": 0.951, |
| "real_called_fake": 39, |
| "fake_called_real": 59 |
| }, |
| { |
| "threshold": 0.55, |
| "accuracy": 0.9535, |
| "real_called_fake": 44, |
| "fake_called_real": 49 |
| }, |
| { |
| "threshold": 0.6, |
| "accuracy": 0.9535, |
| "real_called_fake": 48, |
| "fake_called_real": 45 |
| }, |
| { |
| "threshold": 0.65, |
| "accuracy": 0.95, |
| "real_called_fake": 59, |
| "fake_called_real": 41 |
| }, |
| { |
| "threshold": 0.7, |
| "accuracy": 0.949, |
| "real_called_fake": 66, |
| "fake_called_real": 36 |
| }, |
| { |
| "threshold": 0.75, |
| "accuracy": 0.944, |
| "real_called_fake": 81, |
| "fake_called_real": 31 |
| }, |
| { |
| "threshold": 0.8, |
| "accuracy": 0.939, |
| "real_called_fake": 95, |
| "fake_called_real": 27 |
| }, |
| { |
| "threshold": 0.85, |
| "accuracy": 0.9335, |
| "real_called_fake": 110, |
| "fake_called_real": 23 |
| }, |
| { |
| "threshold": 0.9, |
| "accuracy": 0.9235, |
| "real_called_fake": 137, |
| "fake_called_real": 16 |
| }, |
| { |
| "threshold": 0.95, |
| "accuracy": 0.8955, |
| "real_called_fake": 201, |
| "fake_called_real": 8 |
| } |
| ], |
| "manifest_digest": "c1e5955e936b52d4", |
| "tta": true, |
| "calibration": { |
| "temperature": 0.8760257766124369, |
| "val_ece_raw": 0.008403347671031909, |
| "val_ece_calibrated": 0.0033754417697588602 |
| }, |
| "operating_point": { |
| "criterion": "target_recall", |
| "target": 0.99, |
| "threshold": 0.36217108368873596, |
| "rationale": "Chosen on validation to hold recall on genuine images at 0.99. Youden and F1 both land near 0.761, which raises real-images-called-fake from 39 to 83 without improving accuracy. The full threshold_sweep ships alongside." |
| }, |
| "test_at_0.5": { |
| "threshold": 0.5, |
| "accuracy": 0.951, |
| "roc_auc": 0.989896, |
| "pr_auc": 0.9899874925285668, |
| "confusion_matrix": [ |
| [ |
| 941, |
| 59 |
| ], |
| [ |
| 39, |
| 961 |
| ] |
| ], |
| "real_called_fake": 39, |
| "fake_called_real": 59, |
| "false_accusation_rate": 0.039, |
| "ece": 0.007834798693656874, |
| "n": 2000, |
| "fake_recall": 0.941, |
| "real_recall": 0.961 |
| }, |
| "test_at_threshold": { |
| "threshold": 0.36217108368873596, |
| "accuracy": 0.948, |
| "roc_auc": 0.989896, |
| "pr_auc": 0.9899874925285668, |
| "confusion_matrix": [ |
| [ |
| 921, |
| 79 |
| ], |
| [ |
| 25, |
| 975 |
| ] |
| ], |
| "real_called_fake": 25, |
| "fake_called_real": 79, |
| "false_accusation_rate": 0.025, |
| "ece": 0.007834798693656874, |
| "n": 2000, |
| "fake_recall": 0.921, |
| "real_recall": 0.975 |
| }, |
| "risk_coverage": [ |
| { |
| "coverage": 1.0, |
| "n": 2000, |
| "accuracy": 0.951, |
| "min_confidence": 0.5005548000335693 |
| }, |
| { |
| "coverage": 0.953, |
| "n": 1905, |
| "accuracy": 0.9690288713910761, |
| "min_confidence": 0.6960272789001465 |
| }, |
| { |
| "coverage": 0.905, |
| "n": 1810, |
| "accuracy": 0.9773480662983426, |
| "min_confidence": 0.824684739112854 |
| }, |
| { |
| "coverage": 0.858, |
| "n": 1715, |
| "accuracy": 0.9860058309037901, |
| "min_confidence": 0.8969419002532959 |
| }, |
| { |
| "coverage": 0.811, |
| "n": 1621, |
| "accuracy": 0.9907464528069093, |
| "min_confidence": 0.9355311989784241 |
| }, |
| { |
| "coverage": 0.763, |
| "n": 1526, |
| "accuracy": 0.9927916120576671, |
| "min_confidence": 0.9599580764770508 |
| }, |
| { |
| "coverage": 0.716, |
| "n": 1431, |
| "accuracy": 0.9937106918238994, |
| "min_confidence": 0.9731913805007935 |
| }, |
| { |
| "coverage": 0.668, |
| "n": 1336, |
| "accuracy": 0.9940119760479041, |
| "min_confidence": 0.981025755405426 |
| }, |
| { |
| "coverage": 0.621, |
| "n": 1242, |
| "accuracy": 0.9943639291465378, |
| "min_confidence": 0.986376166343689 |
| }, |
| { |
| "coverage": 0.574, |
| "n": 1147, |
| "accuracy": 0.997384481255449, |
| "min_confidence": 0.9896464943885803 |
| }, |
| { |
| "coverage": 0.526, |
| "n": 1052, |
| "accuracy": 0.9990494296577946, |
| "min_confidence": 0.992717444896698 |
| }, |
| { |
| "coverage": 0.479, |
| "n": 957, |
| "accuracy": 0.9989550679205852, |
| "min_confidence": 0.9947149753570557 |
| }, |
| { |
| "coverage": 0.432, |
| "n": 863, |
| "accuracy": 0.9988412514484357, |
| "min_confidence": 0.9961569905281067 |
| }, |
| { |
| "coverage": 0.384, |
| "n": 768, |
| "accuracy": 0.9986979166666666, |
| "min_confidence": 0.9970833659172058 |
| }, |
| { |
| "coverage": 0.337, |
| "n": 673, |
| "accuracy": 0.9985141158989599, |
| "min_confidence": 0.99782794713974 |
| }, |
| { |
| "coverage": 0.289, |
| "n": 578, |
| "accuracy": 1.0, |
| "min_confidence": 0.9984116554260254 |
| }, |
| { |
| "coverage": 0.242, |
| "n": 484, |
| "accuracy": 1.0, |
| "min_confidence": 0.9988195300102234 |
| }, |
| { |
| "coverage": 0.195, |
| "n": 389, |
| "accuracy": 1.0, |
| "min_confidence": 0.9991379976272583 |
| }, |
| { |
| "coverage": 0.147, |
| "n": 294, |
| "accuracy": 1.0, |
| "min_confidence": 0.9994103908538818 |
| }, |
| { |
| "coverage": 0.1, |
| "n": 200, |
| "accuracy": 1.0, |
| "min_confidence": 0.9996227025985718 |
| } |
| ], |
| "per_corruption": { |
| "clean": { |
| "threshold": 0.7607657313346863, |
| "accuracy": 0.9405, |
| "roc_auc": 0.9898959999999999, |
| "pr_auc": 0.9899874874900219, |
| "confusion_matrix": [ |
| [ |
| 973, |
| 27 |
| ], |
| [ |
| 92, |
| 908 |
| ] |
| ], |
| "real_called_fake": 92, |
| "fake_called_real": 27, |
| "false_accusation_rate": 0.092, |
| "ece": 0.01360419222712515, |
| "n": 2000, |
| "fake_recall": 0.973, |
| "real_recall": 0.908 |
| }, |
| "desaturate": { |
| "threshold": 0.7607657313346863, |
| "accuracy": 0.9255, |
| "roc_auc": 0.9873709999999999, |
| "pr_auc": 0.987042223907956, |
| "confusion_matrix": [ |
| [ |
| 976, |
| 24 |
| ], |
| [ |
| 125, |
| 875 |
| ] |
| ], |
| "real_called_fake": 125, |
| "fake_called_real": 24, |
| "false_accusation_rate": 0.125, |
| "ece": 0.012363795459270471, |
| "n": 2000, |
| "fake_recall": 0.976, |
| "real_recall": 0.875 |
| }, |
| "colour_cast": { |
| "threshold": 0.7607657313346863, |
| "accuracy": 0.923, |
| "roc_auc": 0.9873129999999999, |
| "pr_auc": 0.9875305928718285, |
| "confusion_matrix": [ |
| [ |
| 978, |
| 22 |
| ], |
| [ |
| 132, |
| 868 |
| ] |
| ], |
| "real_called_fake": 132, |
| "fake_called_real": 22, |
| "false_accusation_rate": 0.132, |
| "ece": 0.010169397294521293, |
| "n": 2000, |
| "fake_recall": 0.978, |
| "real_recall": 0.868 |
| }, |
| "gaussian_noise": { |
| "threshold": 0.7607657313346863, |
| "accuracy": 0.9105, |
| "roc_auc": 0.982704, |
| "pr_auc": 0.9828172399928267, |
| "confusion_matrix": [ |
| [ |
| 965, |
| 35 |
| ], |
| [ |
| 144, |
| 856 |
| ] |
| ], |
| "real_called_fake": 144, |
| "fake_called_real": 35, |
| "false_accusation_rate": 0.144, |
| "ece": 0.008556965082883835, |
| "n": 2000, |
| "fake_recall": 0.965, |
| "real_recall": 0.856 |
| }, |
| "speckle": { |
| "threshold": 0.7607657313346863, |
| "accuracy": 0.8745, |
| "roc_auc": 0.978552, |
| "pr_auc": 0.9775095733546318, |
| "confusion_matrix": [ |
| [ |
| 976, |
| 24 |
| ], |
| [ |
| 227, |
| 773 |
| ] |
| ], |
| "real_called_fake": 227, |
| "fake_called_real": 24, |
| "false_accusation_rate": 0.227, |
| "ece": 0.012776412010192894, |
| "n": 2000, |
| "fake_recall": 0.976, |
| "real_recall": 0.773 |
| }, |
| "blur": { |
| "threshold": 0.7607657313346863, |
| "accuracy": 0.9025, |
| "roc_auc": 0.9797769999999999, |
| "pr_auc": 0.9788085527230819, |
| "confusion_matrix": [ |
| [ |
| 960, |
| 40 |
| ], |
| [ |
| 155, |
| 845 |
| ] |
| ], |
| "real_called_fake": 155, |
| "fake_called_real": 40, |
| "false_accusation_rate": 0.155, |
| "ece": 0.01519329649209979, |
| "n": 2000, |
| "fake_recall": 0.96, |
| "real_recall": 0.845 |
| }, |
| "jpeg_artifact": { |
| "threshold": 0.7607657313346863, |
| "accuracy": 0.926, |
| "roc_auc": 0.985421, |
| "pr_auc": 0.9849191247097082, |
| "confusion_matrix": [ |
| [ |
| 966, |
| 34 |
| ], |
| [ |
| 114, |
| 886 |
| ] |
| ], |
| "real_called_fake": 114, |
| "fake_called_real": 34, |
| "false_accusation_rate": 0.114, |
| "ece": 0.010649448722600941, |
| "n": 2000, |
| "fake_recall": 0.966, |
| "real_recall": 0.886 |
| }, |
| "pixelate": { |
| "threshold": 0.7607657313346863, |
| "accuracy": 0.8725, |
| "roc_auc": 0.9622930000000001, |
| "pr_auc": 0.9616763347010586, |
| "confusion_matrix": [ |
| [ |
| 943, |
| 57 |
| ], |
| [ |
| 198, |
| 802 |
| ] |
| ], |
| "real_called_fake": 198, |
| "fake_called_real": 57, |
| "false_accusation_rate": 0.198, |
| "ece": 0.03926752486824989, |
| "n": 2000, |
| "fake_recall": 0.943, |
| "real_recall": 0.802 |
| }, |
| "brightness_shift": { |
| "threshold": 0.7607657313346863, |
| "accuracy": 0.921, |
| "roc_auc": 0.986505, |
| "pr_auc": 0.9866079174092961, |
| "confusion_matrix": [ |
| [ |
| 975, |
| 25 |
| ], |
| [ |
| 133, |
| 867 |
| ] |
| ], |
| "real_called_fake": 133, |
| "fake_called_real": 25, |
| "false_accusation_rate": 0.133, |
| "ece": 0.01671682763099669, |
| "n": 2000, |
| "fake_recall": 0.975, |
| "real_recall": 0.867 |
| }, |
| "occlusion": { |
| "threshold": 0.7607657313346863, |
| "accuracy": 0.93, |
| "roc_auc": 0.985571, |
| "pr_auc": 0.9846328275987414, |
| "confusion_matrix": [ |
| [ |
| 968, |
| 32 |
| ], |
| [ |
| 108, |
| 892 |
| ] |
| ], |
| "real_called_fake": 108, |
| "fake_called_real": 32, |
| "false_accusation_rate": 0.108, |
| "ece": 0.019010440915823004, |
| "n": 2000, |
| "fake_recall": 0.968, |
| "real_recall": 0.892 |
| } |
| }, |
| "distribution_shift": { |
| "recall_real_val_at_0.5": 0.986, |
| "recall_real_test_at_0.5": 0.961, |
| "p10_real_scores_val": 0.9771469235420227, |
| "p10_real_scores_test": 0.8302948474884033 |
| } |
| } |
