Publish challenger weights and A/B artifacts

ab_comparison.json

@@ -0,0 +1,44 @@
+{
+  "n_images": 8125,
+  "accuracy_a": 0.9911384615384615,
+  "accuracy_b": 0.9907692307692307,
+  "mcnemar": {
+    "both_correct": 8004,
+    "only_a_correct": 49,
+    "only_b_correct": 46,
+    "both_wrong": 26,
+    "statistic": 0.042105263157894736,
+    "p_value": 0.8374192519835879,
+    "method": "chi-squared (continuity corrected)"
+  },
+  "bootstrap": {
+    "observed_difference": -0.00036923076923078835,
+    "ci_low": -0.0027076923076923,
+    "ci_high": 0.001969230769230834,
+    "n_resamples": 10000,
+    "confidence": 0.95,
+    "p_value": 0.8012
+  },
+  "per_class": {
+    "mean_difference": 0.0037079681416117925,
+    "t_statistic": 1.23458202602619,
+    "p_value": 0.22476960764208556,
+    "df": 37,
+    "ci_low": -0.0023775384859782067,
+    "ci_high": 0.009793474769201792,
+    "cohens_d": 0.2002756507644646,
+    "effect_size_label": "small"
+  },
+  "power": {
+    "effect_size": 0.2002756507644646,
+    "n": 38,
+    "alpha": 0.05,
+    "power": 0.22522047555120145,
+    "n_for_80_percent_power": 198
+  },
+  "notes": [
+    "NOTE: the per-class test is underpowered \u2014 a null result here is inconclusive, not evidence of equivalence."
+  ],
+  "baseline_file": "artifacts/preds_baseline.npz",
+  "challenger_file": "artifacts/preds_challenger.npz"
+}

challenger.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ed5fbaaac55c64f1529a0ea0af804cde8a56d4a840746f5fc47fbc89158486c
+size 111475217

challenger_calibration.json

@@ -0,0 +1,93 @@
+{
+  "temperature": 0.5913705321969809,
+  "fitted_on": "artifacts/preds_challenger_val.npz",
+  "n_fit": 8172,
+  "evaluated_on": "artifacts/preds_challenger.npz",
+  "n_eval": 8125,
+  "before": {
+    "ece": 0.09274875829403217,
+    "mce": 0.3013893961906433,
+    "brier": 0.02436713844387271,
+    "accuracy": 0.9907692307692307,
+    "mean_confidence": 0.8982142806053162,
+    "n_bins": 15
+  },
+  "after": {
+    "ece": 0.004247930189491249,
+    "mce": 0.42238962781387135,
+    "brier": 0.015956286251189052,
+    "accuracy": 0.9907692307692307,
+    "mean_confidence": 0.9948689015885677,
+    "n_bins": 15
+  },
+  "reliability_after": {
+    "bin_center": [
+      0.03333333333333333,
+      0.1,
+      0.16666666666666669,
+      0.23333333333333334,
+      0.3,
+      0.3666666666666667,
+      0.43333333333333335,
+      0.5,
+      0.5666666666666667,
+      0.6333333333333333,
+      0.7,
+      0.7666666666666666,
+      0.8333333333333334,
+      0.9,
+      0.9666666666666667
+    ],
+    "confidence": [
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      0.42238962781387135,
+      0.4982589606799188,
+      0.5613511057946308,
+      0.6282423298211325,
+      0.6923367621433499,
+      0.7696053907150049,
+      0.8401243896322524,
+      0.903614667737649,
+      0.9976575836120416
+    ],
+    "accuracy": [
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      0.0,
+      0.25,
+      0.5384615384615384,
+      0.7142857142857143,
+      0.5,
+      0.6153846153846154,
+      0.7333333333333333,
+      0.8387096774193549,
+      0.9946397407130392
+    ],
+    "count": [
+      0,
+      0,
+      0,
+      0,
+      0,
+      0,
+      1,
+      4,
+      13,
+      7,
+      4,
+      13,
+      30,
+      31,
+      8022
+    ]
+  }
+}

preds_baseline.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2dde94ba05b2c00ae30d8dab50331cba6de66db8b0acf89ae71c32ef3b936980
+size 2439363

preds_challenger.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:95efb4a0f1cd3c643f8347199b06167304222a48837cd563e29a2fb517aa6819
+size 2465185

preds_challenger_val.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c97509af4939914520959398152e652ac0aea58764d7d567ad1dcccd1cc0fe0
+size 2479133