audit/clip_detector.json

{
 "detector": "openai/clip-vit-large-patch14 image tower, 224 center crop, max cosine over 140 bbox-cropped query crops",
 "calibration": {
  "statistic": "max cosine over all 140 bbox-cropped query descriptors",
  "n_pos_images": 3439,
  "n_neg_images": 11315,
  "pos_mean": 0.8742,
  "pos_median": 0.8932,
  "neg_mean": 0.7568,
  "neg_p95": 0.8888,
  "tau": {
   "fpr5.0": 0.8888,
   "fpr1.0": 0.9145,
   "fpr0.5": 0.9233,
   "fpr0.1": 0.943
  },
  "tpr_at_tau": {
   "fpr5.0": 0.5196,
   "fpr1.0": 0.3955,
   "fpr0.5": 0.3472,
   "fpr0.1": 0.2268
  }
 },
 "tau": 0.9145,
 "tpr": 0.3955,
 "fpr": 0.01,
 "ours": {
  "n_fetched": 2529,
  "n_total": 2529,
  "clip": {
   "n_matched": 489,
   "n_total": 2529,
   "raw_precision": 0.1934,
   "raw_ci95": [
    0.1784,
    0.2092
   ],
   "corrected_precision": 0.4756,
   "corrected_ci95": [
    0.4369,
    0.5168
   ]
  },
  "dinov2_same_subset": {
   "n_matched": 757,
   "n_total": 2529,
   "raw_precision": 0.2993,
   "raw_ci95": [
    0.2818,
    0.3175
   ],
   "corrected_precision": 0.4255,
   "corrected_ci95": [
    0.3997,
    0.4522
   ]
  },
  "agreement": {
   "pearson_r": 0.824,
   "spearman_r": 0.841,
   "binary_agreement": 0.8521,
   "cohens_kappa": 0.6077,
   "both_positive": 436,
   "clip_only": 53,
   "dinov2_only": 321,
   "both_negative": 1719
  }
 },
 "rgld": {
  "n_fetched": 1565,
  "n_total": 1565,
  "clip": {
   "n_matched": 538,
   "n_total": 1565,
   "raw_precision": 0.3438,
   "raw_ci95": [
    0.3206,
    0.3677
   ],
   "corrected_precision": 0.8658,
   "corrected_ci95": [
    0.8058,
    0.9278
   ]
  },
  "dinov2_same_subset": {
   "n_matched": 870,
   "n_total": 1565,
   "raw_precision": 0.5559,
   "raw_ci95": [
    0.5312,
    0.5804
   ],
   "corrected_precision": 0.8028,
   "corrected_ci95": [
    0.7665,
    0.8388
   ]
  },
  "agreement": {
   "pearson_r": 0.6999,
   "spearman_r": 0.7121,
   "binary_agreement": 0.7252,
   "cohens_kappa": 0.469,
   "both_positive": 489,
   "clip_only": 49,
   "dinov2_only": 381,
   "both_negative": 646
  }
 },
 "ours_per_rule": {
  "unique": {
   "n_fetched": 1586,
   "n_total": 1586,
   "clip": {
    "n_matched": 428,
    "n_total": 1586,
    "raw_precision": 0.2699,
    "raw_ci95": [
     0.2486,
     0.2922
    ],
    "corrected_precision": 0.6741,
    "corrected_ci95": [
     0.6189,
     0.7322
    ]
   },
   "dinov2_same_subset": {
    "n_matched": 644,
    "n_total": 1586,
    "raw_precision": 0.4061,
    "raw_ci95": [
     0.3821,
     0.4304
    ],
    "corrected_precision": 0.5824,
    "corrected_ci95": [
     0.5473,
     0.6183
    ]
   },
   "agreement": {
    "pearson_r": 0.8103,
    "spearman_r": 0.8066,
    "binary_agreement": 0.8033,
    "cohens_kappa": 0.5693,
    "both_positive": 380,
    "clip_only": 48,
    "dinov2_only": 264,
    "both_negative": 894
   }
  },
  "blanket": {
   "n_fetched": 727,
   "n_total": 727,
   "clip": {
    "n_matched": 55,
    "n_total": 727,
    "raw_precision": 0.0757,
    "raw_ci95": [
     0.0586,
     0.0972
    ],
    "corrected_precision": 0.1703,
    "corrected_ci95": [
     0.126,
     0.2262
    ]
   },
   "dinov2_same_subset": {
    "n_matched": 97,
    "n_total": 727,
    "raw_precision": 0.1334,
    "raw_ci95": [
     0.1106,
     0.1601
    ],
    "corrected_precision": 0.1815,
    "corrected_ci95": [
     0.148,
     0.2207
    ]
   },
   "agreement": {
    "pearson_r": 0.8227,
    "spearman_r": 0.7813,
    "binary_agreement": 0.934,
    "cohens_kappa": 0.6505,
    "both_positive": 52,
    "clip_only": 3,
    "dinov2_only": 45,
    "both_negative": 627
   }
  },
  "ctx": {
   "n_fetched": 216,
   "n_total": 216,
   "clip": {
    "n_matched": 6,
    "n_total": 216,
    "raw_precision": 0.0278,
    "raw_ci95": [
     0.0128,
     0.0593
    ],
    "corrected_precision": 0.0461,
    "corrected_ci95": [
     0.0072,
     0.1278
    ]
   },
   "dinov2_same_subset": {
    "n_matched": 16,
    "n_total": 216,
    "raw_precision": 0.0741,
    "raw_ci95": [
     0.0461,
     0.1169
    ],
    "corrected_precision": 0.0942,
    "corrected_ci95": [
     0.0531,
     0.1572
    ]
   },
   "agreement": {
    "pearson_r": 0.7212,
    "spearman_r": 0.7484,
    "binary_agreement": 0.9352,
    "cohens_kappa": 0.3368,
    "both_positive": 4,
    "clip_only": 2,
    "dinov2_only": 12,
    "both_negative": 198
   }
  }
 }
}