BashirEmran commited on
Commit
951ce90
·
verified ·
1 Parent(s): 99543f3

scorevision: push artifact

Browse files
Files changed (1) hide show
  1. miner.py +104 -1
miner.py CHANGED
@@ -8,7 +8,12 @@ Repo layout:
8
  miner.py this file
9
  weights.onnx detector, input [1,3,S,S] RGB float 0..1, output [1, 4+3, N]
10
  (cx, cy, w, h in input pixels, then per-class sigmoid scores)
11
- config.json {"conf": [cup, can, bottle], "nms_iou": 0.5, "max_det": 100, "gate": 0.0}
 
 
 
 
 
12
 
13
  No `from __future__ import annotations` here: pydantic must see real types when the
14
  chute imports this file dynamically.
@@ -56,6 +61,86 @@ def _nms(boxes: ndarray, scores: ndarray, iou: float) -> list[int]:
56
  return keep
57
 
58
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
59
  class Miner:
60
  def __init__(self, path_hf_repo: Path) -> None:
61
  path_hf_repo = Path(path_hf_repo)
@@ -68,6 +153,7 @@ class Miner:
68
  # per class: if no box passes conf, keep the class's best candidate when it scores >= rescue
69
  # (1.0 = off). map50 averages over the classes in an image, so the first box of a class is worth most.
70
  self.rescue = np.array(self.cfg.get("rescue", [1.0, 1.0, 1.0]), dtype=np.float32)
 
71
  so = ort.SessionOptions()
72
  so.intra_op_num_threads = 2
73
  so.inter_op_num_threads = 1
@@ -121,7 +207,24 @@ class Miner:
121
  keep = sorted(keep, key=lambda i: -conf[i])
122
  return boxes[keep], conf[keep], cls[keep]
123
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
124
  def _detect(self, image: ndarray) -> list[BoundingBox]:
 
 
125
  boxes, conf, cls = self.candidates(image, floor=float(min(self.conf.min(), self.rescue.min(), 0.05)))
126
  if conf.size == 0 or float(conf.max()) < self.gate:
127
  return []
 
8
  miner.py this file
9
  weights.onnx detector, input [1,3,S,S] RGB float 0..1, output [1, 4+3, N]
10
  (cx, cy, w, h in input pixels, then per-class sigmoid scores)
11
+ config.json {"conf": [cup, can, bottle], "nms_iou": 0.5, "max_det": 100, "gate": 0.0, "rescue": [...],
12
+ "esm": {"w": 3x3, "miss": 3x2, "draws": 256, "kmax": 24, "seed": 0}}
13
+
14
+ Which boxes are sent: with an "esm" block in config.json, the boxes that maximise the EXPECTED validator score
15
+ (class _Esm below; numbers fitted on our own model by tune_esm.py). Without it, per-class conf thresholds plus
16
+ the first-box rescue.
17
 
18
  No `from __future__ import annotations` here: pydantic must see real types when the
19
  chute imports this file dynamically.
 
61
  return keep
62
 
63
 
64
+ class _Esm:
65
+ """Expected-score-maximising selection (same maths as esm_core.py, kept here so the repo is one file).
66
+
67
+ The validator scores one image as max(0.6*map50 + 0.4*max(0, 1 - FP/10) - THETA, 0.01), map50 = mean AP over
68
+ the classes that have ground truth. We do not know which candidates are right, so `draws` fixed scenarios are
69
+ drawn: candidate j is right with probability P_j = sigmoid(w . [1, logit(conf), log10(area share) + 3]) and
70
+ the class has Poisson(m0 + m1 * sum P) more objects that no candidate found. For "send the top k of each
71
+ class" the formula is evaluated in every scenario and k is chosen by coordinate ascent on the average.
72
+ The random numbers are fixed at start-up: the same image always gives the same boxes.
73
+ """
74
+ THETA = 0.362454704407324514
75
+
76
+ def __init__(self, cfg: dict) -> None:
77
+ self.w = np.array(cfg["w"], dtype=np.float64)
78
+ self.miss = np.array(cfg["miss"], dtype=np.float64)
79
+ self.draws = int(cfg.get("draws", 256))
80
+ self.kmax = int(cfg.get("kmax", 24))
81
+ rs = np.random.RandomState(int(cfg.get("seed", 0)))
82
+ self.U = rs.random_sample((3, self.draws, self.kmax))
83
+ self.UM = np.stack([(rs.permutation(self.draws) + 0.5) / self.draws for _ in range(3)])
84
+ self.tri = np.triu(np.ones((self.kmax, self.kmax)))
85
+ self.rank = np.arange(1, self.kmax + 1)
86
+ self.logfact = np.concatenate([[0.0], np.cumsum(np.log(np.arange(1, 81)))])
87
+ self.m = np.arange(81)
88
+
89
+ def prob(self, cls: ndarray, conf: ndarray, area: ndarray) -> ndarray:
90
+ c = np.clip(np.asarray(conf, dtype=np.float64), 1e-4, 1 - 1e-4)
91
+ a = np.log10(np.maximum(np.asarray(area, dtype=np.float64), 1e-6)) + 3.0
92
+ w = self.w[cls]
93
+ return 1.0 / (1.0 + np.exp(-(w[:, 0] + w[:, 1] * np.log(c / (1 - c)) + w[:, 2] * a)))
94
+
95
+ def select(self, plist: list) -> list:
96
+ """plist: per class, P(right) of its candidates in sending order (at most kmax). Returns k per class."""
97
+ S = self.draws
98
+ ctps, aps, pres = [], [], np.zeros((S, 3), dtype=bool)
99
+ for c, p in enumerate(plist):
100
+ K = len(p)
101
+ t = self.U[c, :, :K] < p[None, :]
102
+ lam = max(float(self.miss[c, 0] + self.miss[c, 1] * p.sum()), 1e-3)
103
+ cdf = np.cumsum(np.exp(-lam + self.m * np.log(lam) - self.logfact))
104
+ n = t.sum(1) + np.minimum(np.searchsorted(cdf, self.UM[c], side="left"), 80)
105
+ ctp = np.zeros((S, K + 1))
106
+ ap = np.zeros((S, K + 1))
107
+ if K:
108
+ tf = t.astype(np.float64)
109
+ cum = np.cumsum(tf, 1)
110
+ ctp[:, 1:] = cum
111
+ prec = cum / self.rank[:K]
112
+ env = np.maximum.accumulate(prec[:, None, :] * self.tri[:K, :K], axis=2) # AP precision envelope
113
+ ap[:, 1:] = np.einsum("sj,sjk->sk", tf, env) / np.maximum(n, 1)[:, None]
114
+ ctps.append(ctp)
115
+ aps.append(ap)
116
+ pres[:, c] = n > 0
117
+ npres = pres.sum(1)
118
+ den = np.maximum(npres, 1)
119
+ k = [int((p >= 0.5).sum()) for p in plist]
120
+ for _ in range(4):
121
+ changed = False
122
+ for c in range(3):
123
+ oth_ap = np.zeros(S)
124
+ oth_fp = np.zeros(S)
125
+ for d in range(3):
126
+ if d != c:
127
+ oth_ap += pres[:, d] * aps[d][:, k[d]]
128
+ oth_fp += k[d] - ctps[d][:, k[d]]
129
+ K = aps[c].shape[1] - 1
130
+ apm = (oth_ap[:, None] + pres[:, c:c + 1] * aps[c]) / den[:, None]
131
+ fp = oth_fp[:, None] + np.arange(K + 1)[None, :] - ctps[c]
132
+ fps = np.maximum(0.0, 1.0 - fp / 10.0)
133
+ tot = np.where(npres[:, None] > 0, 0.6 * apm + 0.4 * fps, fps)
134
+ ev = np.maximum(tot - self.THETA, 0.01).mean(0)
135
+ b = int(np.argmax(ev))
136
+ if ev[b] > ev[k[c]] + 1e-12:
137
+ k[c] = b
138
+ changed = True
139
+ if not changed:
140
+ break
141
+ return k
142
+
143
+
144
  class Miner:
145
  def __init__(self, path_hf_repo: Path) -> None:
146
  path_hf_repo = Path(path_hf_repo)
 
153
  # per class: if no box passes conf, keep the class's best candidate when it scores >= rescue
154
  # (1.0 = off). map50 averages over the classes in an image, so the first box of a class is worth most.
155
  self.rescue = np.array(self.cfg.get("rescue", [1.0, 1.0, 1.0]), dtype=np.float32)
156
+ self.esm = _Esm(self.cfg["esm"]) if self.cfg.get("esm") else None
157
  so = ort.SessionOptions()
158
  so.intra_op_num_threads = 2
159
  so.inter_op_num_threads = 1
 
207
  keep = sorted(keep, key=lambda i: -conf[i])
208
  return boxes[keep], conf[keep], cls[keep]
209
 
210
+ def _detect_esm(self, image: ndarray) -> list[BoundingBox]:
211
+ boxes, conf, cls = self.candidates(image, floor=0.05)
212
+ q = np.round(boxes).astype(int)
213
+ ok = (q[:, 2] > q[:, 0]) & (q[:, 3] > q[:, 1])
214
+ q, conf, cls = q[ok], conf[ok].astype(np.float64), cls[ok]
215
+ if not len(q):
216
+ return []
217
+ h, w = image.shape[:2]
218
+ p = self.esm.prob(cls, conf, (q[:, 2] - q[:, 0]) * (q[:, 3] - q[:, 1]) / float(h * w))
219
+ idxs = [np.flatnonzero(cls == c)[: self.esm.kmax] for c in range(3)] # candidates are sorted by score
220
+ k = self.esm.select([p[i] for i in idxs])
221
+ out = [(q[i], c, conf[i]) for c in range(3) for i in idxs[c][: k[c]]][: self.max_det]
222
+ return [BoundingBox(x1=int(a[0]), y1=int(a[1]), x2=int(a[2]), y2=int(a[3]), cls_id=int(c), conf=float(f))
223
+ for a, c, f in out]
224
+
225
  def _detect(self, image: ndarray) -> list[BoundingBox]:
226
+ if self.esm is not None:
227
+ return self._detect_esm(image)
228
  boxes, conf, cls = self.candidates(image, floor=float(min(self.conf.min(), self.rescue.min(), 0.05)))
229
  if conf.size == 0 or float(conf.max()) < self.gate:
230
  return []