scorevision: push artifact
Browse files
miner.py
CHANGED
|
@@ -8,7 +8,12 @@ Repo layout:
|
|
| 8 |
miner.py this file
|
| 9 |
weights.onnx detector, input [1,3,S,S] RGB float 0..1, output [1, 4+3, N]
|
| 10 |
(cx, cy, w, h in input pixels, then per-class sigmoid scores)
|
| 11 |
-
config.json {"conf": [cup, can, bottle], "nms_iou": 0.5, "max_det": 100, "gate": 0.0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 12 |
|
| 13 |
No `from __future__ import annotations` here: pydantic must see real types when the
|
| 14 |
chute imports this file dynamically.
|
|
@@ -56,6 +61,86 @@ def _nms(boxes: ndarray, scores: ndarray, iou: float) -> list[int]:
|
|
| 56 |
return keep
|
| 57 |
|
| 58 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 59 |
class Miner:
|
| 60 |
def __init__(self, path_hf_repo: Path) -> None:
|
| 61 |
path_hf_repo = Path(path_hf_repo)
|
|
@@ -68,6 +153,7 @@ class Miner:
|
|
| 68 |
# per class: if no box passes conf, keep the class's best candidate when it scores >= rescue
|
| 69 |
# (1.0 = off). map50 averages over the classes in an image, so the first box of a class is worth most.
|
| 70 |
self.rescue = np.array(self.cfg.get("rescue", [1.0, 1.0, 1.0]), dtype=np.float32)
|
|
|
|
| 71 |
so = ort.SessionOptions()
|
| 72 |
so.intra_op_num_threads = 2
|
| 73 |
so.inter_op_num_threads = 1
|
|
@@ -121,7 +207,24 @@ class Miner:
|
|
| 121 |
keep = sorted(keep, key=lambda i: -conf[i])
|
| 122 |
return boxes[keep], conf[keep], cls[keep]
|
| 123 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 124 |
def _detect(self, image: ndarray) -> list[BoundingBox]:
|
|
|
|
|
|
|
| 125 |
boxes, conf, cls = self.candidates(image, floor=float(min(self.conf.min(), self.rescue.min(), 0.05)))
|
| 126 |
if conf.size == 0 or float(conf.max()) < self.gate:
|
| 127 |
return []
|
|
|
|
| 8 |
miner.py this file
|
| 9 |
weights.onnx detector, input [1,3,S,S] RGB float 0..1, output [1, 4+3, N]
|
| 10 |
(cx, cy, w, h in input pixels, then per-class sigmoid scores)
|
| 11 |
+
config.json {"conf": [cup, can, bottle], "nms_iou": 0.5, "max_det": 100, "gate": 0.0, "rescue": [...],
|
| 12 |
+
"esm": {"w": 3x3, "miss": 3x2, "draws": 256, "kmax": 24, "seed": 0}}
|
| 13 |
+
|
| 14 |
+
Which boxes are sent: with an "esm" block in config.json, the boxes that maximise the EXPECTED validator score
|
| 15 |
+
(class _Esm below; numbers fitted on our own model by tune_esm.py). Without it, per-class conf thresholds plus
|
| 16 |
+
the first-box rescue.
|
| 17 |
|
| 18 |
No `from __future__ import annotations` here: pydantic must see real types when the
|
| 19 |
chute imports this file dynamically.
|
|
|
|
| 61 |
return keep
|
| 62 |
|
| 63 |
|
| 64 |
+
class _Esm:
|
| 65 |
+
"""Expected-score-maximising selection (same maths as esm_core.py, kept here so the repo is one file).
|
| 66 |
+
|
| 67 |
+
The validator scores one image as max(0.6*map50 + 0.4*max(0, 1 - FP/10) - THETA, 0.01), map50 = mean AP over
|
| 68 |
+
the classes that have ground truth. We do not know which candidates are right, so `draws` fixed scenarios are
|
| 69 |
+
drawn: candidate j is right with probability P_j = sigmoid(w . [1, logit(conf), log10(area share) + 3]) and
|
| 70 |
+
the class has Poisson(m0 + m1 * sum P) more objects that no candidate found. For "send the top k of each
|
| 71 |
+
class" the formula is evaluated in every scenario and k is chosen by coordinate ascent on the average.
|
| 72 |
+
The random numbers are fixed at start-up: the same image always gives the same boxes.
|
| 73 |
+
"""
|
| 74 |
+
THETA = 0.362454704407324514
|
| 75 |
+
|
| 76 |
+
def __init__(self, cfg: dict) -> None:
|
| 77 |
+
self.w = np.array(cfg["w"], dtype=np.float64)
|
| 78 |
+
self.miss = np.array(cfg["miss"], dtype=np.float64)
|
| 79 |
+
self.draws = int(cfg.get("draws", 256))
|
| 80 |
+
self.kmax = int(cfg.get("kmax", 24))
|
| 81 |
+
rs = np.random.RandomState(int(cfg.get("seed", 0)))
|
| 82 |
+
self.U = rs.random_sample((3, self.draws, self.kmax))
|
| 83 |
+
self.UM = np.stack([(rs.permutation(self.draws) + 0.5) / self.draws for _ in range(3)])
|
| 84 |
+
self.tri = np.triu(np.ones((self.kmax, self.kmax)))
|
| 85 |
+
self.rank = np.arange(1, self.kmax + 1)
|
| 86 |
+
self.logfact = np.concatenate([[0.0], np.cumsum(np.log(np.arange(1, 81)))])
|
| 87 |
+
self.m = np.arange(81)
|
| 88 |
+
|
| 89 |
+
def prob(self, cls: ndarray, conf: ndarray, area: ndarray) -> ndarray:
|
| 90 |
+
c = np.clip(np.asarray(conf, dtype=np.float64), 1e-4, 1 - 1e-4)
|
| 91 |
+
a = np.log10(np.maximum(np.asarray(area, dtype=np.float64), 1e-6)) + 3.0
|
| 92 |
+
w = self.w[cls]
|
| 93 |
+
return 1.0 / (1.0 + np.exp(-(w[:, 0] + w[:, 1] * np.log(c / (1 - c)) + w[:, 2] * a)))
|
| 94 |
+
|
| 95 |
+
def select(self, plist: list) -> list:
|
| 96 |
+
"""plist: per class, P(right) of its candidates in sending order (at most kmax). Returns k per class."""
|
| 97 |
+
S = self.draws
|
| 98 |
+
ctps, aps, pres = [], [], np.zeros((S, 3), dtype=bool)
|
| 99 |
+
for c, p in enumerate(plist):
|
| 100 |
+
K = len(p)
|
| 101 |
+
t = self.U[c, :, :K] < p[None, :]
|
| 102 |
+
lam = max(float(self.miss[c, 0] + self.miss[c, 1] * p.sum()), 1e-3)
|
| 103 |
+
cdf = np.cumsum(np.exp(-lam + self.m * np.log(lam) - self.logfact))
|
| 104 |
+
n = t.sum(1) + np.minimum(np.searchsorted(cdf, self.UM[c], side="left"), 80)
|
| 105 |
+
ctp = np.zeros((S, K + 1))
|
| 106 |
+
ap = np.zeros((S, K + 1))
|
| 107 |
+
if K:
|
| 108 |
+
tf = t.astype(np.float64)
|
| 109 |
+
cum = np.cumsum(tf, 1)
|
| 110 |
+
ctp[:, 1:] = cum
|
| 111 |
+
prec = cum / self.rank[:K]
|
| 112 |
+
env = np.maximum.accumulate(prec[:, None, :] * self.tri[:K, :K], axis=2) # AP precision envelope
|
| 113 |
+
ap[:, 1:] = np.einsum("sj,sjk->sk", tf, env) / np.maximum(n, 1)[:, None]
|
| 114 |
+
ctps.append(ctp)
|
| 115 |
+
aps.append(ap)
|
| 116 |
+
pres[:, c] = n > 0
|
| 117 |
+
npres = pres.sum(1)
|
| 118 |
+
den = np.maximum(npres, 1)
|
| 119 |
+
k = [int((p >= 0.5).sum()) for p in plist]
|
| 120 |
+
for _ in range(4):
|
| 121 |
+
changed = False
|
| 122 |
+
for c in range(3):
|
| 123 |
+
oth_ap = np.zeros(S)
|
| 124 |
+
oth_fp = np.zeros(S)
|
| 125 |
+
for d in range(3):
|
| 126 |
+
if d != c:
|
| 127 |
+
oth_ap += pres[:, d] * aps[d][:, k[d]]
|
| 128 |
+
oth_fp += k[d] - ctps[d][:, k[d]]
|
| 129 |
+
K = aps[c].shape[1] - 1
|
| 130 |
+
apm = (oth_ap[:, None] + pres[:, c:c + 1] * aps[c]) / den[:, None]
|
| 131 |
+
fp = oth_fp[:, None] + np.arange(K + 1)[None, :] - ctps[c]
|
| 132 |
+
fps = np.maximum(0.0, 1.0 - fp / 10.0)
|
| 133 |
+
tot = np.where(npres[:, None] > 0, 0.6 * apm + 0.4 * fps, fps)
|
| 134 |
+
ev = np.maximum(tot - self.THETA, 0.01).mean(0)
|
| 135 |
+
b = int(np.argmax(ev))
|
| 136 |
+
if ev[b] > ev[k[c]] + 1e-12:
|
| 137 |
+
k[c] = b
|
| 138 |
+
changed = True
|
| 139 |
+
if not changed:
|
| 140 |
+
break
|
| 141 |
+
return k
|
| 142 |
+
|
| 143 |
+
|
| 144 |
class Miner:
|
| 145 |
def __init__(self, path_hf_repo: Path) -> None:
|
| 146 |
path_hf_repo = Path(path_hf_repo)
|
|
|
|
| 153 |
# per class: if no box passes conf, keep the class's best candidate when it scores >= rescue
|
| 154 |
# (1.0 = off). map50 averages over the classes in an image, so the first box of a class is worth most.
|
| 155 |
self.rescue = np.array(self.cfg.get("rescue", [1.0, 1.0, 1.0]), dtype=np.float32)
|
| 156 |
+
self.esm = _Esm(self.cfg["esm"]) if self.cfg.get("esm") else None
|
| 157 |
so = ort.SessionOptions()
|
| 158 |
so.intra_op_num_threads = 2
|
| 159 |
so.inter_op_num_threads = 1
|
|
|
|
| 207 |
keep = sorted(keep, key=lambda i: -conf[i])
|
| 208 |
return boxes[keep], conf[keep], cls[keep]
|
| 209 |
|
| 210 |
+
def _detect_esm(self, image: ndarray) -> list[BoundingBox]:
|
| 211 |
+
boxes, conf, cls = self.candidates(image, floor=0.05)
|
| 212 |
+
q = np.round(boxes).astype(int)
|
| 213 |
+
ok = (q[:, 2] > q[:, 0]) & (q[:, 3] > q[:, 1])
|
| 214 |
+
q, conf, cls = q[ok], conf[ok].astype(np.float64), cls[ok]
|
| 215 |
+
if not len(q):
|
| 216 |
+
return []
|
| 217 |
+
h, w = image.shape[:2]
|
| 218 |
+
p = self.esm.prob(cls, conf, (q[:, 2] - q[:, 0]) * (q[:, 3] - q[:, 1]) / float(h * w))
|
| 219 |
+
idxs = [np.flatnonzero(cls == c)[: self.esm.kmax] for c in range(3)] # candidates are sorted by score
|
| 220 |
+
k = self.esm.select([p[i] for i in idxs])
|
| 221 |
+
out = [(q[i], c, conf[i]) for c in range(3) for i in idxs[c][: k[c]]][: self.max_det]
|
| 222 |
+
return [BoundingBox(x1=int(a[0]), y1=int(a[1]), x2=int(a[2]), y2=int(a[3]), cls_id=int(c), conf=float(f))
|
| 223 |
+
for a, c, f in out]
|
| 224 |
+
|
| 225 |
def _detect(self, image: ndarray) -> list[BoundingBox]:
|
| 226 |
+
if self.esm is not None:
|
| 227 |
+
return self._detect_esm(image)
|
| 228 |
boxes, conf, cls = self.candidates(image, floor=float(min(self.conf.min(), self.rescue.min(), 0.05)))
|
| 229 |
if conf.size == 0 or float(conf.max()) < self.gate:
|
| 230 |
return []
|