Cofiber Threshold: trained weights, COCO mAP 4.0 from 70K params, eval script

eval_coco_map.py

@@ -0,0 +1,198 @@
+"""
+Evaluate a trained detection head on COCO val2017 using pycocotools mAP.
+
+Usage:
+    python eval_coco_map.py --checkpoint outputs/cofiber_threshold_full/head_final.pth --head cofiber_threshold
+"""
+
+import argparse
+import json
+import os
+import sys
+import time
+
+import numpy as np
+import torch
+import torch.nn.functional as F
+from PIL import Image
+from torchvision.transforms import v2
+
+sys.path.insert(0, os.path.dirname(__file__))
+
+EUPE_REPO = os.environ.get("ARENA_BACKBONE_REPO", "/home/zootest/EUPE")
+EUPE_WEIGHTS = os.environ.get("ARENA_BACKBONE_WEIGHTS", "/home/zootest/weights/eupe_vitb/EUPE-ViT-B.pt")
+COCO_ROOT = os.environ.get("ARENA_COCO_ROOT", "/mnt/d/JacobProject/datasets/llava_instruct/coco")
+RESOLUTION = 640
+
+if EUPE_REPO not in sys.path:
+    sys.path.insert(0, EUPE_REPO)
+
+COCO_CONTIG_TO_CAT = [
+    1,2,3,4,5,6,7,8,9,10,11,13,14,15,16,17,18,19,20,21,22,23,24,25,27,28,31,32,
+    33,34,35,36,37,38,39,40,41,42,43,44,46,47,48,49,50,51,52,53,54,55,56,57,58,
+    59,60,61,62,63,64,65,67,70,72,73,74,75,76,77,78,79,80,81,82,84,85,86,87,88,89,90,
+]
+
+
+def letterbox(image, res):
+    W0, H0 = image.size
+    scale = res / max(H0, W0)
+    new_w, new_h = int(round(W0 * scale)), int(round(H0 * scale))
+    resized = image.resize((new_w, new_h), Image.BILINEAR)
+    canvas = Image.new("RGB", (res, res), (0, 0, 0))
+    canvas.paste(resized, (0, 0))
+    return canvas, scale
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--checkpoint", required=True)
+    parser.add_argument("--head", default="cofiber_threshold")
+    parser.add_argument("--score-thresh", type=float, default=0.05)
+    parser.add_argument("--max-images", type=int, default=5000)
+    args = parser.parse_args()
+
+    from pycocotools.coco import COCO
+    from pycocotools.cocoeval import COCOeval
+
+    print("=" * 60)
+    print(f"COCO mAP Evaluation: {args.head}")
+    print("=" * 60)
+
+    # Load backbone
+    print("\nLoading backbone...")
+    backbone = torch.hub.load(EUPE_REPO, "eupe_vitb16", source="local", weights=EUPE_WEIGHTS)
+    backbone = backbone.cuda().eval()
+    for p in backbone.parameters():
+        p.requires_grad = False
+
+    # Load head
+    print(f"Loading head: {args.head}")
+    from heads import get_head
+    head = get_head(args.head)
+    state_dict = torch.load(args.checkpoint, map_location="cuda", weights_only=False)
+    if "head" in state_dict:
+        state_dict = state_dict["head"]
+    head.load_state_dict(state_dict)
+    head = head.cuda().eval()
+    n_params = sum(p.numel() for p in head.parameters())
+    print(f"  {n_params:,} params")
+
+    # Precompute locations
+    with torch.no_grad():
+        dummy = torch.randn(1, 768, RESOLUTION // 16, RESOLUTION // 16, device="cuda")
+        locs = head.get_locs(dummy)
+
+    # Load COCO val
+    ann_file = os.path.join(COCO_ROOT, "annotations", "instances_val2017.json")
+    img_dir = os.path.join(COCO_ROOT, "val2017")
+    coco_gt = COCO(ann_file)
+    img_ids = sorted(coco_gt.getImgIds())[:args.max_images]
+    print(f"  {len(img_ids)} val images")
+
+    normalize = v2.Compose([
+        v2.ToImage(), v2.ToDtype(torch.float32, scale=True),
+        v2.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),
+    ])
+
+    # Run inference
+    print("\nRunning inference...")
+    results = []
+    t0 = time.time()
+
+    for i, img_id in enumerate(img_ids):
+        info = coco_gt.loadImgs(img_id)[0]
+        img = Image.open(os.path.join(img_dir, info["file_name"])).convert("RGB")
+        W0, H0 = img.size
+        canvas, scale = letterbox(img, RESOLUTION)
+        x = normalize(canvas).unsqueeze(0).cuda()
+
+        with torch.no_grad():
+            with torch.autocast("cuda", dtype=torch.bfloat16):
+                out = backbone.forward_features(x)
+            patches = out["x_norm_patchtokens"].float()
+            B, N, D = patches.shape
+            h = w = int(N ** 0.5)
+            spatial = patches.permute(0, 2, 1).reshape(B, D, h, w)
+
+            cls_l, reg_l, ctr_l = head(spatial)
+
+        # Decode
+        from utils.decode import decode_fcos
+        dets = decode_fcos(cls_l, reg_l, ctr_l, locs,
+                           score_thresh=args.score_thresh, nms_thresh=0.5, max_det=100)
+
+        for det in dets:
+            boxes = det["boxes"].cpu().numpy() / scale
+            boxes[:, 0::2] = boxes[:, 0::2].clip(0, W0)
+            boxes[:, 1::2] = boxes[:, 1::2].clip(0, H0)
+            scores = det["scores"].cpu().numpy()
+            labels = det["labels"].cpu().numpy()
+
+            for box, score, label in zip(boxes, scores, labels):
+                x1, y1, x2, y2 = box
+                results.append({
+                    "image_id": img_id,
+                    "category_id": COCO_CONTIG_TO_CAT[int(label)],
+                    "bbox": [float(x1), float(y1), float(x2 - x1), float(y2 - y1)],
+                    "score": float(score),
+                })
+
+        if (i + 1) % 500 == 0:
+            elapsed = time.time() - t0
+            print(f"  {i+1}/{len(img_ids)} ({elapsed:.0f}s, {(i+1)/elapsed:.1f} img/s)", flush=True)
+
+    elapsed = time.time() - t0
+    print(f"\nInference complete: {len(img_ids)} images, {len(results)} detections, {elapsed:.0f}s")
+
+    # Save results
+    results_file = args.checkpoint.replace(".pth", "_coco_results.json")
+    with open(results_file, "w") as f:
+        json.dump(results, f)
+    print(f"Saved: {results_file}")
+
+    # Evaluate
+    if len(results) == 0:
+        print("\nNo detections produced. mAP = 0.0")
+        return
+
+    print("\nRunning pycocotools evaluation...")
+    coco_dt = coco_gt.loadRes(results_file)
+    coco_eval = COCOeval(coco_gt, coco_dt, "bbox")
+    coco_eval.params.imgIds = img_ids
+    coco_eval.evaluate()
+    coco_eval.accumulate()
+    coco_eval.summarize()
+
+    # Save summary
+    summary = {
+        "head": args.head,
+        "params": n_params,
+        "checkpoint": args.checkpoint,
+        "n_images": len(img_ids),
+        "n_detections": len(results),
+        "mAP_0.5_0.95": float(coco_eval.stats[0]),
+        "mAP_0.50": float(coco_eval.stats[1]),
+        "mAP_0.75": float(coco_eval.stats[2]),
+        "mAP_small": float(coco_eval.stats[3]),
+        "mAP_medium": float(coco_eval.stats[4]),
+        "mAP_large": float(coco_eval.stats[5]),
+    }
+    summary_file = args.checkpoint.replace(".pth", "_coco_summary.json")
+    with open(summary_file, "w") as f:
+        json.dump(summary, f, indent=2)
+    print(f"\nSaved: {summary_file}")
+
+    print(f"\n{'='*60}")
+    print(f"  {args.head}: {n_params:,} params")
+    print(f"  mAP@[0.5:0.95] = {summary['mAP_0.5_0.95']:.1f}")
+    print(f"  mAP@0.50       = {summary['mAP_0.50']:.1f}")
+    print(f"  mAP@0.75       = {summary['mAP_0.75']:.1f}")
+    print(f"  mAP small      = {summary['mAP_small']:.1f}")
+    print(f"  mAP medium     = {summary['mAP_medium']:.1f}")
+    print(f"  mAP large      = {summary['mAP_large']:.1f}")
+    print(f"{'='*60}")
+
+
+if __name__ == "__main__":
+    main()

heads/cofiber_threshold/README.md

@@ -1,6 +1,6 @@
 # Cofiber Threshold
 
-Adjoint cofiber decomposition + per-scale LayerNorm + prototype classification. 65,000 parameters.
+Adjoint cofiber decomposition + per-scale LayerNorm + prototype classification. 69,976 parameters.
 
 ## Architecture
 
@@ -11,7 +11,7 @@ Scale decomposition (0 learned params):
   cofiber_k = f_k - upsample(avgpool(f_k))    for k = 0, 1
   residual = avgpool(avgpool(f))               for k = 2
 
-Per-scale prediction (65K learned params):
+Per-scale prediction (~70K learned params):
   For each scale k:
     f = LayerNorm(cofiber_k)
     cls = f @ prototypes.T + bias              (80 classes)
@@ -23,13 +23,37 @@ The cofiber `x - upsample(pool(x))` isolates information present at a given spat
 
 ## Results
 
+### COCO val2017 (pycocotools, 5000 images)
+
+Trained on COCO 2017 train (117,266 images), 8 epochs, batch 64, AdamW lr 1e-3, cosine schedule with 3% warmup. Frozen EUPE-ViT-B backbone.
+
+| Metric | Cofiber Threshold (70K) | Baseline FCOS (16.14M) |
+|--------|------------------------|----------------------|
+| mAP@[0.5:0.95] | 4.0 | 41.0 |
+| mAP@0.50 | 15.8 | 64.8 |
+| mAP@0.75 | 0.8 | 43.2 |
+| mAP small | 1.3 | 21.4 |
+| mAP medium | 4.1 | 44.9 |
+| mAP large | 6.3 | 62.1 |
+| AR@100 | 14.9 | — |
+
+The mAP@0.50 of 15.8 indicates the head locates objects at the correct class; the collapse at mAP@0.75 (0.8) indicates the single-layer box regression cannot produce tight bounding boxes.
+
+### Cross-domain screening (21 domains, COCO + RF100-VL)
+
 | Protocol | Domains | Avg Precision | Avg Recall | Total TP |
 |----------|---------|---------------|------------|----------|
 | 2K steps screening | 21 | 0.475 | 0.193 | 478 |
 | 15K steps extended | 21 | 0.617 | 0.368 | 719 |
 
-Exceeds Baseline FCOS (16.14M params, 0.470 avg precision) at 230x fewer parameters.
+Exceeds Baseline FCOS (16.14M params, 0.470 avg precision) at 230x fewer parameters on the cross-domain screening protocol.
+
+## Files
+
+- `head.py` — architecture implementation
+- `head_final.pth` — trained weights (COCO 2017, 8 epochs)
+- `coco_eval.json` — pycocotools evaluation summary
 
 ## Threshold circuit form
 
-The entire head can be expressed as a depth-3 threshold gate network with 2,184,000 gates. Layers 0-1 use fixed integer weights {-1, 0, 1}. Layer 2 uses INT8 quantized prototypes. See `phanerozoic/threshold-cofiber-detection` for the serialized circuit.
+The head can be expressed as a depth-3 threshold gate network with 2,184,000 gates. Layers 0-1 use fixed integer weights {-1, 0, 1}. Layer 2 uses INT8 quantized prototypes (99.7% detection agreement with FP32). See `phanerozoic/threshold-cofiber-detection` for the serialized circuit.

heads/cofiber_threshold/coco_eval.json

@@ -0,0 +1,13 @@
+{
+  "head": "cofiber_threshold",
+  "params": 69976,
+  "checkpoint": "outputs/cofiber_threshold_full/head_final.pth",
+  "n_images": 5000,
+  "n_detections": 499925,
+  "mAP_0.5_0.95": 0.04049779195869424,
+  "mAP_0.50": 0.15815283266196423,
+  "mAP_0.75": 0.007915213278509007,
+  "mAP_small": 0.012886382966430599,
+  "mAP_medium": 0.04094513118407366,
+  "mAP_large": 0.06285424921271325
+}

heads/cofiber_threshold/head_final.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b260a0fae7ec7a86bb38f3e087700f7c5ec171a7cd16ef0141f04478363001a8
+size 284981

outputs/cofiber_threshold_full/checkpoint.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f9af2408fa926b66c5cb2eb0ac2eabdc89895129c54d69a15f32bd7fd60621c6
+oid sha256:8423ca32fe6517e771d1ec7f009ffa8e58952e5be5ec172d0b227ade1bf01517
 size 284981

outputs/cofiber_threshold_full/head_final.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b260a0fae7ec7a86bb38f3e087700f7c5ec171a7cd16ef0141f04478363001a8
+size 284981

outputs/cofiber_threshold_full/head_final_coco_summary.json

@@ -0,0 +1,13 @@
+{
+  "head": "cofiber_threshold",
+  "params": 69976,
+  "checkpoint": "outputs/cofiber_threshold_full/head_final.pth",
+  "n_images": 5000,
+  "n_detections": 499925,
+  "mAP_0.5_0.95": 0.04049779195869424,
+  "mAP_0.50": 0.15815283266196423,
+  "mAP_0.75": 0.007915213278509007,
+  "mAP_small": 0.012886382966430599,
+  "mAP_medium": 0.04094513118407366,
+  "mAP_large": 0.06285424921271325
+}