Duplicate from mlboydaisuke/RF-DETR-CoreAI

Co-authored-by: Majima <mlboydaisuke@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,53 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+demo_coco_252219.jpg filter=lfs diff=lfs merge=lfs -text
+demo_coco_cats.jpg filter=lfs diff=lfs merge=lfs -text
+rfdetr-large_float32.aimodel/main.mlirb filter=lfs diff=lfs merge=lfs -text
+rfdetr-medium_float32.aimodel/main.mlirb filter=lfs diff=lfs merge=lfs -text
+rfdetr-nano_float32.aimodel/main.mlirb filter=lfs diff=lfs merge=lfs -text
+rfdetr-small_float32.aimodel/main.mlirb filter=lfs diff=lfs merge=lfs -text
+split/rfdetr-nano_backbone.aimodel/main.mlirb filter=lfs diff=lfs merge=lfs -text
+split/rfdetr-nano_head.aimodel/main.mlirb filter=lfs diff=lfs merge=lfs -text
+split/rfdetr-medium_backbone.aimodel/main.mlirb filter=lfs diff=lfs merge=lfs -text
+split/rfdetr-medium_head.aimodel/main.mlirb filter=lfs diff=lfs merge=lfs -text
+rfdetr-seg-nano_float32.aimodel/main.mlirb filter=lfs diff=lfs merge=lfs -text
+rfdetr-seg-small_float32.aimodel/main.mlirb filter=lfs diff=lfs merge=lfs -text
+rfdetr-seg-medium_float32.aimodel/main.mlirb filter=lfs diff=lfs merge=lfs -text
+rfdetr-seg-large_float32.aimodel/main.mlirb filter=lfs diff=lfs merge=lfs -text
+rfdetr-seg-xlarge_float32.aimodel/main.mlirb filter=lfs diff=lfs merge=lfs -text
+rfdetr-seg-2xlarge_float32.aimodel/main.mlirb filter=lfs diff=lfs merge=lfs -text
+demo_seg_coco_cats.jpg filter=lfs diff=lfs merge=lfs -text
+demo_seg_coco_252219.jpg filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,151 @@
+---
+license: apache-2.0
+base_model: roboflow/rf-detr
+tags:
+- coreai
+- aimodel
+- object-detection
+- rf-detr
+- detr
+- apple
+- ios
+- macos
+pipeline_tag: object-detection
+---
+
+# RF-DETR — Core AI (`.aimodel`)
+
+[RF-DETR](https://github.com/roboflow/rf-detr) (Roboflow's real-time detection
+transformer, COCO-pretrained) converted to Apple **Core AI** for iOS 27 / macOS 27 —
+the answer to [apple/coreai-models#14](https://github.com/apple/coreai-models/issues/14).
+**DETR family = no NMS**: post-processing is one sigmoid + top-k.
+
+<p align="center"><img src="demo_coco_cats.jpg" width="440" alt="RF-DETR medium on Core AI"></p>
+
+<!-- gen-cards:use-it begin id=rf-detr (managed by scripts/gen-cards — edit cards.json / QuickStart.swift, not this block) -->
+## Use it
+
+⚡ **One line** — this model is the default behind the kit's task op
+(`import CoreAIOps`; no session, no model plumbing, downloads on first use):
+
+```swift
+let boxes = try await CoreAI.detect(inImageAt: url)
+```
+
+Twenty ops, one shape — [Cookbook](https://github.com/john-rocky/coreai-kit/blob/main/docs/COOKBOOK.md).
+
+▶️ **Run it (source)** — the [DetectCamera runner](https://github.com/john-rocky/coreai-kit/tree/main/Examples/DetectCamera)
+(real-time object detection on the zero-copy camera path):
+
+```bash
+git clone https://github.com/john-rocky/coreai-kit
+open coreai-kit/Examples/DetectCamera/DetectCamera.xcodeproj
+# → Run, then pick "Nano" in the model picker
+
+# agents / headless (macOS):
+cd coreai-kit/Examples/DetectCamera
+swift run detect-cli --model rf-detr --image Resources/gate_image.jpg
+```
+
+💻 **Build with it** — complete; the glue is kit API, copy-paste runs:
+
+```swift
+import CoreAIKitVision
+
+let detector = try await KitDetector(catalog: "rf-detr")
+let image = try ImageFile.load(imageURL)  // any image file → CGImage + EXIF orientation
+let detections = try await detector.detect(in: image.cgImage)
+// detections: [Detection] — label, score, normalized box (top-left origin)
+```
+
+The take-home is [`Examples/DetectCamera/Sources/QuickStart.swift`](https://github.com/john-rocky/coreai-kit/blob/main/Examples/DetectCamera/Sources/QuickStart.swift)
+— this exact code as one typed function, no UI; the CLI is an argument shell over it, and
+the GUI runs the same detector per camera frame on a zero-copy pixel-buffer fast path.
+Real time? Use `detect(in: CVPixelBuffer)` — vImage scales the frame with no CGImage
+round-trip; `CameraFeed` (kit API) streams the buffers.
+
+**Integration checklist**
+
+- SPM: `https://github.com/john-rocky/coreai-kit` → product **CoreAIKitVision**
+- Info.plist: `NSCameraUsageDescription` — only for the live camera; the snippet needs none
+- Entitlements: none needed
+- First run downloads the model — 0.1 GB (Mac) / 0.1 GB (iPhone) — then it loads from the
+  local cache (Application Support; progress via the `downloadProgress` callback)
+- Measure in Release — Debug is ~3× slower on per-token host work
+<!-- gen-cards:use-it end -->
+
+## Files
+
+| file | input | params | M4 Max GPU | iPhone 17 Pro GPU |
+|---|---|---|---|---|
+| `rfdetr-nano_float32.aimodel` | 384×384 | 30.5M | **8.6 ms** (~116 FPS) | **~25 ms (33–39 FPS live)** |
+| `rfdetr-small_float32.aimodel` | 512×512 | 32.1M | **12.0 ms** (~83 FPS) | — |
+| `rfdetr-medium_float32.aimodel` | 576×576 | 33.7M | **14.8 ms** (~68 FPS) | **56–63 ms (15–17 FPS live)** |
+| `rfdetr-large_float32.aimodel` | 704×704 | 33.9M | **19.1 ms** (~52 FPS) | — |
+
+iPhone numbers are end-to-end live-camera measurements from the
+[CoreAIKit DetectCamera example](https://github.com/john-rocky/coreai-kit)
+(Release; zero-copy capture pipeline — AVCaptureVideoPreviewLayer display,
+hardware-scaled 32BGRA buffers, vImage preprocessing overlapped with GPU
+inference). Peak measured 39.6 FPS ≈ the nano model ceiling; sustained
+max-load throughput drops on a hot chassis (thermal).
+
+fp32 is the ship dtype: it gates **detection-set exact** vs the PyTorch fp32 reference on
+CPU and GPU (per confident detection: same class, IoU ≥ 0.999 measured, score within 2e-3),
+and fp16 only bought ~7% latency on M4 Max while adding near-tie ranking noise.
+
+## Graph contract
+
+```
+input  "image"  [1, 3, R, R]  float32, RGB in [0, 1]  (ImageNet mean/std folded in-graph)
+output "dets"   [1, 300, 4]   boxes, cxcywh normalized to [0, 1]
+output "labels" [1, 300, 91]  raw class logits; column index = ORIGINAL COCO id (0 unused, 1=person … 17=cat … 90)
+```
+
+Python decode sketch (Swift is the same three steps):
+
+```python
+import numpy as np, coreai.runtime as rt
+
+model = await rt.AIModel.load(path, rt.SpecializationOptions.default())
+fn = model.load_function("main")
+out = await fn({"image": rt.NDArray(rgb01)})          # rgb01: [1,3,R,R] in [0,1]
+prob = 1 / (1 + np.exp(-out["labels"].numpy()[0]))    # [300, 91]
+scores, classes = prob.max(-1), prob.argmax(-1)       # column index IS the COCO id
+boxes = out["dets"].numpy()[0]                        # cxcywh, multiply by image W/H
+keep = scores > 0.5                                   # done — no NMS
+```
+
+## RF-DETR-Seg (instance segmentation)
+
+`rfdetr-seg-{nano,small,medium,large,xlarge,2xlarge}_float32.aimodel` — same
+contract plus `masks [1, Q, R/4, R/4]`: per-query FULL-FRAME logit planes at
+stride 4 (host: sigmoid > 0.5; no ROI plumbing, no NMS). All six gate on CPU
+and GPU with binary-mask IoU 1.000 on stable scenes. M4 Max GPU:
+seg-nano 312² **10.7 ms** → seg-2xlarge 768² **59.1 ms**.
+
+<p align="center"><img src="demo_seg_coco_cats.jpg" width="440" alt="RF-DETR-Seg nano on Core AI"></p>
+
+## Split deployment (`split/`)
+
+`split/rfdetr-{nano,medium}_{backbone,head}.aimodel` separate the pure-ViT
+backbone (image → features) from the deformable head (features → dets/labels;
+position encodings baked in). The chain is bit-exact vs the monolith. Purpose:
+per-stage compute-unit preferences — e.g. backbone on the Neural Engine.
+Measured honestly: on iOS 27 beta the runtime still executes the backbone on
+the GPU delegate even under `.neuralEngine` preference (identical detection
+fingerprint, no ANE-compile pause), so today the monolith on GPU is the
+fastest config; the split exists so ANE placement can be adopted the moment
+the runtime honors it. Regenerate with `export_rf_detr.py --variant <v> --split`.
+
+## Conversion
+
+Exported with
+[`conversion/export_rf_detr.py`](https://github.com/john-rocky/coreai-model-zoo/blob/main/conversion/export_rf_detr.py)
+from `rfdetr==1.7.1` weights. The port surfaced four Core AI converter/runtime bugs
+(float-arg `arange` abort, int64-comparison buffer clobber, GPU-delegate
+floor/trunc/ceil = identity, cast-pair cancellation) — each worked around numerically
+identically; details and minimal repros in
+[zoo/rf-detr.md](https://github.com/john-rocky/coreai-model-zoo/blob/main/zoo/rf-detr.md).
+
+License: Apache-2.0 (upstream RF-DETR code and COCO-pretrained weights are Apache-2.0).

config.json

@@ -0,0 +1,7 @@
+{
+  "model_type": "coreai-aimodel",
+  "format": "aimodel",
+  "framework": "Apple Core AI (iOS 27 / macOS 27)",
+  "repository": "https://github.com/john-rocky/coreai-model-zoo",
+  "note": "Converted .aimodel bundles for Apple's Core AI framework. See README.md for the bundle layout and run instructions."
+}

rfdetr-large_float32.aimodel/main.hash

@@ -0,0 +1 @@
+<���tu*rR� ܔz!}�E*���]��^\�	

rfdetr-large_float32.aimodel/main.mlirb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c88ac8874752a72527fa520dc947a217df3452a83c713955dffef1e5e5cdc09
+size 121824493

rfdetr-large_float32.aimodel/metadata.json

@@ -0,0 +1,5 @@
+{
+  "assetVersion" : "2.0",
+  "creationDate" : "20260721T040850Z",
+  "producer" : "coreai-core 1.0.0b2"
+}

rfdetr-medium_float32.aimodel/main.hash

@@ -0,0 +1 @@
+�M�y
,N�Nmv���wϗcV{^|,p�1c�

rfdetr-medium_float32.aimodel/main.mlirb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:954da57901172c4e02904e6d147689b58977cf9763567b5e7c182c70ab3163bc
+size 120828626

rfdetr-medium_float32.aimodel/metadata.json

@@ -0,0 +1,5 @@
+{
+  "assetVersion" : "2.0",
+  "producer" : "coreai-core 1.0.0b2",
+  "creationDate" : "20260721T040909Z"
+}

rfdetr-nano_float32.aimodel/main.hash

@@ -0,0 +1 @@
+�L.~�0�e3�9��n�C�!nR�`0�M;X�f

rfdetr-nano_float32.aimodel/main.mlirb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05834c2e7ea730b5650e339139b4806eb443f8216e5297603012da4d3b58d566
+size 107900898

rfdetr-nano_float32.aimodel/metadata.json

@@ -0,0 +1,5 @@
+{
+  "producer" : "coreai-core 1.0.0b2",
+  "assetVersion" : "2.0",
+  "creationDate" : "20260721T040929Z"
+}

rfdetr-seg-2xlarge_float32.aimodel/main.hash

@@ -0,0 +1 @@
+��#G-U�wդY��zq����b�W��8 �_�(

rfdetr-seg-2xlarge_float32.aimodel/main.mlirb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:84be23472d55ce77d5a459b0bb7a71dccb0ce3fa62ba57d3d8382005bb5fec28
+size 140790395

rfdetr-seg-2xlarge_float32.aimodel/metadata.json

@@ -0,0 +1,5 @@
+{
+  "producer" : "coreai-core 1.0.0b2",
+  "creationDate" : "20260721T040949Z",
+  "assetVersion" : "2.0"
+}

rfdetr-seg-large_float32.aimodel/main.hash

@@ -0,0 +1 @@
+T`ae:��t����~�*ü`�`�j�.k�,�

rfdetr-seg-large_float32.aimodel/main.mlirb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:546061653ab1ec181074ba9f8edf7ed10f2ac3bc609a60e96af82e6bd42c0ef3
+size 130758014

rfdetr-seg-large_float32.aimodel/metadata.json

@@ -0,0 +1,5 @@
+{
+  "producer" : "coreai-core 1.0.0b2",
+  "creationDate" : "20260721T041008Z",
+  "assetVersion" : "2.0"
+}

rfdetr-seg-medium_float32.aimodel/main.hash

@@ -0,0 +1 @@
+�-+���A�T�;�=d�G��a��c�r�
��9Z

rfdetr-seg-medium_float32.aimodel/main.mlirb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f12d2bedd9e2418354b83bc73d64a34798f361cdff63ed72e501f2eba5395a0e
+size 130029780

rfdetr-seg-medium_float32.aimodel/metadata.json

@@ -0,0 +1,5 @@
+{
+  "creationDate" : "20260721T041027Z",
+  "producer" : "coreai-core 1.0.0b2",
+  "assetVersion" : "2.0"
+}

rfdetr-seg-nano_float32.aimodel/main.hash

@@ -0,0 +1 @@
+�TD�W�{l�Y��^F��9���?���t2c�

rfdetr-seg-nano_float32.aimodel/main.mlirb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4540444c057bf7b6cc4598af35e46b2da399512c5ee023fafbdfd743263f2ba
+size 122673489

rfdetr-seg-nano_float32.aimodel/metadata.json

@@ -0,0 +1,5 @@
+{
+  "assetVersion" : "2.0",
+  "creationDate" : "20260721T041044Z",
+  "producer" : "coreai-core 1.0.0b2"
+}

rfdetr-seg-small_float32.aimodel/main.hash

@@ -0,0 +1 @@
+0��()5U��gpVC���ܳ9�jGa��B�n?

rfdetr-seg-small_float32.aimodel/main.mlirb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:30d5e228293555a7d067701e5643b5e3a1dadcb339f3116a4761d9db42896e3f
+size 123219375

rfdetr-seg-small_float32.aimodel/metadata.json

@@ -0,0 +1,5 @@
+{
+  "assetVersion" : "2.0",
+  "producer" : "coreai-core 1.0.0b2",
+  "creationDate" : "20260721T041103Z"
+}

rfdetr-seg-xlarge_float32.aimodel/main.hash

@@ -0,0 +1 @@
+Ƨ��F�3�&�g��m�t.�JW���V�s

rfdetr-seg-xlarge_float32.aimodel/main.mlirb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6a78fd60f4603e133c626c3671b831dd26d9f742ead4a571aaa0ffcec56bc73
+size 138618865

rfdetr-seg-xlarge_float32.aimodel/metadata.json

@@ -0,0 +1,5 @@
+{
+  "creationDate" : "20260721T041122Z",
+  "producer" : "coreai-core 1.0.0b2",
+  "assetVersion" : "2.0"
+}

rfdetr-small_float32.aimodel/main.hash

@@ -0,0 +1 @@
+\����N�=U#�O[ʪ$�%s�j�ltP6

rfdetr-small_float32.aimodel/main.mlirb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:175ce6dbcbf1954e0e9c3d5523cc4f5bcaaa24a42516738f6af39d6c74501b36
+size 114501696

rfdetr-small_float32.aimodel/metadata.json

@@ -0,0 +1,5 @@
+{
+  "creationDate" : "20260721T041140Z",
+  "assetVersion" : "2.0",
+  "producer" : "coreai-core 1.0.0b2"
+}

split/rfdetr-medium_backbone.aimodel/main.mlirb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:441c32abfaf12581721bbf4e959e63cd2b9871850009c39b9d9cdd1d7b3c8782
+size 94244255

split/rfdetr-medium_backbone.aimodel/metadata.json

@@ -0,0 +1,5 @@
+{
+  "assetVersion" : "2.0",
+  "producer" : "coreai-core 1.0.0b2",
+  "creationDate" : "20260721T041156Z"
+}

split/rfdetr-medium_head.aimodel/main.hash

@@ -0,0 +1 @@
+���.+J����Aᆏ�#�s��(��|�y��z=

split/rfdetr-medium_head.aimodel/main.mlirb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fde4c12e2b4acce608c8d841e1868fb023fc738fc328fbf17cc379851eba7a3d
+size 26584890

split/rfdetr-medium_head.aimodel/metadata.json

@@ -0,0 +1,5 @@
+{
+  "producer" : "coreai-core 1.0.0b2",
+  "creationDate" : "20260721T041209Z",
+  "assetVersion" : "2.0"
+}

split/rfdetr-nano_backbone.aimodel/main.hash

@@ -0,0 +1,3 @@
+KT�f*�0�y�
+��X�
+(^_)&��F LI6

split/rfdetr-nano_backbone.aimodel/main.mlirb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b54c7662a8a301b80798c0aa7cb58b60a285e5f29261385e946204c4936101a
+size 93138329

split/rfdetr-nano_backbone.aimodel/metadata.json

@@ -0,0 +1,5 @@
+{
+  "assetVersion" : "2.0",
+  "producer" : "coreai-core 1.0.0b2",
+  "creationDate" : "20260721T041220Z"
+}

split/rfdetr-nano_head.aimodel/main.hash

@@ -0,0 +1,2 @@
+[s����t����?�Se�1���+R4�
+Ž���

split/rfdetr-nano_head.aimodel/main.mlirb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b73edbcaaeb74b784c0e03fb6530c65a93197aebb2b5234ff1f0ac5bdddf7b9
+size 14763257

split/rfdetr-nano_head.aimodel/metadata.json

@@ -0,0 +1,5 @@
+{
+  "creationDate" : "20260721T041232Z",
+  "producer" : "coreai-core 1.0.0b2",
+  "assetVersion" : "2.0"
+}