first commit

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.axmodel filter=lfs diff=lfs merge=lfs -text

AX650/pld_650_npu3.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a25c77a90f7cafa42105a7abdfa5bccfc086c61e6a52db5790e4323d2a26f170
+size 1198468

AX650/plr_650_npu3.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef96528b9b350d59ba35c282b61eee29ee9650a9e0e642db4e73ec43f001a69d
+size 955647

README.md

@@ -0,0 +1,141 @@
+---
+license: agpl-3.0
+language:
+- en
+pipeline_tag: object-detection
+tags:
+- Axera
+- License Plate Recognition
+- NPU
+- OCR
+- Object Detection
+---
+
+# plate-axera
+
+This version of **plate-axera** has been converted to run on the Axera NPU using **w8a16** quantization. There are two models included:
+1. **pld_650_npu3.axmodel**: This model is trained to detect the license plate with label 'plate'.
+2. **plr_650_npu3.axmodel**: This model is trained to recognize the characters of detected license plates, and also shows the color of the license plate.
+
+## Supported Classes
+Detection model supports the following classes:
+1. **plate**
+
+## Supported characters
+Recognition model supports the following characters:
+```
+{"皖沪津渝冀晋蒙辽吉黑苏浙京闽赣鲁豫鄂湘粤桂琼川贵云藏陕甘青宁新警学港澳台使领挂OABCDEFGHJKLMNPQRSTUVWXYZ0123456789"}
+```
+## Supported colors
+Recognition model supports the following colors:
+```
+['blue', 'green', 'yellow', 'white', 'black']
+```
+
+Compatible with Pulsar2 version: 5.2.
+
+## Convert tools links:
+
+For those who are interested in model conversion, you can try to export axmodel through:
+- [The repo of AXera Platform](https://github.com/AXERA-TECH/ax-samples), where you can get the detailed guide.
+- [Pulsar2 Link, How to Convert ONNX to axmodel](https://pulsar2-docs.readthedocs.io/en/latest/pulsar2/introduction.html)
+
+## Support Platform
+
+https://docs.m5stack.com/zh_CN/ai_hardware/AI_Pyramid-Pro
+
+- **AX650N/AX8850**
+  - [M4N-Dock(爱芯派Pro)](https://wiki.sipeed.com/hardware/zh/maixIV/m4ndock/m4ndock.html)
+  - [AI Pyramid](https://docs.m5stack.com/zh_CN/ai_hardware/AI_Pyramid-Pro)
+  - [M.2 Accelerator card](https://docs.m5stack.com/en/ai_hardware/LLM-8850_Card)
+
+## How to use
+
+Download all files from this repository to the device.
+
+### python env requirement
+
+#### pyaxengine
+
+https://github.com/AXERA-TECH/pyaxengine
+
+```bash
+wget https://github.com/AXERA-TECH/pyaxengine/releases/download/0.1.3.rc2/axengine-0.1.3-py3-none-any.whl
+pip install axengine-0.1.3-py3-none-any.whl
+```
+
+### Inference with AX650 Host, such as M4N-Dock(爱芯派Pro)
+
+#### Plate Detection
+Input image:
+![](test.jpg)
+
+run
+```bash
+python3 axmodel_infer_pld.py
+```
+
+```bash
+root@ax650:~/plate-axera# python3 axmodel_infer_pld.py
+[INFO] Available providers:  ['AxEngineExecutionProvider', 'AXCLRTExecutionProvider']
+[INFO] Using provider: AxEngineExecutionProvider
+[INFO] Chip type: ChipType.MC50
+[INFO] VNPU type: VNPUType.DISABLED
+[INFO] Engine version: 2.12.0s
+[INFO] Model type: 2 (triple core)
+[INFO] Compiler version: 5.2 eccb31f5
+class: plate left:597 top:417 right:759 bottom:475 conf: 88%
+
+```
+Output image:
+![](det_res.jpg)
+
+#### Plate Recognition
+Input image:
+![](苏A8A68Y.jpg)
+
+run
+```bash
+python3 axmodel_infer_plr.py
+```
+
+```bash
+root@ax650:~/plate-axera# python3 axmodel_infer_plr.py
+[INFO] Available providers:  ['AxEngineExecutionProvider', 'AXCLRTExecutionProvider']
+[INFO] Using provider: AxEngineExecutionProvider
+[INFO] Chip type: ChipType.MC50
+[INFO] VNPU type: VNPUType.DISABLED
+[INFO] Engine version: 2.12.0s
+[INFO] Model type: 2 (triple core)
+[INFO] Compiler version: 5.2 eccb31f5
+Plate: [苏A8A68Y], score: 0.9997, color: [blue], score:1.0000
+
+```
+#### Plate det & Rec End2End
+Input image:
+![](test.jpg)
+
+run
+```bash
+python3 axmodel_infer_plate_end2end.py
+```
+
+```bash
+root@ax650:~/plate-axera# python3 axmodel_infer_plate_end2end.py
+[INFO] Available providers:  ['AxEngineExecutionProvider', 'AXCLRTExecutionProvider']
+[INFO] Using provider: AxEngineExecutionProvider
+[INFO] Chip type: ChipType.MC50
+[INFO] VNPU type: VNPUType.DISABLED
+[INFO] Engine version: 2.12.0s
+[INFO] Model type: 2 (triple core)
+[INFO] Compiler version: 5.2 eccb31f5
+[INFO] Using provider: AxEngineExecutionProvider
+[INFO] Model type: 2 (triple core)
+[INFO] Compiler version: 5.2 eccb31f5
+Det---class:[plate], bbox:[597,417,759,475], conf:0.88
+Rec---Plate:[川A2E7V7], score:0.9991, color:[blue], score:1.0000
+Result saved to: ./plate_end2end_res.jpg
+
+```
+Output image:
+![](plate_end2end_res.jpg)

axmodel_infer_plate_end2end.py

@@ -0,0 +1,309 @@
+import cv2
+import numpy as np
+import axengine as axe
+import argparse
+import matplotlib
+
+plate_colors=['blue', 'green', 'yellow', 'white', 'black']
+
+class Colors:
+    
+    def __init__(self):
+        self.palette = [self.hex2rgb(c) for c in matplotlib.colors.TABLEAU_COLORS.values()]
+        self.n = len(self.palette)
+
+    def __call__(self, i, bgr=False):
+        c = self.palette[int(i) % self.n]
+        return (c[2], c[1], c[0]) if bgr else c
+
+    @staticmethod
+    def hex2rgb(h):  
+        return tuple(int(h[1 + i:1 + i + 2], 16) for i in (0, 2, 4))
+    
+colors = Colors()
+
+def make_grid(nx=20, ny=20, i=0, strides=[8, 16, 32], anchors=[[31,28, 38,32, 60,83],[84,110, 133,118, 200,113]]):
+    y, x = np.arange(ny, dtype=np.int32), np.arange(nx, dtype=np.int32)
+    yv, xv = np.meshgrid(y, x, indexing="ij")
+    grid = np.stack((xv, yv), 2)
+    grid = np.expand_dims(grid, axis=0).repeat(len(anchors[0]) // 2, axis=0)
+    grid = np.expand_dims(grid, axis=0) - 0.5
+    anchor_grid = np.array(anchors[i]).reshape((1, len(anchors[0]) // 2, 1, 1, 2))
+    anchor_grid = anchor_grid.repeat(ny, axis=2).repeat(nx, axis=3)
+    return grid, anchor_grid
+
+def sigmoid(x):
+    return 1 / (1 + np.exp(-x))
+
+def softmax(lpr_pred):
+    max_out = np.max(lpr_pred, axis=1, keepdims=True)
+    exp_out = np.exp(lpr_pred - max_out)
+    sum_exp_out = np.sum(exp_out, axis=1, keepdims=True)
+    return exp_out / sum_exp_out
+
+def xywh2xyxy(x):
+    y = np.copy(x)
+    y[..., 0] = x[..., 0] - x[..., 2] / 2
+    y[..., 1] = x[..., 1] - x[..., 3] / 2
+    y[..., 2] = x[..., 0] + x[..., 2] / 2
+    y[..., 3] = x[..., 1] + x[..., 3] / 2
+    return y
+
+def letterbox(im, new_shape=(640, 640), color=(0, 0, 0), auto=False, scaleFill=False, scaleup=True, stride=32):
+    shape = im.shape[:2]
+    if isinstance(new_shape, int):
+        new_shape = (new_shape, new_shape)
+
+    r = min(new_shape[0] / shape[0], new_shape[1] / shape[1])
+    if not scaleup:
+        r = min(r, 1.0)
+
+    ratio = r, r
+    new_unpad = int(round(shape[1] * r)), int(round(shape[0] * r))
+    dw, dh = new_shape[1] - new_unpad[0], new_shape[0] - new_unpad[1]
+    if auto:
+        dw, dh = np.mod(dw, stride), np.mod(dh, stride)
+    elif scaleFill:
+        dw, dh = 0.0, 0.0
+        new_unpad = (new_shape[1], new_shape[0])
+        ratio = new_shape[1] / shape[1], new_shape[0] / shape[0]
+
+    dw /= 2
+    dh /= 2
+
+    if shape[::-1] != new_unpad:
+        im = cv2.resize(im, new_unpad, interpolation=cv2.INTER_LINEAR)
+
+    top, bottom = int(round(dh - 0.1)), int(round(dh + 0.1))
+    left, right = int(round(dw - 0.1)), int(round(dw + 0.1))
+    im = cv2.copyMakeBorder(im, top, bottom, left, right, cv2.BORDER_CONSTANT, value=color)
+
+    return im, ratio, (dw, dh)
+
+def scale_boxes(img1_shape, boxes, img0_shape, ratio_pad=None):
+    if ratio_pad is None:
+        gain = min(img1_shape[0] / img0_shape[0], img1_shape[1] / img0_shape[1])
+        pad = (img1_shape[1] - img0_shape[1] * gain) / 2, (img1_shape[0] - img0_shape[0] * gain) / 2
+    else:
+        gain = ratio_pad[0][0]
+        pad = ratio_pad[1]
+
+    boxes[..., [0, 2]] -= pad[0]
+    boxes[..., [1, 3]] -= pad[1]
+    boxes[..., :4] /= gain
+    return boxes
+
+def nms(boxes, iou_thresh=0.65):
+    xmin, ymin, xmax, ymax = boxes[:, 0], boxes[:, 1], boxes[:, 2], boxes[:, 3]
+    score = boxes[:, 4]
+    areas = (xmax - xmin + 1)*(ymax - ymin + 1)
+    order = score.argsort()[::-1]
+
+    keep = []
+    while order.size > 0:
+        i = order[0]
+        keep.append(i)
+
+        xxmin = np.maximum(xmin[i], xmin[order[1:]])
+        yymin = np.maximum(ymin[i], ymin[order[1:]])
+        xxmax = np.minimum(xmax[i], xmax[order[1:]])
+        yymax = np.minimum(ymax[i], ymax[order[1:]])
+
+        w = np.maximum(0, xxmax - xxmin + 1)
+        h = np.maximum(0, yymax - yymin + 1)
+        inter = w * h
+
+        iou = inter / (areas[i] + areas[order[1:]] - inter)
+        order = order[np.where(iou <= iou_thresh)[0] + 1]
+
+    return boxes[keep, :]
+
+def nms_multi(boxes, conf_thresh=0.25, iou_thresh=0.65, max_num=300):
+    if len(boxes) == 0:
+        return boxes
+    
+    boxes = boxes[np.where(boxes[:, 4] > conf_thresh)]
+    result = list()
+
+    cls_score = boxes[:, 5:]
+    max_cls_index = np.argmax(cls_score, axis=-1)
+    max_cls_score = np.max(cls_score, axis=-1)
+
+    dets = np.concatenate([boxes[:, :5], max_cls_score[:, np.newaxis], boxes[:, 4:5], max_cls_index[:, np.newaxis]], axis=-1)
+    dets[:, 6] = dets[:, 4] * dets[:, 5]
+    max_det = dets[:, 6].argsort()[::-1][:max_num]
+    dets = dets[max_det, :]
+
+    unique_label = np.unique(max_cls_index)
+
+    for c in unique_label:
+        det = dets[dets[:, -1] == c]
+        nmsed_det = nms(det, iou_thresh=iou_thresh)
+        if len(nmsed_det):
+            result.append(nmsed_det)
+
+    if len(result):
+        result = np.concatenate(result, axis=0)
+        return result
+    else:
+        return []
+
+def GreedyDecode(preb, SEP_IDX=74, plate_string=""):
+    preb_label = list()
+    for j in range(preb.shape[0]):
+        preb_label.append(int(np.argmax(preb[j, :], axis=0)))
+    no_repeat_blank_label = list()
+    plate_index = list()
+    pre_c = preb_label[0]
+    if pre_c != SEP_IDX:
+        no_repeat_blank_label.append(pre_c)
+        plate_index.append(0)
+    for idx, c in enumerate(preb_label):
+        if (pre_c == c) or (c == SEP_IDX):
+            if c == SEP_IDX:
+                pre_c = c
+            continue
+        no_repeat_blank_label.append(c)
+        plate_index.append(idx)
+        pre_c = c
+
+    lpr_string = ''.join([plate_string[idx] for idx in no_repeat_blank_label])
+    lpr_score = 1
+    for idx, c in zip(plate_index, no_repeat_blank_label):
+        lpr_score *= preb[idx, c]
+
+    return lpr_string, lpr_score
+
+def pld_inference(session_pld, input_name_pld, output_name_pld, img, opt):
+    img_letter, ratio, (dw, dh) = letterbox(img, opt.imgsz_pld)
+    input_data = np.expand_dims(img_letter, axis=0)[..., ::-1].transpose((0, 3, 1, 2)).astype(np.uint8)
+
+    outputs = session_pld.run(output_name_pld, {input_name_pld: input_data})
+
+    num_anchor = len(opt.anchors[0]) // 2
+    channel = len(opt.classes) + 5
+    predictions = list()
+
+    for i, output in enumerate(outputs):
+        bs, _, ny, nx = output.shape
+        output = sigmoid(output.reshape(bs, num_anchor, channel, ny, nx).transpose(0, 1, 3, 4, 2))
+
+        grid, anchor_grid = make_grid(nx, ny, i, opt.strides, opt.anchors)
+
+        xy, wh, conf = output[..., :2], output[..., 2:4], output[..., 4:]
+        
+        xy = (xy * 2 + grid) * opt.strides[i]
+        wh = (wh * 2) ** 2 * anchor_grid
+
+        prediction = np.concatenate((xy, wh, conf), 4)
+        prediction = prediction.reshape(bs, num_anchor * nx * ny, channel)
+
+        prediction = xywh2xyxy(prediction)
+        prediction[..., 0:4:2] = np.clip(prediction[..., 0:4:2], a_min=0, a_max=opt.imgsz_pld[1])
+        prediction[..., 1:4:2] = np.clip(prediction[..., 1:4:2], a_min=0, a_max=opt.imgsz_pld[0])
+
+        predictions.append(prediction)
+
+    predictions = np.concatenate(predictions, axis=1).squeeze()
+    predictions = nms_multi(predictions)
+    if len(predictions) > 0:
+        predictions[:, :4] = scale_boxes(img_letter.shape[:2], predictions[:, :4], img.shape).round()
+    
+    return predictions
+
+def plr_inference(session_plr, input_name_plr, output_name_plr, plate_img, opt):
+    img_resized = cv2.resize(plate_img, (opt.imgsz_plr[1], opt.imgsz_plr[0]))
+    input_data = np.expand_dims(img_resized, axis=0)[..., ::-1].transpose((0, 3, 1, 2)).astype(np.uint8)
+    outputs = session_plr.run(output_name_plr, {input_name_plr: input_data})
+
+    SEP_IDX = len(opt.PLATE_STRING) - 1
+    for lpr_out, color_out in zip(*outputs):
+
+        lpr_pred = np.max(lpr_out, axis=1).T
+        lpr_pred = softmax(lpr_pred)
+        lpr_string, lpr_score = GreedyDecode(lpr_pred, SEP_IDX, opt.PLATE_STRING)
+
+        color_out = np.max(np.max(color_out, axis=1), axis=1).reshape((1, -1))
+        color_out = softmax(color_out)[0]
+        color_label = color_out.argmax()
+        color_score  = color_out[color_label]
+
+    return lpr_string, lpr_score, color_label, color_score
+
+def end2end_inference(opt):
+
+    providers = ["AxEngineExecutionProvider"]
+    session_pld = axe.InferenceSession(opt.weights_pld, providers=providers)
+    input_name_pld = session_pld.get_inputs()[0].name
+    output_name_pld = [output.name for output in session_pld.get_outputs()]
+
+    session_plr = axe.InferenceSession(opt.weights_plr, providers=providers)
+    input_name_plr = session_plr.get_inputs()[0].name
+    output_name_plr = [output.name for output in session_plr.get_outputs()]
+
+    img = cv2.imread(opt.source)
+    if img is None:
+        print(f"Failed to read image: {opt.source}")
+        return
+
+    dets = pld_inference(session_pld, input_name_pld, output_name_pld, img, opt)
+
+    results = []
+    if len(dets) > 0:
+        box_xyxy = dets[:, :4].astype(np.int32)
+        scores = dets[:, -2]
+        labels = dets[:, -1].astype(np.int32)
+
+        for (x1, y1, x2, y2), score, label in zip(box_xyxy, scores, labels):
+            x1, y1, x2, y2 = max(0, x1), max(0, y1), min(img.shape[1], x2), min(img.shape[0], y2)
+            if x2 > x1 and y2 > y1:
+                plate_img = img[y1:y2, x1:x2]
+                # You can do some expand or alignment for plate_img if needed
+                lpr_string, lpr_score, color_label, color_score = plr_inference(session_plr, input_name_plr, output_name_plr, plate_img, opt)
+                results.append({
+                    'bbox': (x1, y1, x2, y2),
+                    'det_score': score,
+                    'plate': lpr_string,
+                    'plate_score': lpr_score,
+                    'color': plate_colors[color_label],
+                    'color_score': color_score
+                })
+
+                print(f"Det---class:[{opt.classes[label]}], bbox:[{x1},{y1},{x2},{y2}], conf:{score:.2f}")
+                print(f"Rec---Plate:[{lpr_string}], score:{lpr_score:.4f}, color:[{plate_colors[color_label]}], score:{color_score:.4f}")
+
+    if opt.vis and len(results) > 0:
+        for res in results:
+            x1, y1, x2, y2 = res['bbox']
+            tl = 3 or round(0.002 * (img.shape[0] + img.shape[1]) / 2) + 1
+            img = cv2.rectangle(img, (x1, y1), (x2, y2), colors(0, True), tl)
+            c1, c2 = (int(x1), int(y1)), (int(x2), int(y2))
+            tf = max(tl - 1, 1)
+            text = f"{res['plate']}:{res['plate_score']:.2f}"
+            t_size = cv2.getTextSize(text, 0, fontScale=tl / 6, thickness=tf)[0]
+            c2 = c1[0] + t_size[0], c1[1] - t_size[1] - 3
+            cv2.rectangle(img, c1, c2, colors(0, True), -1, cv2.LINE_AA)
+            cv2.putText(img, text, (c1[0], c1[1] - 2), 0, tl / 6, [225, 255, 255], thickness=tf//2, lineType=cv2.LINE_AA)
+        
+        cv2.imwrite(opt.save_name, img)
+        print(f"Result saved to: {opt.save_name}")
+
+def parse_opt():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--weights_pld", type=str, default="./pld_650_npu3.axmodel", help="plate detection model path")
+    parser.add_argument("--weights_plr", type=str, default="./plr_650_npu3.axmodel", help="plate recognition model path")
+    parser.add_argument("--source", type=str, default="./test.jpg", help="img_path")
+    parser.add_argument("--anchors", type=float, default=[[23, 8, 57, 21, 76, 28],[93, 33, 86, 41, 116, 39], [120, 90, 156, 198, 373, 326]], help="anchor based anchors")
+    parser.add_argument("--strides", type=float, default=[8, 16, 32], help="model strides")
+    parser.add_argument("--imgsz_pld", nargs="+", type=int, default=[416, 416], help="PLD inference size h,w")
+    parser.add_argument("--imgsz_plr", nargs="+", type=int, default=[48, 192], help="PLR inference size h,w")
+    parser.add_argument("--classes", type=str, default=["plate"], help="classes num")
+    parser.add_argument("--PLATE_STRING", type=str, default=u"皖沪津渝冀晋蒙辽吉黑苏浙京闽赣鲁豫鄂湘粤桂琼川贵云藏陕甘青宁新警学港澳台使领挂OABCDEFGHJKLMNPQRSTUVWXYZ0123456789#", help="Plate string map")
+    parser.add_argument("--vis", default=True, help="visualize detect result")
+    parser.add_argument("--save_name", type=str, default="./plate_end2end_res.jpg", help="result img save path")
+    opt = parser.parse_args()
+    return opt
+
+if __name__ == "__main__":
+    opt = parse_opt()
+    end2end_inference(opt)

axmodel_infer_pld.py

@@ -0,0 +1,233 @@
+import cv2
+import numpy as np
+import axengine as axe
+import argparse
+import matplotlib
+
+class Colors:
+    
+    def __init__(self):
+        self.palette = [self.hex2rgb(c) for c in matplotlib.colors.TABLEAU_COLORS.values()]
+        self.n = len(self.palette)
+
+    def __call__(self, i, bgr=False):
+        c = self.palette[int(i) % self.n]
+        return (c[2], c[1], c[0]) if bgr else c
+
+    @staticmethod
+    def hex2rgb(h):  
+        return tuple(int(h[1 + i:1 + i + 2], 16) for i in (0, 2, 4))
+    
+colors = Colors()
+def make_grid(nx=20, ny=20, i=0, strides=[8, 16, 32], anchors=[[31,28, 38,32, 60,83],[84,110, 133,118, 200,113]]):
+    """Generates a mesh grid for anchor boxes"""
+    # shape = 1, len(anchors[i]) // 2, ny, nx, 2  # grid shape
+    y, x = np.arange(ny, dtype=np.int32), np.arange(nx, dtype=np.int32)
+    yv, xv = np.meshgrid(y, x, indexing="ij")
+    grid = np.stack((xv, yv), 2)
+    grid = np.expand_dims(grid, axis=0).repeat(len(anchors[0]) // 2, axis=0)
+    grid = np.expand_dims(grid, axis=0) - 0.5 #add grid offset, i.e. y = 2.0 * x - 0.5
+    # anchor_grid = np.array([anchor*strides[i] for anchor in anchors[i]]).reshape((1, len(anchors[0]) // 2, 1, 1, 2))
+    anchor_grid = np.array(anchors[i]).reshape((1, len(anchors[0]) // 2, 1, 1, 2))
+    anchor_grid = anchor_grid.repeat(ny, axis=2).repeat(nx, axis=3)
+    # print(anchor_grid.shape, shape)
+    return grid, anchor_grid
+
+def sigmoid(x):
+    return 1 / (1 + np.exp(-x))
+
+def xywh2xyxy(x):
+    """Convert nx4 boxes from [x, y, w, h] to [x1, y1, x2, y2] where xy1=top-left, xy2=bottom-right."""
+    y = np.copy(x)
+    y[..., 0] = x[..., 0] - x[..., 2] / 2  # top left x
+    y[..., 1] = x[..., 1] - x[..., 3] / 2  # top left y
+    y[..., 2] = x[..., 0] + x[..., 2] / 2  # bottom right x
+    y[..., 3] = x[..., 1] + x[..., 3] / 2  # bottom right y
+    return y
+
+def letterbox(im, new_shape=(640, 640), color=(0, 0, 0), auto=False, scaleFill=False, scaleup=True, stride=32):
+    """Resizes and pads image to new_shape with stride-multiple constraints, returns resized image, ratio, padding."""
+    shape = im.shape[:2]  # current shape [height, width]
+    if isinstance(new_shape, int):
+        new_shape = (new_shape, new_shape)
+
+    # Scale ratio (new / old)
+    r = min(new_shape[0] / shape[0], new_shape[1] / shape[1])
+    if not scaleup:  # only scale down, do not scale up (for better val mAP)
+        r = min(r, 1.0)
+
+    # Compute padding
+    ratio = r, r  # width, height ratios
+    new_unpad = int(round(shape[1] * r)), int(round(shape[0] * r))
+    dw, dh = new_shape[1] - new_unpad[0], new_shape[0] - new_unpad[1]  # wh padding
+    if auto:  # minimum rectangle
+        dw, dh = np.mod(dw, stride), np.mod(dh, stride)  # wh padding
+    elif scaleFill:  # stretch
+        dw, dh = 0.0, 0.0
+        new_unpad = (new_shape[1], new_shape[0])
+        ratio = new_shape[1] / shape[1], new_shape[0] / shape[0]  # width, height ratios
+
+    dw /= 2  # divide padding into 2 sides
+    dh /= 2
+
+    if shape[::-1] != new_unpad:  # resize
+        im = cv2.resize(im, new_unpad, interpolation=cv2.INTER_LINEAR)
+
+    top, bottom = int(round(dh - 0.1)), int(round(dh + 0.1))
+    left, right = int(round(dw - 0.1)), int(round(dw + 0.1))
+    im = cv2.copyMakeBorder(im, top, bottom, left, right, cv2.BORDER_CONSTANT, value=color)  # add border for 2 sides
+    # im = cv2.copyMakeBorder(im, 0, int(dh), 0, int(dw), cv2.BORDER_CONSTANT, value=color)  # add border for right and bottom
+
+    return im, ratio, (dw, dh)
+
+def scale_boxes(img1_shape, boxes, img0_shape, ratio_pad=None):
+    """Rescales (xyxy) bounding boxes from img1_shape to img0_shape, optionally using provided `ratio_pad`."""
+    if ratio_pad is None:  # calculate from img0_shape
+        gain = min(img1_shape[0] / img0_shape[0], img1_shape[1] / img0_shape[1])  # gain  = old / new
+        pad = (img1_shape[1] - img0_shape[1] * gain) / 2, (img1_shape[0] - img0_shape[0] * gain) / 2  # wh padding
+        # pad = (0, 0)
+    else:
+        gain = ratio_pad[0][0]
+        pad = ratio_pad[1]
+
+    boxes[..., [0, 2]] -= pad[0]  # x padding
+    boxes[..., [1, 3]] -= pad[1]  # y padding
+    boxes[..., :4] /= gain
+    return boxes
+
+def nms(boxes, iou_thresh=0.65):
+    xmin, ymin, xmax, ymax = boxes[:, 0], boxes[:, 1], boxes[:, 2], boxes[:, 3]
+    score = boxes[:, 4]
+    areas = (xmax - xmin + 1)*(ymax - ymin + 1)
+    order = score.argsort()[::-1]
+
+    keep = []
+    while order.size > 0:
+        i = order[0]
+        keep.append(i)
+
+        xxmin = np.maximum(xmin[i], xmin[order[1:]])
+        yymin = np.maximum(ymin[i], ymin[order[1:]])
+        xxmax = np.minimum(xmax[i], xmax[order[1:]])
+        yymax = np.minimum(ymax[i], ymax[order[1:]])
+
+        w = np.maximum(0, xxmax - xxmin + 1)
+        h = np.maximum(0, yymax - yymin + 1)
+        inter = w * h
+
+        iou = inter / (areas[i] + areas[order[1:]] - inter)
+        order = order[np.where(iou <= iou_thresh)[0] + 1] #索引需要加1
+
+    return boxes[keep, :]
+
+def nms_multi(boxes, conf_thresh=0.25, iou_thresh=0.65, max_num=300):
+    if len(boxes) == 0:
+        return boxes
+    
+    boxes = boxes[np.where(boxes[:, 4] > conf_thresh)]
+    result = list()
+
+    cls_score = boxes[:, 5:]
+    max_cls_index = np.argmax(cls_score, axis=-1)
+    max_cls_score = np.max(cls_score, axis=-1)
+
+    dets = np.concatenate([boxes[:, :5], max_cls_score[:, np.newaxis], boxes[:, 4:5], max_cls_index[:, np.newaxis]], axis=-1)
+    dets[:, 6] = dets[:, 4] * dets[:, 5]
+    max_det = dets[:, 6].argsort()[::-1][:max_num]
+    dets = dets[max_det, :]
+
+    unique_label = np.unique(max_cls_index)
+
+    for c in unique_label:
+        det = dets[dets[:, -1] == c]
+        nmsed_det = nms(det, iou_thresh=iou_thresh)
+        if len(nmsed_det):
+            result.append(nmsed_det)
+
+    if len(result):
+        result = np.concatenate(result, axis=0)
+        return result
+    else:
+        return []
+
+def model_inference(opt):
+
+    providers = ["AxEngineExecutionProvider"]
+    session = axe.InferenceSession(opt.model, providers=providers)
+
+    input_name = session.get_inputs()[0].name
+    output_name = [output.name for output in session.get_outputs()]
+
+    img = cv2.imread(f'{opt.source}')
+    img_letter, ratio, (dw, dh) = letterbox(img, opt.imgsz)
+    input_data = np.expand_dims(img_letter, axis=0)[..., ::-1].transpose((0, 3, 1, 2)).astype(np.uint8)
+
+    outputs = session.run(output_name, {input_name:input_data})
+
+    num_anchor = len(opt.anchors[0]) // 2
+    channel = len(opt.classes) + 5
+    predictions = list()
+
+    for i, output in enumerate(outputs):
+        bs, _, ny, nx = output.shape  # x(bs,255,20,20) to x(bs,3,20,20,85)
+        output = sigmoid(output.reshape(bs, num_anchor, channel, ny, nx).transpose(0, 1, 3, 4, 2))
+
+        grid, anchor_grid = make_grid(nx, ny, i, opt.strides, opt.anchors)
+
+        xy, wh, conf = output[..., :2], output[..., 2:4], output[..., 4:]
+        
+        xy = (xy * 2 + grid) * opt.strides[i]  # xy
+        wh = (wh * 2) ** 2 * anchor_grid  # wh
+
+        prediction = np.concatenate((xy, wh, conf), 4)
+        prediction = prediction.reshape(bs, num_anchor * nx * ny, channel)
+
+        prediction = xywh2xyxy(prediction)
+        prediction[..., 0:4:2] = np.clip(prediction[..., 0:4:2], a_min=0, a_max=opt.imgsz[1])
+        prediction[..., 1:4:2] = np.clip(prediction[..., 1:4:2], a_min=0, a_max=opt.imgsz[0])
+
+        predictions.append(prediction)
+
+    predictions = np.concatenate(predictions, axis=1).squeeze()
+    # predictions format: [x1, y1, x2, y2, obj, cls_score, obj*cls_score, label]
+    predictions = nms_multi(predictions) #TODO multi label for one box
+    predictions[:, :4] = scale_boxes(img_letter.shape[:2], predictions[:, :4], img.shape).round()
+    
+    if opt.vis:
+        box_xyxy = predictions[:, :4].astype(np.int32)
+        scores = predictions[:, -2]
+        labels = predictions[:, -1].astype(np.int32)
+
+        for (x1, y1, x2, y2), score, label in zip(box_xyxy, scores, labels):
+            print("class:",opt.classes[label], "left:%.0f" % x1,"top:%.0f" % y1,"right:%.0f" % x2,"bottom:%.0f" % y2, "conf:",'{:.0f}%'.format(float(score)*100))
+            tl = 3 or round(0.002 * (img.shape[0] + img.shape[1]) / 2) + 1
+            img = cv2.rectangle(img, (x1, y1), (x2, y2), colors(label, True), tl)
+            c1, c2 = (int(x1), int(y1)), (int(x2), int(y2))
+            tf = max(tl - 1, 1)  
+            t_size = cv2.getTextSize(f"{opt.classes[label]}:{score:.3f}", 0, fontScale=tl / 6, thickness=tf)[0]
+            c2 = c1[0] + t_size[0], c1[1] - t_size[1] - 3
+            cv2.rectangle(img, c1, c2, colors(label, True), -1, cv2.LINE_AA)
+            cv2.putText(img, f"{opt.classes[label]}:{score:.3f}", (c1[0], c1[1] - 2), 0, tl / 6, [225, 255, 255], thickness=tf//2, lineType=cv2.LINE_AA)
+        
+        cv2.imwrite(f'{opt.save_name}', img)
+
+def parse_opt():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--model", nargs="+", type=str, default="./pld_650_npu3.axmodel", help="axmodel path")
+    parser.add_argument("--source", type=str, default="./test.jpg", help="img_path")
+    parser.add_argument("--anchors", type=float, default=[[23, 8, 57, 21, 76, 28],[93, 33, 86, 41, 116, 39], [120, 90, 156, 198, 373, 326]], help="anchor based anchors")
+    parser.add_argument("--strides", type=float, default=[8, 16, 32], help="model strides")
+    parser.add_argument("--imgsz", "--img", "--img-size", nargs="+", type=int, default=[416, 416], help="inference size h,w")
+    parser.add_argument("--classes", type=str, default=["plate"], help="classes num")
+    parser.add_argument("--conf-thres", type=float, default=0.25, help="confidence threshold")
+    parser.add_argument("--iou-thres", type=float, default=0.45, help="NMS IoU threshold")
+    parser.add_argument("--max-det", type=int, default=50, help="maximum detections per image")
+    parser.add_argument("--vis", default=True, help="visualize detect result")
+    parser.add_argument("--save_name", type=str, default="./det_res.jpg", help="detect img save path")
+    opt = parser.parse_args()
+    return opt
+
+if __name__ == "__main__":
+
+    opt = parse_opt()
+    model_inference(opt)

axmodel_infer_plr.py

@@ -0,0 +1,81 @@
+import cv2
+import numpy as np
+import axengine as axe
+import argparse
+
+plate_colors=['blue', 'green', 'yellow', 'white', 'black']
+
+def softmax(lpr_pred):
+    "maxpool, softmax"
+    max_out = np.max(lpr_pred, axis=1, keepdims=True)
+    exp_out = np.exp(lpr_pred - max_out)
+    sum_exp_out = np.sum(exp_out, axis=1, keepdims=True)
+    return exp_out / sum_exp_out
+
+def GreedyDecode(preb, SEP_IDX=74, plate_string=""):
+    preb_label = list()
+    for j in range(preb.shape[0]):
+        preb_label.append(np.argmax(preb[j, :], axis=0))
+    no_repeat_blank_label = list()
+    plate_index = list()
+    pre_c = preb_label[0]
+    if pre_c != SEP_IDX:
+        no_repeat_blank_label.append(pre_c)
+        plate_index.append(0)
+    for idx, c in enumerate(preb_label): # dropout repeate label and blank label
+        if (pre_c == c) or (c == SEP_IDX):
+            if c == SEP_IDX:
+                pre_c = c
+            continue
+        no_repeat_blank_label.append(c)
+        plate_index.append(idx)
+        pre_c = c
+
+    lpr_string = ''.join([plate_string[idx] for idx in no_repeat_blank_label])
+    lpr_score = 1
+    for idx, c in zip(plate_index, no_repeat_blank_label):
+        lpr_score *= preb[idx, c]
+
+    return lpr_string, lpr_score
+
+def onnx_inference(opt):
+
+    providers = ["AxEngineExecutionProvider"]
+    session = axe.InferenceSession(opt.weights, providers=providers)
+
+    input_name = session.get_inputs()[0].name
+    output_name = [output.name for output in session.get_outputs()]
+
+    img = cv2.imread(f'{opt.source}')
+    img = cv2.resize(img, (opt.imgsz[1], opt.imgsz[0]))
+    input_data = np.expand_dims(img, axis=0)[..., ::-1].transpose((0, 3, 1, 2)).astype(np.uint8)
+    outputs = session.run(output_name, {input_name:input_data})
+
+    SEP_IDX = len(opt.PLATE_STRING) - 1
+    for lpr_out, color_out in zip(*outputs):
+
+        lpr_pred = np.max(lpr_out, axis=1).T
+        lpr_pred = softmax(lpr_pred)
+        lpr_string, lpr_score = GreedyDecode(lpr_pred, SEP_IDX, opt.PLATE_STRING)
+
+        color_out = np.max(np.max(color_out, axis=1), axis=1).reshape((1, -1))
+        color_out = softmax(color_out)[0]
+        color_label = color_out.argmax()
+        color_score  = color_out[color_label]
+
+        print(f"Plate: [{lpr_string}], score: {lpr_score:.4f}, color: [{plate_colors[color_label]}], score:{color_score:.4f}")
+
+def parse_opt():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--weights", nargs="+", type=str, default="./plr_650_npu3.axmodel", help="axmodel path")
+    parser.add_argument("--source", type=str, default="./苏A8A68Y.jpg", help="img_path")
+    parser.add_argument("--imgsz", "--img", "--img-size", nargs="+", type=int, default=[48, 192], help="inference size h,w")
+    parser.add_argument("--PLATE_STRING", type=str, default=u"皖沪津渝冀晋蒙辽吉黑苏浙京闽赣鲁豫鄂湘粤桂琼川贵云藏陕甘青宁新警学港澳台使领挂OABCDEFGHJKLMNPQRSTUVWXYZ0123456789#", help="Plate string map")
+    opt = parser.parse_args()
+    
+    return opt
+
+if __name__ == "__main__":
+
+    opt = parse_opt()
+    onnx_inference(opt)