| |
| |
| """ |
| Depth Anything V2：ONNX 与 AXMODEL 原始输出一致性对比。 |
| |
| 对同一批图片分别运行 ONNX 和 axmodel，在模型分辨率下比较两路深度输出 |
| （相对逆深度，数值越大越近），逐样本计算相似度指标，并汇总为 Markdown 文档。 |
| |
| 两个后端的输入预处理可以不同，但 resize 与颜色顺序必须一致： |
| 默认 ONNX 做 /255 + ImageNet mean/std，axmodel 送 0-255 原始像素 |
| （对应编译时把归一化写进 input_processors / calibration_mean/std 的方案）。 |
| 如果 axmodel 重编译后仍需脚本侧归一化，使用 --axmodel-norm imagenet。 |
| |
| 指标： |
| cosine 两路输出的余弦相似度 |
| pearson 皮尔逊相关系数 |
| mae / rmse 绝对误差均值 / 均方根 |
| max_abs/p99_abs 最大误差 / 99 分位误差 |
| aligned_mae 最小二乘 scale-shift 对齐后的 MAE（区分全局尺度漂移与结构误差） |
| aligned_rmse 对齐后的 RMSE |
| rel_error mae / mean(|onnx|) |
| |
| 依赖：numpy、opencv-python、onnxruntime、axengine（板端 pyaxengine）。 |
| |
| 用法（在 hs/Depth-Anything-V2 目录下执行）： |
| # 离线比较已有结果（开发机，无需 axengine 或模型文件） |
| python3 src/tool/compare_onnx_axmodel.py \ |
| --onnx-results results/onnx_full --axmodel-results results/board_full \ |
| --output results/compare_full/onnx_vs_axmodel.md |
| |
| # 单图 |
| python3 src/tool/compare_onnx_axmodel.py \ |
| --onnx depth_anything_v2_vits.onnx \ |
| --axmodel depth_anything_v2_vits_ax650.axmodel \ |
| --image /path/a.jpg --output results/compare/single.md |
| |
| # 数据集（递归收集图片），默认 axmodel 送 0-255 |
| python3 src/tool/compare_onnx_axmodel.py \ |
| --onnx depth_anything_v2_vits.onnx \ |
| --axmodel depth_anything_v2_vits_ax650.axmodel \ |
| --dataset /path/images --limit 50 \ |
| --output results/compare/onnx_vs_axmodel.md |
| |
| # axmodel 仍需要脚本侧做 ImageNet 归一化时 |
| python3 src/tool/compare_onnx_axmodel.py \ |
| --onnx depth_anything_v2_vits.onnx --axmodel other.axmodel \ |
| --dataset /path/images --axmodel-norm imagenet \ |
| --output results/compare/onnx_vs_axmodel.md |
| |
| 注意： |
| - axmodel 的输出名可能被编译器改写，脚本先按名字、再按 shape 回退绑定。 |
| - 两个后端的输出 shape 必须一致；不一致时直接报错并打印实际接口。 |
| - 结果默认输出到 results/compare/，不写预测 NPZ。 |
| """ |
|
|
| import argparse |
| import json |
| import logging |
| import time |
| from datetime import datetime |
| from pathlib import Path |
|
|
| import cv2 |
| import numpy as np |
|
|
| MEAN = np.array([0.485, 0.456, 0.406], dtype=np.float32) |
| STD = np.array([0.229, 0.224, 0.225], dtype=np.float32) |
| EXTENSIONS = {'.jpg', '.jpeg', '.png', '.bmp', '.tif', '.tiff'} |
| METRICS = ('cosine', 'pearson', 'mae', 'rmse', 'max_abs', 'p99_abs', |
| 'aligned_mae', 'aligned_rmse', 'rel_error') |
|
|
|
|
| def meta_type(meta): |
| return str(getattr(meta, 'type', getattr(meta, 'dtype', ''))).lower() |
|
|
|
|
| def input_spec(shape, type_name, model_name): |
| shape = list(shape) |
| if len(shape) != 4 or not all(isinstance(v, int) and v > 0 for v in shape): |
| raise ValueError(f'{model_name} 输入必须是静态 4D NHWC/NCHW: {shape}') |
| if shape[0] != 1: |
| raise ValueError(f'{model_name} 仅支持 batch=1: {shape}') |
| if shape[1] == 3: |
| layout, size = 'NCHW', (shape[2], shape[3]) |
| elif shape[3] == 3: |
| layout, size = 'NHWC', (shape[1], shape[2]) |
| else: |
| raise ValueError(f'{model_name} 输入必须含 3 通道: {shape}') |
| if 'float' in type_name: |
| dtype = np.float32 |
| elif 'uint8' in type_name or type_name.endswith('u8'): |
| dtype = np.uint8 |
| elif 'int8' in type_name: |
| dtype = np.int8 |
| else: |
| raise ValueError(f'{model_name} 不支持的输入类型: {type_name!r}') |
| return size, layout, dtype |
|
|
|
|
| def onnx_interface(session): |
| inputs, outputs = session.get_inputs(), session.get_outputs() |
| logging.info('ONNX 输入: %s', [(m.name, m.shape, m.type) for m in inputs]) |
| logging.info('ONNX 输出: %s', [(m.name, m.shape, m.type) for m in outputs]) |
| if len(inputs) != 1: |
| raise ValueError(f'ONNX 应有 1 个输入: {[(m.name, m.shape) for m in inputs]}') |
| meta = inputs[0] |
| if 'float' not in str(meta.type): |
| raise ValueError(f'ONNX 输入要求 float32: {meta.name} {meta.type}') |
| size, layout, dtype = input_spec(meta.shape, str(meta.type), 'ONNX') |
| return meta.name, size, layout, dtype, outputs[0].name |
|
|
|
|
| def axmodel_interface(session): |
| inputs, outputs = session.get_inputs(), session.get_outputs() |
| logging.info('AXMODEL 输入: %s', [(m.name, m.shape, meta_type(m)) for m in inputs]) |
| logging.info('AXMODEL 输出: %s', [(m.name, m.shape, meta_type(m)) for m in outputs]) |
| if len(inputs) != 1: |
| raise ValueError(f'axmodel 应有 1 个输入: {[(m.name, m.shape) for m in inputs]}') |
| meta = inputs[0] |
| size, layout, dtype = input_spec(meta.shape, meta_type(meta), 'AXMODEL') |
| return meta, size, layout, dtype, outputs |
|
|
|
|
| def preprocess(image, size, layout, dtype, resize_mode, color_order, norm): |
| target_h, target_w = size |
| original_h, original_w = image.shape[:2] |
| if color_order == 'rgb': |
| image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) |
| if resize_mode == 'pad': |
| scale = min(target_h / original_h, target_w / original_w) |
| new_h = max(1, min(target_h, int(round(original_h * scale)))) |
| new_w = max(1, min(target_w, int(round(original_w * scale)))) |
| resized = cv2.resize(image, (new_w, new_h), interpolation=cv2.INTER_LINEAR) |
| canvas = np.zeros((target_h, target_w, 3), dtype=np.float32) |
| canvas[:new_h, :new_w] = resized |
| image_size = (new_h, new_w) |
| else: |
| canvas = cv2.resize(image, (target_w, target_h), |
| interpolation=cv2.INTER_LINEAR).astype(np.float32) |
| image_size = (target_h, target_w) |
| canvas = canvas.astype(np.float32, copy=False) |
| if norm == 'imagenet': |
| if dtype != np.float32: |
| raise ValueError('整数输入不能使用 ImageNet 归一化') |
| canvas = (canvas / 255.0 - MEAN) / STD |
| if dtype != np.float32: |
| canvas = np.rint(np.clip(canvas, 0, 255)).astype(dtype) |
| tensor = canvas.transpose(2, 0, 1) if layout == 'NCHW' else canvas |
| return np.ascontiguousarray(tensor[None]), image_size |
|
|
|
|
| def to_2d(array, model_name): |
| array = np.asarray(array) |
| if array.ndim == 4: |
| if array.shape[0] != 1: |
| raise ValueError(f'{model_name} 输出 batch 必须为 1: {array.shape}') |
| if array.shape[1] == 1: |
| array = array[0, 0] |
| elif array.shape[3] == 1: |
| array = array[0, :, :, 0] |
| else: |
| raise ValueError(f'{model_name} 输出无法压缩为 2D: {array.shape}') |
| elif array.ndim == 3: |
| if array.shape[0] == 1: |
| array = array[0] |
| elif array.shape[2] == 1: |
| array = array[:, :, 0] |
| else: |
| raise ValueError(f'{model_name} 输出无法压缩为 2D: {array.shape}') |
| if array.ndim != 2: |
| raise ValueError(f'{model_name} 输出必须是 2D 深度图: {array.shape}') |
| array = array.astype(np.float32, copy=False) |
| if not np.isfinite(array).all(): |
| raise ValueError(f'{model_name} 输出包含 nan/inf') |
| return array |
|
|
|
|
| def select_ax_output(outputs, metas, expected_shape): |
| if not isinstance(outputs, (list, tuple)): |
| outputs = [outputs] |
| if len(outputs) != len(metas): |
| raise ValueError(f'axmodel 输出数量不一致: {len(outputs)} vs {len(metas)}') |
| for index, meta in enumerate(metas): |
| if any(token in str(meta.name).lower() for token in ('depth', 'output')): |
| array = to_2d(outputs[index], 'AXMODEL') |
| if array.shape == expected_shape: |
| return array, str(meta.name) |
| candidates = [] |
| for index, array in enumerate(outputs): |
| candidate = to_2d(array, 'AXMODEL') |
| if candidate.shape == expected_shape: |
| candidates.append((index, candidate)) |
| if len(candidates) != 1: |
| raise ValueError('axmodel 输出无法按 name 或 shape 唯一匹配' |
| f' {expected_shape}: {[(m.name, np.asarray(o).shape) for m, o in zip(metas, outputs)]}') |
| index, candidate = candidates[0] |
| return candidate, str(metas[index].name) |
|
|
|
|
| def compare_arrays(reference, actual): |
| ref = reference.astype(np.float64).ravel() |
| act = actual.astype(np.float64).ravel() |
| diff = act - ref |
| ref_norm = np.linalg.norm(ref) |
| act_norm = np.linalg.norm(act) |
| cosine = float(np.dot(ref, act) / (ref_norm * act_norm + 1e-12)) |
| if ref.std() > 1e-12 and act.std() > 1e-12: |
| pearson = float(np.corrcoef(ref, act)[0, 1]) |
| else: |
| pearson = float('nan') |
| slope, offset = np.polyfit(ref, act, 1) |
| aligned = act - (slope * ref + offset) |
| abs_diff = np.abs(diff) |
| return { |
| 'cosine': cosine, |
| 'pearson': pearson, |
| 'mae': float(abs_diff.mean()), |
| 'rmse': float(np.sqrt(np.mean(diff ** 2))), |
| 'max_abs': float(abs_diff.max()), |
| 'p99_abs': float(np.percentile(abs_diff, 99)), |
| 'aligned_mae': float(np.abs(aligned).mean()), |
| 'aligned_rmse': float(np.sqrt(np.mean(aligned ** 2))), |
| 'rel_error': float(abs_diff.mean() / (np.abs(ref).mean() + 1e-12)), |
| } |
|
|
|
|
| def summarize(records): |
| summary = {} |
| for key in METRICS: |
| values = np.array([record[key] for record in records], dtype=np.float64) |
| values = values[np.isfinite(values)] |
| if values.size == 0: |
| summary[key] = None |
| continue |
| summary[key] = {'mean': float(values.mean()), |
| 'median': float(np.percentile(values, 50)), |
| 'p90': float(np.percentile(values, 90)), |
| 'p99': float(np.percentile(values, 99)), |
| 'min': float(values.min()), |
| 'max': float(values.max())} |
| return summary |
|
|
|
|
| def format_value(value): |
| if value is None: |
| return '-' |
| if isinstance(value, float) and not np.isfinite(value): |
| return '-' |
| return f'{value:.6g}' |
|
|
|
|
| def write_markdown(path, report): |
| lines = ['# Depth Anything V2：ONNX 与 AXMODEL 输出一致性', ''] |
| lines.append(f'- 生成时间：{report["created"]}') |
| lines.append(f'- 样本数：{report["sample_count"]}') |
| lines.append(f'- 结果目录：`{report["output_dir"]}`') |
| if report['limit'] is not None: |
| lines.append(f'- 样本上限：{report["limit"]}') |
| lines.append('') |
| if report.get('mode') == 'results': |
| lines.extend([ |
| '## 结果来源', '', |
| f'- ONNX：`{report["onnx_results"]}`', |
| f'- AXMODEL：`{report["axmodel_results"]}`', |
| '- 按相对路径匹配同名 `.depth.npy`，比较保存的深度图，不重新推理或 resize。', |
| '- 这些文件由推理脚本还原至原图尺寸，指标对应保存结果的像素空间。', |
| '- 离线比较不测量推理耗时；预处理配置以原推理记录为准。', '', |
| ]) |
| else: |
| lines.append('## 模型与预处理') |
| lines.append('') |
| lines.append('| 项 | ONNX | AXMODEL |') |
| lines.append('| --- | --- | --- |') |
| lines.append(f'| 模型 | `{report["onnx"]["path"]}` | `{report["axmodel"]["path"]}` |') |
| lines.append(f'| 运行时 | {report["onnx"]["runtime"]} | {report["axmodel"]["runtime"]} |') |
| lines.append(f'| 输入 | {report["onnx"]["input"]} | {report["axmodel"]["input"]} |') |
| lines.append(f'| 输出 | {report["onnx"]["output"]} | {report["axmodel"]["output"]} |') |
| lines.append(f'| 归一化 | {report["onnx"]["norm"]} | {report["axmodel"]["norm"]} |') |
| lines.append(f'| resize | {report["resize_mode"]} | {report["resize_mode"]} |') |
| lines.append(f'| 颜色顺序 | {report["color_order"]} | {report["color_order"]} |') |
| lines.append(f'| warmup | {report["warmup"]} | {report["warmup"]} |') |
| lines.append('') |
| lines.append('## 汇总指标') |
| lines.append('') |
| lines.append('| 指标 | mean | median | p90 | p99 | min | max |') |
| lines.append('| --- | --- | --- | --- | --- | --- | --- |') |
| for key in METRICS: |
| stats = report['summary'][key] |
| if stats is None: |
| lines.append(f'| {key} | - | - | - | - | - | - |') |
| else: |
| lines.append(f'| {key} | {format_value(stats["mean"])} | ' |
| f'{format_value(stats["median"])} | {format_value(stats["p90"])} | ' |
| f'{format_value(stats["p99"])} | {format_value(stats["min"])} | ' |
| f'{format_value(stats["max"])} |') |
| lines.append('') |
| lines.append('## 逐样本结果') |
| lines.append('') |
| lines.append('| # | 图片 | cosine | pearson | mae | rmse | max_abs | p99_abs | ' |
| 'aligned_mae | aligned_rmse | rel_error | ONNX ms | AXMODEL ms |') |
| lines.append('| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |') |
| for index, sample in enumerate(report['samples'], 1): |
| metrics = sample['metrics'] |
| lines.append( |
| f'| {index} | `{sample["name"]}` | {format_value(metrics["cosine"])} | ' |
| f'{format_value(metrics["pearson"])} | {format_value(metrics["mae"])} | ' |
| f'{format_value(metrics["rmse"])} | {format_value(metrics["max_abs"])} | ' |
| f'{format_value(metrics["p99_abs"])} | {format_value(metrics["aligned_mae"])} | ' |
| f'{format_value(metrics["aligned_rmse"])} | {format_value(metrics["rel_error"])} | ' |
| f'{format_value(sample["onnx_ms"])} | {format_value(sample["axmodel_ms"])} |') |
| lines.append('') |
| lines.append('## 指标说明') |
| lines.append('') |
| lines.append('- `cosine`：两路输出的余弦相似度，1 表示方向完全一致。') |
| lines.append('- `pearson`：线性相关系数，衡量结构一致性，对全局尺度不敏感。') |
| lines.append('- `mae` / `rmse`：未对齐的绝对误差，包含量化误差与整体尺度漂移。') |
| lines.append('- `max_abs` / `p99_abs`：最大误差和 99 分位误差，反映局部离群点。') |
| lines.append('- `aligned_mae` / `aligned_rmse`：先做最小二乘 scale-shift 对齐再计算，' |
| '用于区分全局尺度漂移和真正的结构偏差。') |
| lines.append('- `rel_error`：`mae / mean(|ONNX|)`，便于跨图片比较。') |
| lines.append('') |
| lines.append('## 判定参考') |
| lines.append('') |
| lines.append(f'- 余弦相似度均值：{format_value(report["summary"]["cosine"]["mean"])}') |
| lines.append(f'- 对齐后 MAE 均值：{format_value(report["summary"]["aligned_mae"]["mean"])}') |
| lines.append(f'- 未对齐相对误差均值：' |
| f'{format_value(report["summary"]["rel_error"]["mean"])}') |
| lines.append('') |
| lines.append('若 `cosine` 接近 1 但 `mae` 明显偏大、`aligned_mae` 很小，说明两路输出只存在' |
| '整体尺度/偏移差异；若 `aligned_mae` 也明显偏大，则更可能是量化本身造成的' |
| '结构失真，需要结合逐层精度分析继续定位。') |
| path.parent.mkdir(parents=True, exist_ok=True) |
| path.write_text('\n'.join(lines) + '\n', encoding='utf-8') |
|
|
|
|
| def collect_images(dataset): |
| paths = sorted(p for p in dataset.rglob('*') |
| if p.is_file() and p.suffix.lower() in EXTENSIONS) |
| if not paths: |
| raise ValueError(f'目录中没有图片: {dataset}') |
| return paths |
|
|
|
|
| def compare_result_dirs(args): |
| from tqdm import tqdm |
|
|
| files = [] |
| for directory in (args.onnx_results, args.axmodel_results): |
| if not directory.is_dir(): |
| raise ValueError(f'结果目录不存在: {directory}') |
| mapping = {p.relative_to(directory).as_posix(): p |
| for p in directory.rglob('*.depth.npy') if p.is_file()} |
| if not mapping: |
| raise ValueError(f'结果目录中没有 *.depth.npy: {directory}') |
| files.append(mapping) |
| onnx_files, axmodel_files = files |
| if onnx_files.keys() != axmodel_files.keys(): |
| onnx_only = sorted(onnx_files.keys() - axmodel_files.keys()) |
| axmodel_only = sorted(axmodel_files.keys() - onnx_files.keys()) |
| raise ValueError(f'结果文件未完全配对: ONNX 独有 {len(onnx_only)} 个 ' |
| f'{onnx_only[:5]}, AXMODEL 独有 {len(axmodel_only)} 个 ' |
| f'{axmodel_only[:5]}') |
| names = sorted(onnx_files) |
| if args.limit is not None: |
| names = names[:args.limit] |
| records = [] |
| for name in tqdm(names, desc='比较深度输出', disable=len(names) == 1): |
| reference = to_2d(np.load(onnx_files[name], allow_pickle=False), f'ONNX {name}') |
| actual = to_2d(np.load(axmodel_files[name], allow_pickle=False), f'AXMODEL {name}') |
| if reference.size == 0 or reference.shape != actual.shape: |
| raise ValueError(f'{name} 深度图为空或 shape 不一致: ' |
| f'ONNX {reference.shape} vs AXMODEL {actual.shape}') |
| metrics = compare_arrays(reference, actual) |
| if args.debug: |
| logging.info('%s: %s', name, metrics) |
| records.append({'name': name[:-len('.depth.npy')], 'metrics': metrics, |
| 'onnx_ms': None, 'axmodel_ms': None}) |
| report = { |
| 'mode': 'results', |
| 'created': datetime.now().astimezone().isoformat(timespec='seconds'), |
| 'output_dir': str(args.output.resolve().parent), |
| 'onnx_results': str(args.onnx_results.resolve()), |
| 'axmodel_results': str(args.axmodel_results.resolve()), |
| 'sample_count': len(records), 'limit': args.limit, |
| 'summary': summarize([record['metrics'] for record in records]), |
| 'samples': records, |
| } |
| write_markdown(args.output, report) |
| logging.info('样本数 %d，cosine mean=%.6f，aligned_mae mean=%.6g，报告: %s', |
| len(records), report['summary']['cosine']['mean'], |
| report['summary']['aligned_mae']['mean'], args.output) |
|
|
|
|
| def main(): |
| parser = argparse.ArgumentParser( |
| description='计算 ONNX 与 AXMODEL 深度输出的相似度并生成 Markdown 报告', |
| formatter_class=argparse.RawDescriptionHelpFormatter, epilog=__doc__) |
| parser.add_argument('--onnx', type=Path, help='FP32 ONNX 模型') |
| parser.add_argument('--axmodel', type=Path, help='编译后的 axmodel') |
| parser.add_argument('--onnx-results', type=Path, help='ONNX 结果目录（递归读取 *.depth.npy）') |
| parser.add_argument('--axmodel-results', type=Path, help='AXMODEL 结果目录（相对路径必须完全配对）') |
| source = parser.add_mutually_exclusive_group() |
| source.add_argument('--image', type=Path, help='单张图片') |
| source.add_argument('--dataset', type=Path, help='图片目录（递归）') |
| parser.add_argument('--output', type=Path, |
| default=Path(__file__).resolve().parents[2] / 'results' / |
| 'compare' / 'onnx_vs_axmodel.md', |
| help='Markdown 输出路径，默认 results/compare/onnx_vs_axmodel.md') |
| parser.add_argument('--resize-mode', choices=('stretch', 'pad'), default='stretch', |
| help='两个后端共用的 resize 方式，默认 stretch') |
| parser.add_argument('--color-order', choices=('rgb', 'bgr'), default='rgb', |
| help='两个后端共用的颜色顺序，默认 rgb') |
| parser.add_argument('--onnx-norm', choices=('imagenet', 'raw'), default='imagenet', |
| help='ONNX 输入归一化，默认 imagenet') |
| parser.add_argument('--axmodel-norm', choices=('imagenet', 'raw'), default='raw', |
| help='axmodel 输入归一化，默认 raw（编译时已吸收归一化）') |
| parser.add_argument('--warmup', type=int, default=3, |
| help='数据集模式每个后端的 warmup 次数，默认 3') |
| parser.add_argument('--limit', type=int, help='数据集模式只处理前 N 张') |
| parser.add_argument('--debug', action='store_true', help='打印逐样本输入/输出统计') |
| args = parser.parse_args() |
| logging.basicConfig(level=logging.INFO, format='%(message)s') |
| if args.warmup < 0: |
| parser.error('--warmup 不能为负数') |
| if args.limit is not None and args.limit < 1: |
| parser.error('--limit 必须为正数') |
| if args.onnx_results is not None or args.axmodel_results is not None: |
| if args.onnx_results is None or args.axmodel_results is None: |
| parser.error('--onnx-results 和 --axmodel-results 必须同时提供') |
| if any(value is not None for value in (args.onnx, args.axmodel, args.image, args.dataset)): |
| parser.error('结果目录模式不能同时指定模型或图片输入') |
| compare_result_dirs(args) |
| return |
| if args.onnx is None or args.axmodel is None or (args.image is None and args.dataset is None): |
| parser.error('推理模式必须提供 --onnx、--axmodel 和 --image/--dataset；' |
| '或使用 --onnx-results 和 --axmodel-results 进行离线比较') |
| if not args.onnx.is_file(): |
| parser.error(f'--onnx 不存在: {args.onnx}') |
| if not args.axmodel.is_file(): |
| parser.error(f'--axmodel 不存在: {args.axmodel}') |
|
|
| import onnxruntime as ort |
| import axengine |
| onnx_session = ort.InferenceSession(str(args.onnx)) |
| axmodel_session = axengine.InferenceSession(str(args.axmodel)) |
| onnx_input, onnx_size, onnx_layout, onnx_dtype, onnx_output = onnx_interface(onnx_session) |
| ax_meta, ax_size, ax_layout, ax_dtype, ax_outputs = axmodel_interface(axmodel_session) |
| if onnx_size != ax_size: |
| raise ValueError(f'两个后端输入尺寸不一致: ONNX {onnx_size} vs AXMODEL {ax_size}') |
|
|
| if args.image: |
| if not args.image.is_file(): |
| parser.error(f'--image 不是文件: {args.image}') |
| images = [args.image] |
| root = args.image.parent |
| else: |
| if not args.dataset.is_dir(): |
| parser.error(f'--dataset 不是目录: {args.dataset}') |
| images = collect_images(args.dataset) |
| if args.limit is not None: |
| images = images[:args.limit] |
| root = args.dataset |
|
|
| if args.axmodel_norm == 'imagenet' and ax_dtype != np.float32: |
| parser.error('axmodel 输入为整数类型，不能使用 --axmodel-norm imagenet') |
|
|
| |
| if len(images) > 1 and args.warmup: |
| first = cv2.imread(str(images[0]), cv2.IMREAD_COLOR) |
| if first is None: |
| raise ValueError(f'读取图片失败: {images[0]}') |
| onnx_tensor, _ = preprocess(first, onnx_size, onnx_layout, onnx_dtype, |
| args.resize_mode, args.color_order, args.onnx_norm) |
| ax_tensor, _ = preprocess(first, ax_size, ax_layout, ax_dtype, |
| args.resize_mode, args.color_order, args.axmodel_norm) |
| for _ in range(args.warmup): |
| onnx_session.run([onnx_output], {onnx_input: onnx_tensor}) |
| axmodel_session.run(None, {ax_meta.name: ax_tensor}) |
| logging.info('两个后端各 warmup %d 次', args.warmup) |
|
|
| records = [] |
| for path in images: |
| name = str(path.relative_to(root)) if path != root else path.name |
| image = cv2.imread(str(path), cv2.IMREAD_COLOR) |
| if image is None: |
| raise ValueError(f'读取图片失败: {path}') |
| onnx_tensor, _ = preprocess(image, onnx_size, onnx_layout, onnx_dtype, |
| args.resize_mode, args.color_order, args.onnx_norm) |
| ax_tensor, _ = preprocess(image, ax_size, ax_layout, ax_dtype, |
| args.resize_mode, args.color_order, args.axmodel_norm) |
|
|
| start = time.perf_counter() |
| onnx_raw = onnx_session.run([onnx_output], {onnx_input: onnx_tensor})[0] |
| onnx_ms = (time.perf_counter() - start) * 1000 |
| start = time.perf_counter() |
| ax_outputs_data = axmodel_session.run(None, {ax_meta.name: ax_tensor}) |
| axmodel_ms = (time.perf_counter() - start) * 1000 |
|
|
| reference = to_2d(onnx_raw, 'ONNX') |
| actual, ax_output_name = select_ax_output(ax_outputs_data, ax_outputs, |
| reference.shape) |
| metrics = compare_arrays(reference, actual) |
| if not np.isfinite(metrics['cosine']): |
| raise ValueError(f'{name} 无法计算相似度，请检查输出是否常数') |
| if args.debug: |
| logging.info('%s: ONNX min=%.4f max=%.4f mean=%.4f | AXMODEL min=%.4f ' |
| 'max=%.4f mean=%.4f | cosine=%.6f mae=%.6g', |
| name, float(reference.min()), float(reference.max()), |
| float(reference.mean()), float(actual.min()), |
| float(actual.max()), float(actual.mean()), |
| metrics['cosine'], metrics['mae']) |
| records.append({'name': name, 'metrics': metrics, |
| 'onnx_ms': onnx_ms, 'axmodel_ms': axmodel_ms}) |
| if len(images) == 1: |
| logging.info('输出绑定: ONNX=%s AXMODEL=%s', onnx_output, ax_output_name) |
| logging.info('指标: %s', json.dumps(metrics, ensure_ascii=False)) |
|
|
| report = { |
| 'created': datetime.now().astimezone().isoformat(timespec='seconds'), |
| 'output_dir': str(args.output.resolve().parent), |
| 'sample_count': len(records), |
| 'limit': args.limit if not args.image else None, |
| 'warmup': args.warmup if len(images) > 1 else 0, |
| 'resize_mode': args.resize_mode, |
| 'color_order': args.color_order, |
| 'onnx': {'path': str(args.onnx.resolve()), |
| 'runtime': 'onnxruntime ' + ort.__version__, |
| 'input': f'{onnx_input} {onnx_tensor.shape}', |
| 'output': onnx_output, |
| 'norm': args.onnx_norm}, |
| 'axmodel': {'path': str(args.axmodel.resolve()), |
| 'runtime': getattr(axengine, '__version__', 'unknown'), |
| 'input': f'{ax_meta.name} {tuple(ax_meta.shape)} {np.dtype(ax_dtype).name}', |
| 'output': ax_output_name, |
| 'norm': args.axmodel_norm}, |
| 'summary': summarize([record['metrics'] for record in records]), |
| 'samples': records, |
| } |
| write_markdown(args.output, report) |
| logging.info('样本数 %d，cosine mean=%.6f，aligned_mae mean=%.6g，报告: %s', |
| len(records), report['summary']['cosine']['mean'], |
| report['summary']['aligned_mae']['mean'], args.output) |
|
|
|
|
| if __name__ == '__main__': |
| main() |
|
|
