|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
#include "picodet.h" |
|
|
#include <benchmark.h> |
|
|
#include <iostream> |
|
|
|
|
|
inline float fast_exp(float x) { |
|
|
union { |
|
|
uint32_t i; |
|
|
float f; |
|
|
} v{}; |
|
|
v.i = (1 << 23) * (1.4426950409 * x + 126.93490512f); |
|
|
return v.f; |
|
|
} |
|
|
|
|
|
inline float sigmoid(float x) { return 1.0f / (1.0f + fast_exp(-x)); } |
|
|
|
|
|
template <typename _Tp> |
|
|
int activation_function_softmax(const _Tp *src, _Tp *dst, int length) { |
|
|
const _Tp alpha = *std::max_element(src, src + length); |
|
|
_Tp denominator{0}; |
|
|
|
|
|
for (int i = 0; i < length; ++i) { |
|
|
dst[i] = fast_exp(src[i] - alpha); |
|
|
denominator += dst[i]; |
|
|
} |
|
|
|
|
|
for (int i = 0; i < length; ++i) { |
|
|
dst[i] /= denominator; |
|
|
} |
|
|
|
|
|
return 0; |
|
|
} |
|
|
|
|
|
bool PicoDet::hasGPU = false; |
|
|
PicoDet *PicoDet::detector = nullptr; |
|
|
|
|
|
PicoDet::PicoDet(const char *param, const char *bin, int input_width, |
|
|
int input_hight, bool useGPU, float score_threshold_ = 0.5, |
|
|
float nms_threshold_ = 0.3) { |
|
|
this->Net = new ncnn::Net(); |
|
|
#if NCNN_VULKAN |
|
|
this->hasGPU = ncnn::get_gpu_count() > 0; |
|
|
#endif |
|
|
this->Net->opt.use_vulkan_compute = this->hasGPU && useGPU; |
|
|
this->Net->opt.use_fp16_arithmetic = true; |
|
|
this->Net->load_param(param); |
|
|
this->Net->load_model(bin); |
|
|
this->in_w = input_width; |
|
|
this->in_h = input_hight; |
|
|
this->score_threshold = score_threshold_; |
|
|
this->nms_threshold = nms_threshold_; |
|
|
} |
|
|
|
|
|
PicoDet::~PicoDet() { delete this->Net; } |
|
|
|
|
|
void PicoDet::preprocess(cv::Mat &image, ncnn::Mat &in) { |
|
|
|
|
|
int img_w = image.cols; |
|
|
int img_h = image.rows; |
|
|
in = ncnn::Mat::from_pixels_resize(image.data, ncnn::Mat::PIXEL_BGR, img_w, |
|
|
img_h, this->in_w, this->in_h); |
|
|
const float mean_vals[3] = {103.53f, 116.28f, 123.675f}; |
|
|
const float norm_vals[3] = {0.017429f, 0.017507f, 0.017125f}; |
|
|
in.substract_mean_normalize(mean_vals, norm_vals); |
|
|
} |
|
|
|
|
|
int PicoDet::detect(cv::Mat image, std::vector<BoxInfo> &result_list, |
|
|
bool has_postprocess) { |
|
|
|
|
|
ncnn::Mat input; |
|
|
preprocess(image, input); |
|
|
auto ex = this->Net->create_extractor(); |
|
|
ex.set_light_mode(false); |
|
|
ex.set_num_threads(4); |
|
|
#if NCNN_VULKAN |
|
|
ex.set_vulkan_compute(this->hasGPU); |
|
|
#endif |
|
|
ex.input("image", input); |
|
|
|
|
|
this->image_h = image.rows; |
|
|
this->image_w = image.cols; |
|
|
|
|
|
std::vector<std::vector<BoxInfo>> results; |
|
|
results.resize(this->num_class); |
|
|
|
|
|
if (has_postprocess) { |
|
|
ncnn::Mat dis_pred; |
|
|
ncnn::Mat cls_pred; |
|
|
ex.extract(this->nms_heads_info[0].c_str(), dis_pred); |
|
|
ex.extract(this->nms_heads_info[1].c_str(), cls_pred); |
|
|
std::cout << dis_pred.h << " " << dis_pred.w << std::endl; |
|
|
std::cout << cls_pred.h << " " << cls_pred.w << std::endl; |
|
|
this->nms_boxes(cls_pred, dis_pred, this->score_threshold, results); |
|
|
} else { |
|
|
for (const auto &head_info : this->non_postprocess_heads_info) { |
|
|
ncnn::Mat dis_pred; |
|
|
ncnn::Mat cls_pred; |
|
|
ex.extract(head_info.dis_layer.c_str(), dis_pred); |
|
|
ex.extract(head_info.cls_layer.c_str(), cls_pred); |
|
|
this->decode_infer(cls_pred, dis_pred, head_info.stride, |
|
|
this->score_threshold, results); |
|
|
} |
|
|
} |
|
|
|
|
|
for (int i = 0; i < (int)results.size(); i++) { |
|
|
this->nms(results[i], this->nms_threshold); |
|
|
|
|
|
for (auto box : results[i]) { |
|
|
box.x1 = box.x1 / this->in_w * this->image_w; |
|
|
box.x2 = box.x2 / this->in_w * this->image_w; |
|
|
box.y1 = box.y1 / this->in_h * this->image_h; |
|
|
box.y2 = box.y2 / this->in_h * this->image_h; |
|
|
result_list.push_back(box); |
|
|
} |
|
|
} |
|
|
return 0; |
|
|
} |
|
|
|
|
|
void PicoDet::nms_boxes(ncnn::Mat &cls_pred, ncnn::Mat &dis_pred, |
|
|
float score_threshold, |
|
|
std::vector<std::vector<BoxInfo>> &result_list) { |
|
|
BoxInfo bbox; |
|
|
int i, j; |
|
|
for (i = 0; i < dis_pred.h; i++) { |
|
|
bbox.x1 = dis_pred.row(i)[0]; |
|
|
bbox.y1 = dis_pred.row(i)[1]; |
|
|
bbox.x2 = dis_pred.row(i)[2]; |
|
|
bbox.y2 = dis_pred.row(i)[3]; |
|
|
const float *scores = cls_pred.row(i); |
|
|
float score = 0; |
|
|
int cur_label = 0; |
|
|
for (int label = 0; label < this->num_class; label++) { |
|
|
float score_ = cls_pred.row(label)[i]; |
|
|
if (score_ > score) { |
|
|
score = score_; |
|
|
cur_label = label; |
|
|
} |
|
|
} |
|
|
bbox.score = score; |
|
|
bbox.label = cur_label; |
|
|
result_list[cur_label].push_back(bbox); |
|
|
} |
|
|
} |
|
|
|
|
|
void PicoDet::decode_infer(ncnn::Mat &cls_pred, ncnn::Mat &dis_pred, int stride, |
|
|
float threshold, |
|
|
std::vector<std::vector<BoxInfo>> &results) { |
|
|
int feature_h = ceil((float)this->in_w / stride); |
|
|
int feature_w = ceil((float)this->in_h / stride); |
|
|
|
|
|
for (int idx = 0; idx < feature_h * feature_w; idx++) { |
|
|
const float *scores = cls_pred.row(idx); |
|
|
int row = idx / feature_w; |
|
|
int col = idx % feature_w; |
|
|
float score = 0; |
|
|
int cur_label = 0; |
|
|
for (int label = 0; label < this->num_class; label++) { |
|
|
if (scores[label] > score) { |
|
|
score = scores[label]; |
|
|
cur_label = label; |
|
|
} |
|
|
} |
|
|
if (score > threshold) { |
|
|
const float *bbox_pred = dis_pred.row(idx); |
|
|
results[cur_label].push_back( |
|
|
this->disPred2Bbox(bbox_pred, cur_label, score, col, row, stride)); |
|
|
} |
|
|
} |
|
|
} |
|
|
|
|
|
BoxInfo PicoDet::disPred2Bbox(const float *&dfl_det, int label, float score, |
|
|
int x, int y, int stride) { |
|
|
float ct_x = (x + 0.5) * stride; |
|
|
float ct_y = (y + 0.5) * stride; |
|
|
std::vector<float> dis_pred; |
|
|
dis_pred.resize(4); |
|
|
for (int i = 0; i < 4; i++) { |
|
|
float dis = 0; |
|
|
float *dis_after_sm = new float[this->reg_max + 1]; |
|
|
activation_function_softmax(dfl_det + i * (this->reg_max + 1), dis_after_sm, |
|
|
this->reg_max + 1); |
|
|
for (int j = 0; j < this->reg_max + 1; j++) { |
|
|
dis += j * dis_after_sm[j]; |
|
|
} |
|
|
dis *= stride; |
|
|
dis_pred[i] = dis; |
|
|
delete[] dis_after_sm; |
|
|
} |
|
|
float xmin = (std::max)(ct_x - dis_pred[0], .0f); |
|
|
float ymin = (std::max)(ct_y - dis_pred[1], .0f); |
|
|
float xmax = (std::min)(ct_x + dis_pred[2], (float)this->in_w); |
|
|
float ymax = (std::min)(ct_y + dis_pred[3], (float)this->in_w); |
|
|
return BoxInfo{xmin, ymin, xmax, ymax, score, label}; |
|
|
} |
|
|
|
|
|
void PicoDet::nms(std::vector<BoxInfo> &input_boxes, float NMS_THRESH) { |
|
|
std::sort(input_boxes.begin(), input_boxes.end(), |
|
|
[](BoxInfo a, BoxInfo b) { return a.score > b.score; }); |
|
|
std::vector<float> vArea(input_boxes.size()); |
|
|
for (int i = 0; i < int(input_boxes.size()); ++i) { |
|
|
vArea[i] = (input_boxes.at(i).x2 - input_boxes.at(i).x1 + 1) * |
|
|
(input_boxes.at(i).y2 - input_boxes.at(i).y1 + 1); |
|
|
} |
|
|
for (int i = 0; i < int(input_boxes.size()); ++i) { |
|
|
for (int j = i + 1; j < int(input_boxes.size());) { |
|
|
float xx1 = (std::max)(input_boxes[i].x1, input_boxes[j].x1); |
|
|
float yy1 = (std::max)(input_boxes[i].y1, input_boxes[j].y1); |
|
|
float xx2 = (std::min)(input_boxes[i].x2, input_boxes[j].x2); |
|
|
float yy2 = (std::min)(input_boxes[i].y2, input_boxes[j].y2); |
|
|
float w = (std::max)(float(0), xx2 - xx1 + 1); |
|
|
float h = (std::max)(float(0), yy2 - yy1 + 1); |
|
|
float inter = w * h; |
|
|
float ovr = inter / (vArea[i] + vArea[j] - inter); |
|
|
if (ovr >= NMS_THRESH) { |
|
|
input_boxes.erase(input_boxes.begin() + j); |
|
|
vArea.erase(vArea.begin() + j); |
|
|
} else { |
|
|
j++; |
|
|
} |
|
|
} |
|
|
} |
|
|
} |
|
|
|
