File size: 6,308 Bytes
dffe123 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 | // This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// Author, PengyuLiu, 1872918507@qq.com
#include "../precomp.hpp"
#ifdef HAVE_OPENCV_DNN
#include "opencv2/dnn.hpp"
#endif
namespace cv {
TrackerVit::TrackerVit()
{
// nothing
}
TrackerVit::~TrackerVit()
{
// nothing
}
TrackerVit::Params::Params()
{
net = "vitTracker.onnx";
meanvalue = Scalar{0.485, 0.456, 0.406}; // normalized mean (already divided by 255)
stdvalue = Scalar{0.229, 0.224, 0.225}; // normalized std (already divided by 255)
#ifdef HAVE_OPENCV_DNN
backend = dnn::DNN_BACKEND_DEFAULT;
target = dnn::DNN_TARGET_CPU;
#else
backend = -1; // invalid value
target = -1; // invalid value
#endif
tracking_score_threshold = 0.20f; // safe threshold to filter out black frames
}
#ifdef HAVE_OPENCV_DNN
class TrackerVitImpl : public TrackerVit
{
public:
TrackerVitImpl(const TrackerVit::Params& parameters)
: params(parameters)
{
net = dnn::readNet(params.net);
CV_Assert(!net.empty());
net.setPreferableBackend(params.backend);
net.setPreferableTarget(params.target);
i2bp.mean = params.meanvalue * 255.0;
i2bp.scalefactor = (1.0 / params.stdvalue) * (1 / 255.0);
}
void init(InputArray image, const Rect& boundingBox) CV_OVERRIDE;
bool update(InputArray image, Rect& boundingBox) CV_OVERRIDE;
float getTrackingScore() CV_OVERRIDE;
Rect rect_last;
float tracking_score;
TrackerVit::Params params;
dnn::Image2BlobParams i2bp;
protected:
void preprocess(const Mat& src, Mat& dst, Size size);
const Size searchSize{256, 256};
const Size templateSize{128, 128};
Mat hanningWindow;
dnn::Net net;
};
static int crop_image(const Mat& src, Mat& dst, Rect box, int factor)
{
int x = box.x, y = box.y, w = box.width, h = box.height;
int crop_sz = cvCeil(sqrt(w * h) * factor);
int x1 = x + (w - crop_sz) / 2;
int x2 = x1 + crop_sz;
int y1 = y + (h - crop_sz) / 2;
int y2 = y1 + crop_sz;
int x1_pad = std::max(0, -x1);
int y1_pad = std::max(0, -y1);
int x2_pad = std::max(x2 - src.size[1] + 1, 0);
int y2_pad = std::max(y2 - src.size[0] + 1, 0);
Rect roi(x1 + x1_pad, y1 + y1_pad, x2 - x2_pad - x1 - x1_pad, y2 - y2_pad - y1 - y1_pad);
Mat im_crop = src(roi);
copyMakeBorder(im_crop, dst, y1_pad, y2_pad, x1_pad, x2_pad, BORDER_CONSTANT);
return crop_sz;
}
void TrackerVitImpl::preprocess(const Mat& src, Mat& dst, Size size)
{
Mat img;
resize(src, img, size);
dst = dnn::blobFromImageWithParams(img, i2bp);
}
static Mat hann1d(int sz, bool centered = true) {
Mat hanningWindow(sz, 1, CV_32FC1);
float* data = hanningWindow.ptr<float>(0);
if(centered) {
for(int i = 0; i < sz; i++) {
float val = 0.5f * (1.f - std::cos(static_cast<float>(2 * M_PI / (sz + 1)) * (i + 1)));
data[i] = val;
}
}
else {
int half_sz = sz / 2;
for(int i = 0; i <= half_sz; i++) {
float val = 0.5f * (1.f + std::cos(static_cast<float>(2 * M_PI / (sz + 2)) * i));
data[i] = val;
data[sz - 1 - i] = val;
}
}
return hanningWindow;
}
static Mat hann2d(Size size, bool centered = true) {
int rows = size.height;
int cols = size.width;
Mat hanningWindowRows = hann1d(rows, centered);
Mat hanningWindowCols = hann1d(cols, centered);
Mat hanningWindow = hanningWindowRows * hanningWindowCols.t();
return hanningWindow;
}
static void updateLastRect(float cx, float cy, float w, float h, int crop_size, Rect &rect_last)
{
int x0 = rect_last.x + (rect_last.width - crop_size) / 2;
int y0 = rect_last.y + (rect_last.height - crop_size) / 2;
float x1 = cx - w / 2, y1 = cy - h / 2;
rect_last.x = cvFloor(x1 * crop_size + x0);
rect_last.y = cvFloor(y1 * crop_size + y0);
rect_last.width = cvFloor(w * crop_size);
rect_last.height = cvFloor(h * crop_size);
}
void TrackerVitImpl::init(InputArray image_, const Rect &boundingBox_)
{
Mat image = image_.getMat();
Mat crop;
crop_image(image, crop, boundingBox_, 2);
Mat blob;
preprocess(crop, blob, templateSize);
net.setInput(blob, "template");
Size size(16, 16);
hanningWindow = hann2d(size, true);
rect_last = boundingBox_;
}
bool TrackerVitImpl::update(InputArray image_, Rect &boundingBoxRes)
{
Mat image = image_.getMat();
Mat crop;
int crop_size = crop_image(image, crop, rect_last, 4); // crop: [crop_size, crop_size]
Mat blob;
preprocess(crop, blob, searchSize);
net.setInput(blob, "search");
std::vector<String> outputName = {"output1", "output2", "output3"};
std::vector<Mat> outs;
net.forward(outs, outputName);
CV_Assert(outs.size() == 3);
Mat conf_map = outs[0].reshape(0, {16, 16});
Mat size_map = outs[1].reshape(0, {2, 16, 16});
Mat offset_map = outs[2].reshape(0, {2, 16, 16});
multiply(conf_map, hanningWindow, conf_map);
double maxVal;
Point maxLoc;
minMaxLoc(conf_map, nullptr, &maxVal, nullptr, &maxLoc);
tracking_score = static_cast<float>(maxVal);
if (tracking_score >= params.tracking_score_threshold) {
float cx = (maxLoc.x + offset_map.at<float>(0, maxLoc.y, maxLoc.x)) / 16;
float cy = (maxLoc.y + offset_map.at<float>(1, maxLoc.y, maxLoc.x)) / 16;
float w = size_map.at<float>(0, maxLoc.y, maxLoc.x);
float h = size_map.at<float>(1, maxLoc.y, maxLoc.x);
updateLastRect(cx, cy, w, h, crop_size, rect_last);
boundingBoxRes = rect_last;
return true;
} else {
return false;
}
}
float TrackerVitImpl::getTrackingScore()
{
return tracking_score;
}
Ptr<TrackerVit> TrackerVit::create(const TrackerVit::Params& parameters)
{
return makePtr<TrackerVitImpl>(parameters);
}
#else // OPENCV_HAVE_DNN
Ptr<TrackerVit> TrackerVit::create(const TrackerVit::Params& parameters)
{
CV_UNUSED(parameters);
CV_Error(Error::StsNotImplemented, "to use vittrack, the tracking module needs to be built with opencv_dnn !");
}
#endif // OPENCV_HAVE_DNN
}
|