Download modules/imgproc/src/filter.simd.hpp from tgollogly/test: direct link, hf CLI and curl.
- Browser
- Download file 149 kB
-
https://huggingface.co/tgollogly/test/resolve/main/modules/imgproc/src/filter.simd.hpp
- Command line
-
hf download hf://tgollogly/test/modules/imgproc/src/filter.simd.hpp
-
curl -L -o filter.simd.hpp https://huggingface.co/tgollogly/test/resolve/main/modules/imgproc/src/filter.simd.hpp
149 kB
/*M///////////////////////////////////////////////////////////////////////////////////////
//
// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
// By downloading, copying, installing or using the software you agree to this license.
// If you do not agree to this license, do not download, install,
// copy or use the software.
//
//
// License Agreement
// For Open Source Computer Vision Library
//
// Copyright (C) 2000-2008, Intel Corporation, all rights reserved.
// Copyright (C) 2009, Willow Garage Inc., all rights reserved.
// Third party copyrights are property of their respective owners.
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
// * Redistribution's of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
//
// * Redistribution's in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// * The name of the copyright holders may not be used to endorse or promote products
// derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors "as is" and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the Intel Corporation or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/
#include "precomp.hpp"
#include "opencv2/core/hal/intrin.hpp"
#include "filter.hpp"
#include <cstddef>
#if defined(CV_CPU_BASELINE_MODE)
#if IPP_VERSION_X100 >= 710
#define USE_IPP_SEP_FILTERS 1
#else
#undef USE_IPP_SEP_FILTERS
#endif
#endif
/****************************************************************************************\
Base Image Filter
\****************************************************************************************/
namespace cv {
CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN
// forward declarations
int FilterEngine__start(FilterEngine& this_, const Size &_wholeSize, const Size &sz, const Point &ofs);
int FilterEngine__proceed(FilterEngine& this_, const uchar* src, int srcstep, int count,
uchar* dst, int dststep);
void FilterEngine__apply(FilterEngine& this_, const Mat& src, Mat& dst, const Size& wsz, const Point& ofs);
Ptr<BaseRowFilter> getLinearRowFilter(
int srcType, int bufType,
const Mat& kernel, int anchor,
int symmetryType);
Ptr<BaseColumnFilter> getLinearColumnFilter(
int bufType, int dstType,
const Mat& kernel, int anchor,
int symmetryType, double delta,
int bits);
Ptr<BaseFilter> getLinearFilter(
int srcType, int dstType,
const Mat& filter_kernel, Point anchor,
double delta, int bits);
#ifndef CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY
#define VEC_ALIGN CV_MALLOC_ALIGN
int FilterEngine__start(FilterEngine& this_, const Size &_wholeSize, const Size &sz, const Point &ofs)
{
CV_INSTRUMENT_REGION();
int i, j;
this_.wholeSize = _wholeSize;
this_.roi = Rect(ofs, sz);
CV_Assert( this_.roi.x >= 0 && this_.roi.y >= 0 && this_.roi.width >= 0 && this_.roi.height >= 0 &&
this_.roi.x + this_.roi.width <= this_.wholeSize.width &&
this_.roi.y + this_.roi.height <= this_.wholeSize.height );
int esz = (int)getElemSize(this_.srcType);
int bufElemSize = (int)getElemSize(this_.bufType);
const uchar* constVal = !this_.constBorderValue.empty() ? &this_.constBorderValue[0] : 0;
int _maxBufRows = std::max(this_.ksize.height + 3,
std::max(this_.anchor.y,
this_.ksize.height-this_.anchor.y-1)*2+1);
if (this_.maxWidth < this_.roi.width || _maxBufRows != (int)this_.rows.size() )
{
this_.rows.resize(_maxBufRows);
this_.maxWidth = std::max(this_.maxWidth, this_.roi.width);
int cn = CV_MAT_CN(this_.srcType);
this_.srcRow.resize(esz*(this_.maxWidth + this_.ksize.width - 1));
if (this_.columnBorderType == BORDER_CONSTANT)
{
CV_Assert(constVal != NULL);
this_.constBorderRow.resize(getElemSize(this_.bufType)*(this_.maxWidth + this_.ksize.width - 1 + VEC_ALIGN));
uchar *dst = alignPtr(&this_.constBorderRow[0], VEC_ALIGN);
int n = (int)this_.constBorderValue.size();
int N = (this_.maxWidth + this_.ksize.width - 1)*esz;
uchar *tdst = this_.isSeparable() ? &this_.srcRow[0] : dst;
for( i = 0; i < N; i += n )
{
n = std::min( n, N - i );
for(j = 0; j < n; j++)
tdst[i+j] = constVal[j];
}
if (this_.isSeparable())
(*this_.rowFilter)(&this_.srcRow[0], dst, this_.maxWidth, cn);
}
int maxBufStep = bufElemSize*(int)alignSize(this_.maxWidth +
(!this_.isSeparable() ? this_.ksize.width - 1 : 0), VEC_ALIGN);
this_.ringBuf.resize(maxBufStep*this_.rows.size()+VEC_ALIGN);
}
// adjust bufstep so that the used part of the ring buffer stays compact in memory
this_.bufStep = bufElemSize*(int)alignSize(this_.roi.width + (!this_.isSeparable() ? this_.ksize.width - 1 : 0), VEC_ALIGN);
this_.dx1 = std::max(this_.anchor.x - this_.roi.x, 0);
this_.dx2 = std::max(this_.ksize.width - this_.anchor.x - 1 + this_.roi.x + this_.roi.width - this_.wholeSize.width, 0);
// recompute border tables
if (this_.dx1 > 0 || this_.dx2 > 0)
{
if (this_.rowBorderType == BORDER_CONSTANT )
{
CV_Assert(constVal != NULL);
int nr = this_.isSeparable() ? 1 : (int)this_.rows.size();
for( i = 0; i < nr; i++ )
{
uchar* dst = this_.isSeparable() ? &this_.srcRow[0] : alignPtr(&this_.ringBuf[0], VEC_ALIGN) + this_.bufStep*i;
memcpy(dst, constVal, this_.dx1*esz);
memcpy(dst + (this_.roi.width + this_.ksize.width - 1 - this_.dx2)*esz, constVal, this_.dx2*esz);
}
}
else
{
int xofs1 = std::min(this_.roi.x, this_.anchor.x) - this_.roi.x;
int btab_esz = this_.borderElemSize, wholeWidth = this_.wholeSize.width;
int* btab = (int*)&this_.borderTab[0];
for( i = 0; i < this_.dx1; i++ )
{
int p0 = (borderInterpolate(i-this_.dx1, wholeWidth, this_.rowBorderType) + xofs1)*btab_esz;
for( j = 0; j < btab_esz; j++ )
btab[i*btab_esz + j] = p0 + j;
}
for( i = 0; i < this_.dx2; i++ )
{
int p0 = (borderInterpolate(wholeWidth + i, wholeWidth, this_.rowBorderType) + xofs1)*btab_esz;
for( j = 0; j < btab_esz; j++ )
btab[(i + this_.dx1)*btab_esz + j] = p0 + j;
}
}
}
this_.rowCount = this_.dstY = 0;
this_.startY = this_.startY0 = std::max(this_.roi.y - this_.anchor.y, 0);
this_.endY = std::min(this_.roi.y + this_.roi.height + this_.ksize.height - this_.anchor.y - 1, this_.wholeSize.height);
if (this_.columnFilter)
this_.columnFilter->reset();
if (this_.filter2D)
this_.filter2D->reset();
return this_.startY;
}
int FilterEngine__proceed(FilterEngine& this_, const uchar* src, int srcstep, int count,
uchar* dst, int dststep)
{
CV_INSTRUMENT_REGION();
CV_DbgAssert(this_.wholeSize.width > 0 && this_.wholeSize.height > 0 );
const int *btab = &this_.borderTab[0];
int esz = (int)getElemSize(this_.srcType), btab_esz = this_.borderElemSize;
uchar** brows = &this_.rows[0];
int bufRows = (int)this_.rows.size();
int cn = CV_MAT_CN(this_.bufType);
int width = this_.roi.width, kwidth = this_.ksize.width;
int kheight = this_.ksize.height, ay = this_.anchor.y;
int _dx1 = this_.dx1, _dx2 = this_.dx2;
int width1 = this_.roi.width + kwidth - 1;
int xofs1 = std::min(this_.roi.x, this_.anchor.x);
bool isSep = this_.isSeparable();
bool makeBorder = (_dx1 > 0 || _dx2 > 0) && this_.rowBorderType != BORDER_CONSTANT;
int dy = 0, i = 0;
src -= xofs1*esz;
count = std::min(count, this_.remainingInputRows());
CV_Assert(src && dst && count > 0);
for(;; dst += dststep*i, dy += i)
{
int dcount = bufRows - ay - this_.startY - this_.rowCount + this_.roi.y;
dcount = dcount > 0 ? dcount : bufRows - kheight + 1;
dcount = std::min(dcount, count);
count -= dcount;
for( ; dcount-- > 0; src += srcstep )
{
int bi = (this_.startY - this_.startY0 + this_.rowCount) % bufRows;
uchar* brow = alignPtr(&this_.ringBuf[0], VEC_ALIGN) + bi*this_.bufStep;
uchar* row = isSep ? &this_.srcRow[0] : brow;
if (++this_.rowCount > bufRows)
{
--this_.rowCount;
++this_.startY;
}
memcpy( row + _dx1*esz, src, (width1 - _dx2 - _dx1)*esz );
if( makeBorder )
{
if( btab_esz*(int)sizeof(int) == esz )
{
const int* isrc = (const int*)src;
int* irow = (int*)row;
for( i = 0; i < _dx1*btab_esz; i++ )
irow[i] = isrc[btab[i]];
for( i = 0; i < _dx2*btab_esz; i++ )
irow[i + (width1 - _dx2)*btab_esz] = isrc[btab[i+_dx1*btab_esz]];
}
else
{
for( i = 0; i < _dx1*esz; i++ )
row[i] = src[btab[i]];
for( i = 0; i < _dx2*esz; i++ )
row[i + (width1 - _dx2)*esz] = src[btab[i+_dx1*esz]];
}
}
if( isSep )
(*this_.rowFilter)(row, brow, width, CV_MAT_CN(this_.srcType));
}
int max_i = std::min(bufRows, this_.roi.height - (this_.dstY + dy) + (kheight - 1));
for( i = 0; i < max_i; i++ )
{
int srcY = borderInterpolate(this_.dstY + dy + i + this_.roi.y - ay,
this_.wholeSize.height, this_.columnBorderType);
if( srcY < 0 ) // can happen only with constant border type
brows[i] = alignPtr(&this_.constBorderRow[0], VEC_ALIGN);
else
{
CV_Assert(srcY >= this_.startY);
if( srcY >= this_.startY + this_.rowCount)
break;
int bi = (srcY - this_.startY0) % bufRows;
brows[i] = alignPtr(&this_.ringBuf[0], VEC_ALIGN) + bi*this_.bufStep;
}
}
if( i < kheight )
break;
i -= kheight - 1;
if (isSep)
(*this_.columnFilter)((const uchar**)brows, dst, dststep, i, this_.roi.width*cn);
else
(*this_.filter2D)((const uchar**)brows, dst, dststep, i, this_.roi.width, cn);
}
this_.dstY += dy;
CV_Assert(this_.dstY <= this_.roi.height);
return dy;
}
void FilterEngine__apply(FilterEngine& this_, const Mat& src, Mat& dst, const Size& wsz, const Point& ofs)
{
CV_INSTRUMENT_REGION();
CV_DbgAssert(src.type() == this_.srcType && dst.type() == this_.dstType);
FilterEngine__start(this_, wsz, src.size(), ofs);
int y = this_.startY - ofs.y;
FilterEngine__proceed(this_,
src.ptr() + y * (ptrdiff_t)src.step,
(int)src.step,
this_.endY - this_.startY,
dst.ptr(),
(int)dst.step );
}
struct RowNoVec
{
RowNoVec() {}
RowNoVec(const Mat&) {}
int operator()(const uchar*, uchar*, int, int) const { return 0; }
};
struct ColumnNoVec
{
ColumnNoVec() {}
ColumnNoVec(const Mat&, int, int, double) {}
int operator()(const uchar**, uchar*, int) const { return 0; }
};
struct SymmRowSmallNoVec
{
SymmRowSmallNoVec() {}
SymmRowSmallNoVec(const Mat&, int) {}
int operator()(const uchar*, uchar*, int, int) const { return 0; }
};
struct SymmColumnSmallNoVec
{
SymmColumnSmallNoVec() {}
SymmColumnSmallNoVec(const Mat&, int, int, double) {}
int operator()(const uchar**, uchar*, int) const { return 0; }
};
struct FilterNoVec
{
FilterNoVec() {}
FilterNoVec(const Mat&, int, double) {}
int operator()(const uchar**, uchar*, int) const { return 0; }
};
#if (CV_SIMD || CV_SIMD_SCALABLE)
///////////////////////////////////// 8u-16s & 8u-8u //////////////////////////////////
struct RowVec_8u32s
{
RowVec_8u32s() { smallValues = false; }
RowVec_8u32s( const Mat& _kernel )
{
kernel = _kernel;
smallValues = true;
int k, ksize = kernel.rows + kernel.cols - 1;
for( k = 0; k < ksize; k++ )
{
int v = kernel.ptr<int>()[k];
if( v < SHRT_MIN || v > SHRT_MAX )
{
smallValues = false;
break;
}
}
}
int operator()(const uchar* _src, uchar* _dst, int width, int cn) const
{
CV_INSTRUMENT_REGION();
int i = 0, k, _ksize = kernel.rows + kernel.cols - 1;
int* dst = (int*)_dst;
const int* _kx = kernel.ptr<int>();
width *= cn;
if( smallValues )
{
for( ; i <= width - VTraits<v_uint8>::vlanes(); i += VTraits<v_uint8>::vlanes() )
{
const uchar* src = _src + i;
v_int32 s0 = vx_setzero_s32();
v_int32 s1 = vx_setzero_s32();
v_int32 s2 = vx_setzero_s32();
v_int32 s3 = vx_setzero_s32();
k = 0;
for (; k <= _ksize - 2; k += 2, src += 2 * cn)
{
v_int32 f = vx_setall_s32((_kx[k] & 0xFFFF) | (_kx[k + 1] << 16));
v_uint8 x0, x1;
v_zip(vx_load(src), vx_load(src + cn), x0, x1);
s0 = v_add(s0, v_dotprod(v_reinterpret_as_s16(v_expand_low(x0)), v_reinterpret_as_s16(f)));
s1 = v_add(s1, v_dotprod(v_reinterpret_as_s16(v_expand_high(x0)), v_reinterpret_as_s16(f)));
s2 = v_add(s2, v_dotprod(v_reinterpret_as_s16(v_expand_low(x1)), v_reinterpret_as_s16(f)));
s3 = v_add(s3, v_dotprod(v_reinterpret_as_s16(v_expand_high(x1)), v_reinterpret_as_s16(f)));
}
if (k < _ksize)
{
v_int32 f = vx_setall_s32(_kx[k]);
v_uint16 x0, x1;
v_expand(vx_load(src), x0, x1);
s0 = v_add(s0, v_dotprod(v_reinterpret_as_s16(v_expand_low(x0)), v_reinterpret_as_s16(f)));
s1 = v_add(s1, v_dotprod(v_reinterpret_as_s16(v_expand_high(x0)), v_reinterpret_as_s16(f)));
s2 = v_add(s2, v_dotprod(v_reinterpret_as_s16(v_expand_low(x1)), v_reinterpret_as_s16(f)));
s3 = v_add(s3, v_dotprod(v_reinterpret_as_s16(v_expand_high(x1)), v_reinterpret_as_s16(f)));
}
v_store(dst + i, s0);
v_store(dst + i + VTraits<v_int32>::vlanes(), s1);
v_store(dst + i + 2*VTraits<v_int32>::vlanes(), s2);
v_store(dst + i + 3*VTraits<v_int32>::vlanes(), s3);
}
if( i <= width - VTraits<v_uint16>::vlanes() )
{
const uchar* src = _src + i;
v_int32 s0 = vx_setzero_s32();
v_int32 s1 = vx_setzero_s32();
k = 0;
for( ; k <= _ksize - 2; k += 2, src += 2*cn )
{
v_int32 f = vx_setall_s32((_kx[k] & 0xFFFF) | (_kx[k + 1] << 16));
v_uint16 x0, x1;
v_zip(vx_load_expand(src), vx_load_expand(src + cn), x0, x1);
s0 = v_add(s0, v_dotprod(v_reinterpret_as_s16(x0), v_reinterpret_as_s16(f)));
s1 = v_add(s1, v_dotprod(v_reinterpret_as_s16(x1), v_reinterpret_as_s16(f)));
}
if( k < _ksize )
{
v_int32 f = vx_setall_s32(_kx[k]);
v_uint32 x0, x1;
v_expand(vx_load_expand(src), x0, x1);
s0 = v_add(s0, v_dotprod(v_reinterpret_as_s16(x0), v_reinterpret_as_s16(f)));
s1 = v_add(s1, v_dotprod(v_reinterpret_as_s16(x1), v_reinterpret_as_s16(f)));
}
v_store(dst + i, s0);
v_store(dst + i + VTraits<v_int32>::vlanes(), s1);
i += VTraits<v_uint16>::vlanes();
}
if( i <= width - VTraits<v_uint32>::vlanes() )
{
v_int32 d = vx_setzero_s32();
k = 0;
const uchar* src = _src + i;
for (; k <= _ksize - 2; k += 2, src += 2*cn)
{
v_int32 f = vx_setall_s32((_kx[k] & 0xFFFF) | (_kx[k + 1] << 16));
v_uint32 x0, x1;
v_zip(vx_load_expand_q(src), vx_load_expand_q(src + cn), x0, x1);
d = v_add(d, v_dotprod(v_pack(v_reinterpret_as_s32(x0), v_reinterpret_as_s32(x1)), v_reinterpret_as_s16(f)));
}
if (k < _ksize)
d = v_add(d, v_dotprod(v_reinterpret_as_s16(vx_load_expand_q(src)), v_reinterpret_as_s16(vx_setall_s32(_kx[k]))));
v_store(dst + i, d);
i += VTraits<v_uint32>::vlanes();
}
}
return i;
}
Mat kernel;
bool smallValues;
};
struct RowVec_8u32f
{
RowVec_8u32f() {}
RowVec_8u32f( const Mat& _kernel ) : kernel(_kernel) {}
int operator()(const uchar* _src, uchar* _dst, int width, int cn) const
{
CV_INSTRUMENT_REGION();
int i = 0, k, _ksize = kernel.rows + kernel.cols - 1;
float* dst = (float*)_dst;
const float* _kx = kernel.ptr<float>();
width *= cn;
for( ; i <= width - VTraits<v_uint8>::vlanes(); i += VTraits<v_uint8>::vlanes() )
{
v_float32 s0 = vx_setzero_f32();
v_float32 s1 = vx_setzero_f32();
v_float32 s2 = vx_setzero_f32();
v_float32 s3 = vx_setzero_f32();
k = 0;
for( ; k < _ksize ; k++ )
{
v_float32 f = vx_setall_f32(_kx[k]);
const uchar* src = (const uchar*)_src + i + k * cn;
v_float32 vs_ll = v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(src)));
v_float32 vs_lh = v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(src + VTraits<v_float32>::vlanes())));
v_float32 vs_hl = v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(src + 2*VTraits<v_float32>::vlanes())));
v_float32 vs_hh = v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(src + 3*VTraits<v_float32>::vlanes())));
s0 = v_muladd(vs_ll, f, s0);
s1 = v_muladd(vs_lh, f, s1);
s2 = v_muladd(vs_hl, f, s2);
s3 = v_muladd(vs_hh, f, s3);
}
v_store(dst + i, s0);
v_store(dst + i + VTraits<v_float32>::vlanes(), s1);
v_store(dst + i + 2*VTraits<v_float32>::vlanes(), s2);
v_store(dst + i + 3*VTraits<v_float32>::vlanes(), s3);
}
return i;
}
Mat kernel;
};
struct SymmRowSmallVec_8u32s
{
SymmRowSmallVec_8u32s() { smallValues = false; symmetryType = 0; }
SymmRowSmallVec_8u32s( const Mat& _kernel, int _symmetryType )
{
kernel = _kernel;
symmetryType = _symmetryType;
smallValues = true;
int k, ksize = kernel.rows + kernel.cols - 1;
for( k = 0; k < ksize; k++ )
{
int v = kernel.ptr<int>()[k];
if( v < SHRT_MIN || v > SHRT_MAX )
{
smallValues = false;
break;
}
}
}
int operator()(const uchar* src, uchar* _dst, int width, int cn) const
{
CV_INSTRUMENT_REGION();
int i = 0, j, k, _ksize = kernel.rows + kernel.cols - 1;
int* dst = (int*)_dst;
bool symmetrical = (symmetryType & KERNEL_SYMMETRICAL) != 0;
const int* kx = kernel.ptr<int>() + _ksize/2;
if( !smallValues )
return 0;
src += (_ksize/2)*cn;
width *= cn;
if( symmetrical )
{
if( _ksize == 1 )
return 0;
if( _ksize == 3 )
{
if( kx[0] == 2 && kx[1] == 1 )
{
for( ; i <= width - VTraits<v_uint8>::vlanes(); i += VTraits<v_uint8>::vlanes(), src += VTraits<v_uint8>::vlanes() )
{
v_uint16 x0l, x0h, x1l, x1h, x2l, x2h;
v_expand(vx_load(src - cn), x0l, x0h);
v_expand(vx_load(src), x1l, x1h);
v_expand(vx_load(src + cn), x2l, x2h);
x1l = v_add_wrap(v_add_wrap(x1l, x1l), v_add_wrap(x0l, x2l));
x1h = v_add_wrap(v_add_wrap(x1h, x1h), v_add_wrap(x0h, x2h));
v_store(dst + i, v_reinterpret_as_s32(v_expand_low(x1l)));
v_store(dst + i + VTraits<v_int32>::vlanes(), v_reinterpret_as_s32(v_expand_high(x1l)));
v_store(dst + i + 2*VTraits<v_int32>::vlanes(), v_reinterpret_as_s32(v_expand_low(x1h)));
v_store(dst + i + 3*VTraits<v_int32>::vlanes(), v_reinterpret_as_s32(v_expand_high(x1h)));
}
if( i <= width - VTraits<v_uint16>::vlanes() )
{
v_uint16 x = vx_load_expand(src);
x = v_add_wrap(v_add_wrap(x, x), v_add_wrap(vx_load_expand(src - cn), vx_load_expand(src + cn)));
v_store(dst + i, v_reinterpret_as_s32(v_expand_low(x)));
v_store(dst + i + VTraits<v_int32>::vlanes(), v_reinterpret_as_s32(v_expand_high(x)));
i += VTraits<v_uint16>::vlanes(); src += VTraits<v_uint16>::vlanes();
}
if( i <= width - VTraits<v_uint32>::vlanes() )
{
v_uint32 x = vx_load_expand_q(src);
x = v_add(v_add(v_add(x, x), vx_load_expand_q(src - cn)), vx_load_expand_q(src + cn));
v_store(dst + i, v_reinterpret_as_s32(x));
i += VTraits<v_uint32>::vlanes();
}
}
else if( kx[0] == -2 && kx[1] == 1 )
{
for( ; i <= width - VTraits<v_uint8>::vlanes(); i += VTraits<v_uint8>::vlanes(), src += VTraits<v_uint8>::vlanes() )
{
v_uint16 x0l, x0h, x1l, x1h, x2l, x2h;
v_expand(vx_load(src - cn), x0l, x0h);
v_expand(vx_load(src), x1l, x1h);
v_expand(vx_load(src + cn), x2l, x2h);
x1l = v_sub_wrap(v_add_wrap(x0l, x2l), v_add_wrap(x1l, x1l));
x1h = v_sub_wrap(v_add_wrap(x0h, x2h), v_add_wrap(x1h, x1h));
v_store(dst + i, v_expand_low(v_reinterpret_as_s16(x1l)));
v_store(dst + i + VTraits<v_int32>::vlanes(), v_expand_high(v_reinterpret_as_s16(x1l)));
v_store(dst + i + 2*VTraits<v_int32>::vlanes(), v_expand_low(v_reinterpret_as_s16(x1h)));
v_store(dst + i + 3*VTraits<v_int32>::vlanes(), v_expand_high(v_reinterpret_as_s16(x1h)));
}
if( i <= width - VTraits<v_uint16>::vlanes() )
{
v_uint16 x = vx_load_expand(src);
x = v_sub_wrap(v_add_wrap(vx_load_expand(src - cn), vx_load_expand(src + cn)), v_add_wrap(x, x));
v_store(dst + i, v_expand_low(v_reinterpret_as_s16(x)));
v_store(dst + i + VTraits<v_int32>::vlanes(), v_expand_high(v_reinterpret_as_s16(x)));
i += VTraits<v_uint16>::vlanes(); src += VTraits<v_uint16>::vlanes();
}
if( i <= width - VTraits<v_uint32>::vlanes() )
{
v_int32 x = v_reinterpret_as_s32(vx_load_expand_q(src));
x = v_sub(v_reinterpret_as_s32(v_add(vx_load_expand_q(src - cn), vx_load_expand_q(src + cn))), v_add(x, x));
v_store(dst + i, x);
i += VTraits<v_uint32>::vlanes();
}
}
else
{
v_int16 k0 = vx_setall_s16((short)kx[0]);
v_int16 k1 = vx_setall_s16((short)kx[1]);
for( ; i <= width - VTraits<v_uint8>::vlanes(); i += VTraits<v_uint8>::vlanes(), src += VTraits<v_uint8>::vlanes() )
{
v_uint16 x0l, x0h, x1l, x1h, x2l, x2h;
v_expand(vx_load(src - cn), x0l, x0h);
v_expand(vx_load(src), x1l, x1h);
v_expand(vx_load(src + cn), x2l, x2h);
v_int32 dl, dh;
v_int16 x0, x1;
v_mul_expand(v_reinterpret_as_s16(x1l), k0, dl, dh);
v_zip(v_reinterpret_as_s16(x0l), v_reinterpret_as_s16(x2l), x0, x1);
dl = v_add(dl, v_dotprod(x0, k1));
dh = v_add(dh, v_dotprod(x1, k1));
v_store(dst + i, dl);
v_store(dst + i + VTraits<v_int32>::vlanes(), dh);
v_mul_expand(v_reinterpret_as_s16(x1h), k0, dl, dh);
v_zip(v_reinterpret_as_s16(x0h), v_reinterpret_as_s16(x2h), x0, x1);
dl = v_add(dl, v_dotprod(x0, k1));
dh = v_add(dh, v_dotprod(x1, k1));
v_store(dst + i + 2*VTraits<v_int32>::vlanes(), dl);
v_store(dst + i + 3*VTraits<v_int32>::vlanes(), dh);
}
if ( i <= width - VTraits<v_uint16>::vlanes() )
{
v_int32 dl, dh;
v_mul_expand(v_reinterpret_as_s16(vx_load_expand(src)), k0, dl, dh);
v_int16 x0, x1;
v_zip(v_reinterpret_as_s16(vx_load_expand(src - cn)), v_reinterpret_as_s16(vx_load_expand(src + cn)), x0, x1);
dl = v_add(dl, v_dotprod(x0, k1));
dh = v_add(dh, v_dotprod(x1, k1));
v_store(dst + i, dl);
v_store(dst + i + VTraits<v_int32>::vlanes(), dh);
i += VTraits<v_uint16>::vlanes(); src += VTraits<v_uint16>::vlanes();
}
if ( i <= width - VTraits<v_uint32>::vlanes() )
{
v_store(dst + i, v_muladd(v_reinterpret_as_s32(vx_load_expand_q(src)), vx_setall_s32(kx[0]), v_mul(v_reinterpret_as_s32(v_add(vx_load_expand_q(src - cn), vx_load_expand_q(src + cn))), vx_setall_s32(kx[1]))));
i += VTraits<v_uint32>::vlanes();
}
}
}
else if( _ksize == 5 )
{
if( kx[0] == -2 && kx[1] == 0 && kx[2] == 1 )
{
for( ; i <= width - VTraits<v_uint8>::vlanes(); i += VTraits<v_uint8>::vlanes(), src += VTraits<v_uint8>::vlanes() )
{
v_uint16 x0l, x0h, x1l, x1h, x2l, x2h;
v_expand(vx_load(src - 2*cn), x0l, x0h);
v_expand(vx_load(src), x1l, x1h);
v_expand(vx_load(src + 2*cn), x2l, x2h);
x1l = v_sub_wrap(v_add_wrap(x0l, x2l), v_add_wrap(x1l, x1l));
x1h = v_sub_wrap(v_add_wrap(x0h, x2h), v_add_wrap(x1h, x1h));
v_store(dst + i, v_expand_low(v_reinterpret_as_s16(x1l)));
v_store(dst + i + VTraits<v_int32>::vlanes(), v_expand_high(v_reinterpret_as_s16(x1l)));
v_store(dst + i + 2*VTraits<v_int32>::vlanes(), v_expand_low(v_reinterpret_as_s16(x1h)));
v_store(dst + i + 3*VTraits<v_int32>::vlanes(), v_expand_high(v_reinterpret_as_s16(x1h)));
}
if( i <= width - VTraits<v_uint16>::vlanes() )
{
v_uint16 x = vx_load_expand(src);
x = v_sub_wrap(v_add_wrap(vx_load_expand(src - 2*cn), vx_load_expand(src + 2*cn)), v_add_wrap(x, x));
v_store(dst + i, v_expand_low(v_reinterpret_as_s16(x)));
v_store(dst + i + VTraits<v_int32>::vlanes(), v_expand_high(v_reinterpret_as_s16(x)));
i += VTraits<v_uint16>::vlanes(); src += VTraits<v_uint16>::vlanes();
}
if( i <= width - VTraits<v_uint32>::vlanes() )
{
v_int32 x = v_reinterpret_as_s32(vx_load_expand_q(src));
x = v_sub(v_reinterpret_as_s32(v_add(vx_load_expand_q(src - 2 * cn), vx_load_expand_q(src + 2 * cn))), v_add(x, x));
v_store(dst + i, x);
i += VTraits<v_uint32>::vlanes();
}
}
else
{
v_int16 k0 = vx_setall_s16((short)(kx[0]));
v_int16 k12 = v_reinterpret_as_s16(vx_setall_s32((kx[1] & 0xFFFF) | (kx[2] << 16)));
for( ; i <= width - VTraits<v_uint8>::vlanes(); i += VTraits<v_uint8>::vlanes(), src += VTraits<v_uint8>::vlanes() )
{
v_int32 x0, x1, x2, x3;
v_uint16 x0l, x0h, x1l, x1h, x2l, x2h, x3l, x3h;
v_int16 xl, xh;
v_expand(vx_load(src), x0l, x0h);
v_mul_expand(v_reinterpret_as_s16(x0l), k0, x0, x1);
v_mul_expand(v_reinterpret_as_s16(x0h), k0, x2, x3);
v_expand(vx_load(src - cn), x0l, x0h);
v_expand(vx_load(src + cn), x1l, x1h);
v_expand(vx_load(src - 2*cn), x2l, x2h);
v_expand(vx_load(src + 2*cn), x3l, x3h);
v_zip(v_reinterpret_as_s16(v_add(x0l, x1l)), v_reinterpret_as_s16(v_add(x2l, x3l)), xl, xh);
x0 = v_add(x0, v_dotprod(xl, k12));
x1 = v_add(x1, v_dotprod(xh, k12));
v_zip(v_reinterpret_as_s16(v_add(x0h, x1h)), v_reinterpret_as_s16(v_add(x2h, x3h)), xl, xh);
x2 = v_add(x2, v_dotprod(xl, k12));
x3 = v_add(x3, v_dotprod(xh, k12));
v_store(dst + i, x0);
v_store(dst + i + VTraits<v_int32>::vlanes(), x1);
v_store(dst + i + 2*VTraits<v_int32>::vlanes(), x2);
v_store(dst + i + 3*VTraits<v_int32>::vlanes(), x3);
}
if( i <= width - VTraits<v_uint16>::vlanes() )
{
v_int32 x1, x2;
v_mul_expand(v_reinterpret_as_s16(vx_load_expand(src)), k0, x1, x2);
v_int16 xl, xh;
v_zip(v_reinterpret_as_s16(v_add(vx_load_expand(src - cn), vx_load_expand(src + cn))), v_reinterpret_as_s16(v_add(vx_load_expand(src - 2 * cn), vx_load_expand(src + 2 * cn))), xl, xh);
x1 = v_add(x1, v_dotprod(xl, k12));
x2 = v_add(x2, v_dotprod(xh, k12));
v_store(dst + i, x1);
v_store(dst + i + VTraits<v_int32>::vlanes(), x2);
i += VTraits<v_uint16>::vlanes(), src += VTraits<v_uint16>::vlanes();
}
if( i <= width - VTraits<v_uint32>::vlanes() )
{
v_store(dst + i, v_muladd(v_reinterpret_as_s32(vx_load_expand_q(src)), vx_setall_s32(kx[0]),
v_muladd(v_reinterpret_as_s32(v_add(vx_load_expand_q(src - cn), vx_load_expand_q(src + cn))), vx_setall_s32(kx[1]),
v_mul(v_reinterpret_as_s32(v_add(vx_load_expand_q(src - 2 * cn), vx_load_expand_q(src + 2 * cn))), vx_setall_s32(kx[2])))));
i += VTraits<v_uint32>::vlanes();
}
}
}
else
{
v_int16 k0 = vx_setall_s16((short)(kx[0]));
for( ; i <= width - VTraits<v_uint8>::vlanes(); i += VTraits<v_uint8>::vlanes(), src += VTraits<v_uint8>::vlanes() )
{
v_uint8 v_src = vx_load(src);
v_int32 s0, s1, s2, s3;
v_mul_expand(v_reinterpret_as_s16(v_expand_low(v_src)), k0, s0, s1);
v_mul_expand(v_reinterpret_as_s16(v_expand_high(v_src)), k0, s2, s3);
for (k = 1, j = cn; k <= _ksize / 2 - 1; k += 2, j += 2 * cn)
{
v_int16 k12 = v_reinterpret_as_s16(vx_setall_s32((kx[k] & 0xFFFF) | (kx[k + 1] << 16)));
v_uint8 v_src0 = vx_load(src - j);
v_uint8 v_src1 = vx_load(src - j - cn);
v_uint8 v_src2 = vx_load(src + j);
v_uint8 v_src3 = vx_load(src + j + cn);
v_int16 xl, xh;
v_zip(v_reinterpret_as_s16(v_add(v_expand_low(v_src0), v_expand_low(v_src2))), v_reinterpret_as_s16(v_add(v_expand_low(v_src1), v_expand_low(v_src3))), xl, xh);
s0 = v_add(s0, v_dotprod(xl, k12));
s1 = v_add(s1, v_dotprod(xh, k12));
v_zip(v_reinterpret_as_s16(v_add(v_expand_high(v_src0), v_expand_high(v_src2))), v_reinterpret_as_s16(v_add(v_expand_high(v_src1), v_expand_high(v_src3))), xl, xh);
s2 = v_add(s2, v_dotprod(xl, k12));
s3 = v_add(s3, v_dotprod(xh, k12));
}
if( k < _ksize / 2 + 1 )
{
v_int16 k1 = vx_setall_s16((short)(kx[k]));
v_uint8 v_src0 = vx_load(src - j);
v_uint8 v_src1 = vx_load(src + j);
v_int16 xl, xh;
v_zip(v_reinterpret_as_s16(v_expand_low(v_src0)), v_reinterpret_as_s16(v_expand_low(v_src1)), xl, xh);
s0 = v_add(s0, v_dotprod(xl, k1));
s1 = v_add(s1, v_dotprod(xh, k1));
v_zip(v_reinterpret_as_s16(v_expand_high(v_src0)), v_reinterpret_as_s16(v_expand_high(v_src1)), xl, xh);
s2 = v_add(s2, v_dotprod(xl, k1));
s3 = v_add(s3, v_dotprod(xh, k1));
}
v_store(dst + i, s0);
v_store(dst + i + VTraits<v_int32>::vlanes(), s1);
v_store(dst + i + 2*VTraits<v_int32>::vlanes(), s2);
v_store(dst + i + 3*VTraits<v_int32>::vlanes(), s3);
}
if( i <= width - VTraits<v_uint16>::vlanes() )
{
v_int32 s0, s1;
v_mul_expand(v_reinterpret_as_s16(vx_load_expand(src)), k0, s0, s1);
for (k = 1, j = cn; k <= _ksize / 2 - 1; k+=2, j += 2*cn)
{
v_int16 xl, xh;
v_zip(v_reinterpret_as_s16(v_add(vx_load_expand(src - j), vx_load_expand(src + j))), v_reinterpret_as_s16(v_add(vx_load_expand(src - j - cn), vx_load_expand(src + j + cn))), xl, xh);
v_int16 k12 = v_reinterpret_as_s16(vx_setall_s32((kx[k] & 0xFFFF) | (kx[k+1] << 16)));
s0 = v_add(s0, v_dotprod(xl, k12));
s1 = v_add(s1, v_dotprod(xh, k12));
}
if ( k < _ksize / 2 + 1 )
{
v_int16 xl, xh;
v_zip(v_reinterpret_as_s16(vx_load_expand(src - j)), v_reinterpret_as_s16(vx_load_expand(src + j)), xl, xh);
v_int16 k1 = vx_setall_s16((short)(kx[k]));
s0 = v_add(s0, v_dotprod(xl, k1));
s1 = v_add(s1, v_dotprod(xh, k1));
}
v_store(dst + i, s0);
v_store(dst + i + VTraits<v_int32>::vlanes(), s1);
i += VTraits<v_uint16>::vlanes(); src += VTraits<v_uint16>::vlanes();
}
if( i <= width - VTraits<v_uint32>::vlanes() )
{
v_int32 s0 = v_mul(v_reinterpret_as_s32(vx_load_expand_q(src)), vx_setall_s32(kx[0]));
for( k = 1, j = cn; k < _ksize / 2 + 1; k++, j += cn )
s0 = v_muladd(v_reinterpret_as_s32(v_add(vx_load_expand_q(src - j), vx_load_expand_q(src + j))), vx_setall_s32(kx[k]), s0);
v_store(dst + i, s0);
i += VTraits<v_uint32>::vlanes();
}
}
}
else
{
if( _ksize == 3 )
{
if( kx[0] == 0 && kx[1] == 1 )
{
for( ; i <= width - VTraits<v_uint8>::vlanes(); i += VTraits<v_uint8>::vlanes(), src += VTraits<v_uint8>::vlanes() )
{
v_uint16 x0l, x0h, x2l, x2h;
v_expand(vx_load(src - cn), x0l, x0h);
v_expand(vx_load(src + cn), x2l, x2h);
v_int16 dl = v_reinterpret_as_s16(v_sub_wrap(x2l, x0l));
v_int16 dh = v_reinterpret_as_s16(v_sub_wrap(x2h, x0h));
v_store(dst + i, v_expand_low(dl));
v_store(dst + i + VTraits<v_int32>::vlanes(), v_expand_high(dl));
v_store(dst + i + 2*VTraits<v_int32>::vlanes(), v_expand_low(dh));
v_store(dst + i + 3*VTraits<v_int32>::vlanes(), v_expand_high(dh));
}
if( i <= width - VTraits<v_uint16>::vlanes() )
{
v_int16 dl = v_reinterpret_as_s16(v_sub_wrap(vx_load_expand(src + cn), vx_load_expand(src - cn)));
v_store(dst + i, v_expand_low(dl));
v_store(dst + i + VTraits<v_int32>::vlanes(), v_expand_high(dl));
i += VTraits<v_uint16>::vlanes(); src += VTraits<v_uint16>::vlanes();
}
if (i <= width - VTraits<v_uint32>::vlanes())
{
v_store(dst + i, v_sub(v_reinterpret_as_s32(vx_load_expand_q(src + cn)), v_reinterpret_as_s32(vx_load_expand_q(src - cn))));
i += VTraits<v_uint32>::vlanes();
}
}
else
{
v_int16 k0 = v_reinterpret_as_s16(vx_setall_s32((kx[1] & 0xFFFF) | (-kx[1] << 16)));
for( ; i <= width - VTraits<v_uint8>::vlanes(); i += VTraits<v_uint8>::vlanes(), src += VTraits<v_uint8>::vlanes() )
{
v_uint16 x0l, x0h, x2l, x2h;
v_expand(vx_load(src - cn), x0l, x0h);
v_expand(vx_load(src + cn), x2l, x2h);
v_int16 xl, xh;
v_zip(v_reinterpret_as_s16(x2l), v_reinterpret_as_s16(x0l), xl, xh);
v_store(dst + i, v_dotprod(xl, k0));
v_store(dst + i + VTraits<v_int32>::vlanes(), v_dotprod(xh, k0));
v_zip(v_reinterpret_as_s16(x2h), v_reinterpret_as_s16(x0h), xl, xh);
v_store(dst + i + 2*VTraits<v_int32>::vlanes(), v_dotprod(xl, k0));
v_store(dst + i + 3*VTraits<v_int32>::vlanes(), v_dotprod(xh, k0));
}
if( i <= width - VTraits<v_uint16>::vlanes() )
{
v_int16 xl, xh;
v_zip(v_reinterpret_as_s16(vx_load_expand(src + cn)), v_reinterpret_as_s16(vx_load_expand(src - cn)), xl, xh);
v_store(dst + i, v_dotprod(xl, k0));
v_store(dst + i + VTraits<v_int32>::vlanes(), v_dotprod(xh, k0));
i += VTraits<v_uint16>::vlanes(); src += VTraits<v_uint16>::vlanes();
}
if (i <= width - VTraits<v_uint32>::vlanes())
{
v_store(dst + i, v_muladd(v_reinterpret_as_s32(vx_load_expand_q(src + cn)), vx_setall_s32(kx[1]), v_mul(v_reinterpret_as_s32(vx_load_expand_q(src - cn)), vx_setall_s32(-kx[1]))));
i += VTraits<v_uint32>::vlanes();
}
}
}
else if( _ksize == 5 )
{
v_int16 k0 = v_reinterpret_as_s16(vx_setall_s32((kx[1] & 0xFFFF) | (kx[2] << 16)));
for( ; i <= width - VTraits<v_uint8>::vlanes(); i += VTraits<v_uint8>::vlanes(), src += VTraits<v_uint8>::vlanes() )
{
v_uint16 x0l, x0h, x1l, x1h, x2l, x2h, x3l, x3h;
v_expand(vx_load(src - cn), x0l, x0h);
v_expand(vx_load(src - 2*cn), x1l, x1h);
v_expand(vx_load(src + cn), x2l, x2h);
v_expand(vx_load(src + 2*cn), x3l, x3h);
v_int16 x0, x1;
v_zip(v_reinterpret_as_s16(v_sub_wrap(x2l, x0l)), v_reinterpret_as_s16(v_sub_wrap(x3l, x1l)), x0, x1);
v_store(dst + i, v_dotprod(x0, k0));
v_store(dst + i + VTraits<v_int32>::vlanes(), v_dotprod(x1, k0));
v_zip(v_reinterpret_as_s16(v_sub_wrap(x2h, x0h)), v_reinterpret_as_s16(v_sub_wrap(x3h, x1h)), x0, x1);
v_store(dst + i + 2*VTraits<v_int32>::vlanes(), v_dotprod(x0, k0));
v_store(dst + i + 3*VTraits<v_int32>::vlanes(), v_dotprod(x1, k0));
}
if( i <= width - VTraits<v_uint16>::vlanes() )
{
v_int16 x0, x1;
v_zip(v_reinterpret_as_s16(v_sub_wrap(vx_load_expand(src + cn), vx_load_expand(src - cn))),
v_reinterpret_as_s16(v_sub_wrap(vx_load_expand(src + 2*cn), vx_load_expand(src - 2*cn))), x0, x1);
v_store(dst + i, v_dotprod(x0, k0));
v_store(dst + i + VTraits<v_int32>::vlanes(), v_dotprod(x1, k0));
i += VTraits<v_uint16>::vlanes(); src += VTraits<v_uint16>::vlanes();
}
if( i <= width - VTraits<v_uint32>::vlanes() )
{
v_store(dst + i, v_muladd(v_sub(v_reinterpret_as_s32(vx_load_expand_q(src + cn)), v_reinterpret_as_s32(vx_load_expand_q(src - cn))), vx_setall_s32(kx[1]),
v_mul(v_sub(v_reinterpret_as_s32(vx_load_expand_q(src + 2 * cn)), v_reinterpret_as_s32(vx_load_expand_q(src - 2 * cn))), vx_setall_s32(kx[2]))));
i += VTraits<v_uint32>::vlanes();
}
}
else
{
v_int16 k0 = vx_setall_s16((short)(kx[0]));
for( ; i <= width - VTraits<v_uint8>::vlanes(); i += VTraits<v_uint8>::vlanes(), src += VTraits<v_uint8>::vlanes() )
{
v_uint8 v_src = vx_load(src);
v_int32 s0, s1, s2, s3;
v_mul_expand(v_reinterpret_as_s16(v_expand_low(v_src)), k0, s0, s1);
v_mul_expand(v_reinterpret_as_s16(v_expand_high(v_src)), k0, s2, s3);
for( k = 1, j = cn; k <= _ksize / 2 - 1; k += 2, j += 2 * cn )
{
v_int16 k12 = v_reinterpret_as_s16(vx_setall_s32((kx[k] & 0xFFFF) | (kx[k + 1] << 16)));
v_uint8 v_src0 = vx_load(src - j);
v_uint8 v_src1 = vx_load(src - j - cn);
v_uint8 v_src2 = vx_load(src + j);
v_uint8 v_src3 = vx_load(src + j + cn);
v_int16 xl, xh;
v_zip(v_reinterpret_as_s16(v_sub_wrap(v_expand_low(v_src2), v_expand_low(v_src0))), v_reinterpret_as_s16(v_sub_wrap(v_expand_low(v_src3), v_expand_low(v_src1))), xl, xh);
s0 = v_add(s0, v_dotprod(xl, k12));
s1 = v_add(s1, v_dotprod(xh, k12));
v_zip(v_reinterpret_as_s16(v_sub_wrap(v_expand_high(v_src2), v_expand_high(v_src0))), v_reinterpret_as_s16(v_sub_wrap(v_expand_high(v_src3), v_expand_high(v_src1))), xl, xh);
s2 = v_add(s2, v_dotprod(xl, k12));
s3 = v_add(s3, v_dotprod(xh, k12));
}
if( k < _ksize / 2 + 1 )
{
v_int16 k12 = v_reinterpret_as_s16(vx_setall_s32((kx[k] & 0xFFFF) | (-kx[k] << 16)));
v_uint8 v_src0 = vx_load(src - j);
v_uint8 v_src1 = vx_load(src + j);
v_int16 xl, xh;
v_zip(v_reinterpret_as_s16(v_expand_low(v_src1)), v_reinterpret_as_s16(v_expand_low(v_src0)), xl, xh);
s0 = v_add(s0, v_dotprod(xl, k12));
s1 = v_add(s1, v_dotprod(xh, k12));
v_zip(v_reinterpret_as_s16(v_expand_high(v_src1)), v_reinterpret_as_s16(v_expand_high(v_src0)), xl, xh);
s2 = v_add(s2, v_dotprod(xl, k12));
s3 = v_add(s3, v_dotprod(xh, k12));
}
v_store(dst + i, s0);
v_store(dst + i + VTraits<v_int32>::vlanes(), s1);
v_store(dst + i + 2*VTraits<v_int32>::vlanes(), s2);
v_store(dst + i + 3*VTraits<v_int32>::vlanes(), s3);
}
if( i <= width - VTraits<v_uint16>::vlanes() )
{
v_int32 s0, s1;
v_mul_expand(v_reinterpret_as_s16(vx_load_expand(src)), k0, s0, s1);
for( k = 1, j = cn; k <= _ksize / 2 - 1; k += 2, j += 2 * cn )
{
v_int16 xl, xh;
v_zip(v_reinterpret_as_s16(v_sub_wrap(vx_load_expand(src + j), vx_load_expand(src - j))), v_reinterpret_as_s16(v_sub_wrap(vx_load_expand(src + j + cn), vx_load_expand(src - j - cn))), xl, xh);
v_int16 k12 = v_reinterpret_as_s16(vx_setall_s32((kx[k] & 0xFFFF) | (kx[k + 1] << 16)));
s0 = v_add(s0, v_dotprod(xl, k12));
s1 = v_add(s1, v_dotprod(xh, k12));
}
if( k < _ksize / 2 + 1 )
{
v_int16 k1 = v_reinterpret_as_s16(vx_setall_s32((kx[k] & 0xFFFF) | (-kx[k] << 16)));
v_int16 xl, xh;
v_zip(v_reinterpret_as_s16(vx_load_expand(src + j)), v_reinterpret_as_s16(vx_load_expand(src - j)), xl, xh);
s0 = v_add(s0, v_dotprod(xl, k1));
s1 = v_add(s1, v_dotprod(xh, k1));
}
v_store(dst + i, s0);
v_store(dst + i + VTraits<v_int32>::vlanes(), s1);
i += VTraits<v_uint16>::vlanes(); src += VTraits<v_uint16>::vlanes();
}
if( i <= width - VTraits<v_uint32>::vlanes() )
{
v_int32 s0 = v_mul(v_reinterpret_as_s32(vx_load_expand_q(src)), vx_setall_s32(kx[0]));
for (k = 1, j = cn; k < _ksize / 2 + 1; k++, j += cn)
s0 = v_muladd(v_sub(v_reinterpret_as_s32(vx_load_expand_q(src + j)), v_reinterpret_as_s32(vx_load_expand_q(src - j))), vx_setall_s32(kx[k]), s0);
v_store(dst + i, s0);
i += VTraits<v_uint32>::vlanes();
}
}
}
return i;
}
Mat kernel;
int symmetryType;
bool smallValues;
};
struct SymmColumnVec_32s8u
{
SymmColumnVec_32s8u() { symmetryType=0; delta = 0; }
SymmColumnVec_32s8u(const Mat& _kernel, int _symmetryType, int _bits, double _delta)
{
symmetryType = _symmetryType;
_kernel.convertTo(kernel, CV_32F, 1./(1 << _bits), 0);
delta = (float)(_delta/(1 << _bits));
CV_Assert( (symmetryType & (KERNEL_SYMMETRICAL | KERNEL_ASYMMETRICAL)) != 0 );
}
int operator()(const uchar** _src, uchar* dst, int width) const
{
CV_INSTRUMENT_REGION();
int _ksize = kernel.rows + kernel.cols - 1;
if( _ksize == 1 )
return 0;
int ksize2 = _ksize/2;
const float* ky = kernel.ptr<float>() + ksize2;
int i = 0, k;
bool symmetrical = (symmetryType & KERNEL_SYMMETRICAL) != 0;
const int** src = (const int**)_src;
v_float32 d4 = vx_setall_f32(delta);
if( symmetrical )
{
v_float32 f0 = vx_setall_f32(ky[0]);
v_float32 f1 = vx_setall_f32(ky[1]);
for( ; i <= width - VTraits<v_uint8>::vlanes(); i += VTraits<v_uint8>::vlanes() )
{
const int* S = src[0] + i;
v_float32 s0 = v_muladd(v_cvt_f32(vx_load(S)), f0, d4);
v_float32 s1 = v_muladd(v_cvt_f32(vx_load(S + VTraits<v_int32>::vlanes())), f0, d4);
v_float32 s2 = v_muladd(v_cvt_f32(vx_load(S + 2*VTraits<v_int32>::vlanes())), f0, d4);
v_float32 s3 = v_muladd(v_cvt_f32(vx_load(S + 3*VTraits<v_int32>::vlanes())), f0, d4);
const int* S0 = src[1] + i;
const int* S1 = src[-1] + i;
s0 = v_muladd(v_cvt_f32(v_add(vx_load(S0), vx_load(S1))), f1, s0);
s1 = v_muladd(v_cvt_f32(v_add(vx_load(S0 + VTraits<v_int32>::vlanes()), vx_load(S1 + VTraits<v_int32>::vlanes()))), f1, s1);
s2 = v_muladd(v_cvt_f32(v_add(vx_load(S0 + 2 * VTraits<v_int32>::vlanes()), vx_load(S1 + 2 * VTraits<v_int32>::vlanes()))), f1, s2);
s3 = v_muladd(v_cvt_f32(v_add(vx_load(S0 + 3 * VTraits<v_int32>::vlanes()), vx_load(S1 + 3 * VTraits<v_int32>::vlanes()))), f1, s3);
for( k = 2; k <= ksize2; k++ )
{
v_float32 f = vx_setall_f32(ky[k]);
S0 = src[k] + i;
S1 = src[-k] + i;
s0 = v_muladd(v_cvt_f32(v_add(vx_load(S0), vx_load(S1))), f, s0);
s1 = v_muladd(v_cvt_f32(v_add(vx_load(S0 + VTraits<v_int32>::vlanes()), vx_load(S1 + VTraits<v_int32>::vlanes()))), f, s1);
s2 = v_muladd(v_cvt_f32(v_add(vx_load(S0 + 2 * VTraits<v_int32>::vlanes()), vx_load(S1 + 2 * VTraits<v_int32>::vlanes()))), f, s2);
s3 = v_muladd(v_cvt_f32(v_add(vx_load(S0 + 3 * VTraits<v_int32>::vlanes()), vx_load(S1 + 3 * VTraits<v_int32>::vlanes()))), f, s3);
}
v_store(dst + i, v_pack_u(v_pack(v_round(s0), v_round(s1)), v_pack(v_round(s2), v_round(s3))));
}
if( i <= width - VTraits<v_uint16>::vlanes() )
{
const int* S = src[0] + i;
v_float32 s0 = v_muladd(v_cvt_f32(vx_load(S)), f0, d4);
v_float32 s1 = v_muladd(v_cvt_f32(vx_load(S + VTraits<v_int32>::vlanes())), f0, d4);
const int* S0 = src[1] + i;
const int* S1 = src[-1] + i;
s0 = v_muladd(v_cvt_f32(v_add(vx_load(S0), vx_load(S1))), f1, s0);
s1 = v_muladd(v_cvt_f32(v_add(vx_load(S0 + VTraits<v_int32>::vlanes()), vx_load(S1 + VTraits<v_int32>::vlanes()))), f1, s1);
for( k = 2; k <= ksize2; k++ )
{
v_float32 f = vx_setall_f32(ky[k]);
S0 = src[k] + i;
S1 = src[-k] + i;
s0 = v_muladd(v_cvt_f32(v_add(vx_load(S0), vx_load(S1))), f, s0);
s1 = v_muladd(v_cvt_f32(v_add(vx_load(S0 + VTraits<v_int32>::vlanes()), vx_load(S1 + VTraits<v_int32>::vlanes()))), f, s1);
}
v_pack_u_store(dst + i, v_pack(v_round(s0), v_round(s1)));
i += VTraits<v_uint16>::vlanes();
}
}
else
{
v_float32 f1 = vx_setall_f32(ky[1]);
for( ; i <= width - VTraits<v_uint8>::vlanes(); i += VTraits<v_uint8>::vlanes() )
{
const int* S0 = src[1] + i;
const int* S1 = src[-1] + i;
v_float32 s0 = v_muladd(v_cvt_f32(v_sub(vx_load(S0), vx_load(S1))), f1, d4);
v_float32 s1 = v_muladd(v_cvt_f32(v_sub(vx_load(S0 + VTraits<v_int32>::vlanes()), vx_load(S1 + VTraits<v_int32>::vlanes()))), f1, d4);
v_float32 s2 = v_muladd(v_cvt_f32(v_sub(vx_load(S0 + 2 * VTraits<v_int32>::vlanes()), vx_load(S1 + 2 * VTraits<v_int32>::vlanes()))), f1, d4);
v_float32 s3 = v_muladd(v_cvt_f32(v_sub(vx_load(S0 + 3 * VTraits<v_int32>::vlanes()), vx_load(S1 + 3 * VTraits<v_int32>::vlanes()))), f1, d4);
for ( k = 2; k <= ksize2; k++ )
{
v_float32 f = vx_setall_f32(ky[k]);
S0 = src[k] + i;
S1 = src[-k] + i;
s0 = v_muladd(v_cvt_f32(v_sub(vx_load(S0), vx_load(S1))), f, s0);
s1 = v_muladd(v_cvt_f32(v_sub(vx_load(S0 + VTraits<v_int32>::vlanes()), vx_load(S1 + VTraits<v_int32>::vlanes()))), f, s1);
s2 = v_muladd(v_cvt_f32(v_sub(vx_load(S0 + 2 * VTraits<v_int32>::vlanes()), vx_load(S1 + 2 * VTraits<v_int32>::vlanes()))), f, s2);
s3 = v_muladd(v_cvt_f32(v_sub(vx_load(S0 + 3 * VTraits<v_int32>::vlanes()), vx_load(S1 + 3 * VTraits<v_int32>::vlanes()))), f, s3);
}
v_store(dst + i, v_pack_u(v_pack(v_round(s0), v_round(s1)), v_pack(v_round(s2), v_round(s3))));
}
if( i <= width - VTraits<v_uint16>::vlanes() )
{
const int* S0 = src[1] + i;
const int* S1 = src[-1] + i;
v_float32 s0 = v_muladd(v_cvt_f32(v_sub(vx_load(S0), vx_load(S1))), f1, d4);
v_float32 s1 = v_muladd(v_cvt_f32(v_sub(vx_load(S0 + VTraits<v_int32>::vlanes()), vx_load(S1 + VTraits<v_int32>::vlanes()))), f1, d4);
for ( k = 2; k <= ksize2; k++ )
{
v_float32 f = vx_setall_f32(ky[k]);
S0 = src[k] + i;
S1 = src[-k] + i;
s0 = v_muladd(v_cvt_f32(v_sub(vx_load(S0), vx_load(S1))), f, s0);
s1 = v_muladd(v_cvt_f32(v_sub(vx_load(S0 + VTraits<v_int32>::vlanes()), vx_load(S1 + VTraits<v_int32>::vlanes()))), f, s1);
}
v_pack_u_store(dst + i, v_pack(v_round(s0), v_round(s1)));
i += VTraits<v_uint16>::vlanes();
}
}
return i;
}
int symmetryType;
float delta;
Mat kernel;
};
struct SymmColumnVec_32f8u
{
SymmColumnVec_32f8u() { symmetryType = 0; delta = 0; }
SymmColumnVec_32f8u(const Mat& _kernel, int _symmetryType, int, double _delta)
{
symmetryType = _symmetryType;
kernel = _kernel;
delta = (float)_delta;
CV_Assert( (symmetryType & (KERNEL_SYMMETRICAL | KERNEL_ASYMMETRICAL)) != 0 );
}
int operator()(const uchar** _src, uchar* _dst, int width) const
{
CV_INSTRUMENT_REGION();
int _ksize = kernel.rows + kernel.cols - 1;
if( _ksize == 1 ) return 0;
const int ksize2 = _ksize / 2;
const float* ky = kernel.ptr<float>() + ksize2;
int i = 0, k;
bool symmetrical = (symmetryType & KERNEL_SYMMETRICAL) != 0;
const float** src = (const float**)_src;
if( symmetrical )
{
for( ; i <= width - VTraits<v_uint8>::vlanes(); i += VTraits<v_uint8>::vlanes() )
{
v_float32 v_ky0 = vx_setall_f32(ky[0]);
v_float32 v32_delta = vx_setall_f32(delta);
const float* S = src[0] + i;
v_float32 s0 = v_muladd(v_ky0, vx_load(S), v32_delta);
v_float32 s1 = v_muladd(v_ky0, vx_load(S + VTraits<v_float32>::vlanes()), v32_delta);
v_float32 s2 = v_muladd(v_ky0, vx_load(S + 2*VTraits<v_float32>::vlanes()), v32_delta);
v_float32 s3 = v_muladd(v_ky0, vx_load(S + 3*VTraits<v_float32>::vlanes()), v32_delta);
for( k = 1; k <= ksize2; k++ )
{
v_float32 v_kyk = vx_setall_f32(ky[k]);
const float* S0 = src[k] + i;
const float* S1 = src[-k] + i;
s0 = v_muladd(v_kyk, v_add(vx_load(S0), vx_load(S1)), s0);
s1 = v_muladd(v_kyk, v_add(vx_load(S0 + VTraits<v_float32>::vlanes()), vx_load(S1 + VTraits<v_float32>::vlanes())), s1);
s2 = v_muladd(v_kyk, v_add(vx_load(S0 + 2 * VTraits<v_float32>::vlanes()), vx_load(S1 + 2 * VTraits<v_float32>::vlanes())), s2);
s3 = v_muladd(v_kyk, v_add(vx_load(S0 + 3 * VTraits<v_float32>::vlanes()), vx_load(S1 + 3 * VTraits<v_float32>::vlanes())), s3);
}
v_store(_dst + i, v_pack_u(v_pack(v_round(s0), v_round(s1)), v_pack(v_round(s2), v_round(s3))));
}
}
else
{
for( ; i <= width - VTraits<v_uint8>::vlanes(); i += VTraits<v_uint8>::vlanes() )
{
v_float32 s0 = vx_setall_f32(delta);
v_float32 s1 = vx_setall_f32(delta);
v_float32 s2 = vx_setall_f32(delta);
v_float32 s3 = vx_setall_f32(delta);
for( k = 1; k <= ksize2; k++ )
{
v_float32 v_kyk = vx_setall_f32(ky[k]);
const float* S0 = src[k] + i;
const float* S1 = src[-k] + i;
s0 = v_muladd(v_kyk, v_sub(vx_load(S0), vx_load(S1)), s0);
s1 = v_muladd(v_kyk, v_sub(vx_load(S0 + VTraits<v_float32>::vlanes()), vx_load(S1 + VTraits<v_float32>::vlanes())), s1);
s2 = v_muladd(v_kyk, v_sub(vx_load(S0 + 2 * VTraits<v_float32>::vlanes()), vx_load(S1 + 2 * VTraits<v_float32>::vlanes())), s2);
s3 = v_muladd(v_kyk, v_sub(vx_load(S0 + 3 * VTraits<v_float32>::vlanes()), vx_load(S1 + 3 * VTraits<v_float32>::vlanes())), s3);
}
v_store(_dst + i, v_pack_u(v_pack(v_round(s0), v_round(s1)), v_pack(v_round(s2), v_round(s3))));
}
}
return i;
}
int symmetryType;
float delta;
Mat kernel;
};
struct SymmColumnSmallVec_32s16s
{
SymmColumnSmallVec_32s16s() { symmetryType=0; delta = 0; }
SymmColumnSmallVec_32s16s(const Mat& _kernel, int _symmetryType, int _bits, double _delta)
{
symmetryType = _symmetryType;
_kernel.convertTo(kernel, CV_32F, 1./(1 << _bits), 0);
delta = (float)(_delta/(1 << _bits));
CV_Assert( (symmetryType & (KERNEL_SYMMETRICAL | KERNEL_ASYMMETRICAL)) != 0 );
}
int operator()(const uchar** _src, uchar* _dst, int width) const
{
CV_INSTRUMENT_REGION();
int ksize2 = (kernel.rows + kernel.cols - 1)/2;
const float* ky = kernel.ptr<float>() + ksize2;
int i = 0;
bool symmetrical = (symmetryType & KERNEL_SYMMETRICAL) != 0;
const int** src = (const int**)_src;
const int *S0 = src[-1], *S1 = src[0], *S2 = src[1];
short* dst = (short*)_dst;
v_float32 df4 = vx_setall_f32(delta);
int d = cvRound(delta);
v_int16 d8 = vx_setall_s16((short)d);
if( symmetrical )
{
if( ky[0] == 2 && ky[1] == 1 )
{
for( ; i <= width - 2*VTraits<v_int16>::vlanes(); i += 2*VTraits<v_int16>::vlanes() )
{
v_int32 s0 = vx_load(S1 + i);
v_int32 s1 = vx_load(S1 + i + VTraits<v_int32>::vlanes());
v_int32 s2 = vx_load(S1 + i + 2*VTraits<v_int32>::vlanes());
v_int32 s3 = vx_load(S1 + i + 3*VTraits<v_int32>::vlanes());
v_store(dst + i, v_add(v_pack(v_add(v_add(vx_load(S0 + i), vx_load(S2 + i)), v_add(s0, s0)), v_add(v_add(vx_load(S0 + i + VTraits<v_int32>::vlanes()), vx_load(S2 + i + VTraits<v_int32>::vlanes())), v_add(s1, s1))), d8));
v_store(dst + i + VTraits<v_int16>::vlanes(), v_add(v_pack(v_add(v_add(vx_load(S0 + i + 2 * VTraits<v_int32>::vlanes()), vx_load(S2 + i + 2 * VTraits<v_int32>::vlanes())), v_add(s2, s2)), v_add(v_add(vx_load(S0 + i + 3 * VTraits<v_int32>::vlanes()), vx_load(S2 + i + 3 * VTraits<v_int32>::vlanes())), v_add(s3, s3))), d8));
}
if( i <= width - VTraits<v_int16>::vlanes() )
{
v_int32 sl = vx_load(S1 + i);
v_int32 sh = vx_load(S1 + i + VTraits<v_int32>::vlanes());
v_store(dst + i, v_add(v_pack(v_add(v_add(vx_load(S0 + i), vx_load(S2 + i)), v_add(sl, sl)), v_add(v_add(vx_load(S0 + i + VTraits<v_int32>::vlanes()), vx_load(S2 + i + VTraits<v_int32>::vlanes())), v_add(sh, sh))), d8));
i += VTraits<v_int16>::vlanes();
}
if( i <= width - VTraits<v_int32>::vlanes() )
{
v_int32 s = vx_load(S1 + i);
v_pack_store(dst + i, v_add(v_add(v_add(vx_load(S0 + i), vx_load(S2 + i)), vx_setall_s32(d)), v_add(s, s)));
i += VTraits<v_int32>::vlanes();
}
}
else if( ky[0] == -2 && ky[1] == 1 )
{
for( ; i <= width - 2*VTraits<v_int16>::vlanes(); i += 2*VTraits<v_int16>::vlanes() )
{
v_int32 s0 = vx_load(S1 + i);
v_int32 s1 = vx_load(S1 + i + VTraits<v_int32>::vlanes());
v_int32 s2 = vx_load(S1 + i + 2*VTraits<v_int32>::vlanes());
v_int32 s3 = vx_load(S1 + i + 3*VTraits<v_int32>::vlanes());
v_store(dst + i, v_add(v_pack(v_sub(v_add(vx_load(S0 + i), vx_load(S2 + i)), v_add(s0, s0)), v_sub(v_add(vx_load(S0 + i + VTraits<v_int32>::vlanes()), vx_load(S2 + i + VTraits<v_int32>::vlanes())), v_add(s1, s1))), d8));
v_store(dst + i + VTraits<v_int16>::vlanes(), v_add(v_pack(v_sub(v_add(vx_load(S0 + i + 2 * VTraits<v_int32>::vlanes()), vx_load(S2 + i + 2 * VTraits<v_int32>::vlanes())), v_add(s2, s2)), v_sub(v_add(vx_load(S0 + i + 3 * VTraits<v_int32>::vlanes()), vx_load(S2 + i + 3 * VTraits<v_int32>::vlanes())), v_add(s3, s3))), d8));
}
if( i <= width - VTraits<v_int16>::vlanes() )
{
v_int32 sl = vx_load(S1 + i);
v_int32 sh = vx_load(S1 + i + VTraits<v_int32>::vlanes());
v_store(dst + i, v_add(v_pack(v_sub(v_add(vx_load(S0 + i), vx_load(S2 + i)), v_add(sl, sl)), v_sub(v_add(vx_load(S0 + i + VTraits<v_int32>::vlanes()), vx_load(S2 + i + VTraits<v_int32>::vlanes())), v_add(sh, sh))), d8));
i += VTraits<v_int16>::vlanes();
}
if( i <= width - VTraits<v_int32>::vlanes() )
{
v_int32 s = vx_load(S1 + i);
v_pack_store(dst + i, v_sub(v_add(v_add(vx_load(S0 + i), vx_load(S2 + i)), vx_setall_s32(d)), v_add(s, s)));
i += VTraits<v_int32>::vlanes();
}
}
#if CV_NEON
else if( ky[0] == (float)((int)ky[0]) && ky[1] == (float)((int)ky[1]) )
{
v_int32 k0 = vx_setall_s32((int)ky[0]), k1 = vx_setall_s32((int)ky[1]);
v_int32 d4 = vx_setall_s32(d);
for( ; i <= width - 2*VTraits<v_int16>::vlanes(); i += 2*VTraits<v_int16>::vlanes() )
{
v_store(dst + i, v_pack(v_muladd(v_add(vx_load(S0 + i), vx_load(S2 + i)), k1, v_muladd(vx_load(S1 + i), k0, d4)),
v_muladd(v_add(vx_load(S0 + i + VTraits<v_int32>::vlanes()), vx_load(S2 + i + VTraits<v_int32>::vlanes())), k1, v_muladd(vx_load(S1 + i + VTraits<v_int32>::vlanes()), k0, d4))));
v_store(dst + i + VTraits<v_int16>::vlanes(), v_pack(v_muladd(v_add(vx_load(S0 + i + 2 * VTraits<v_int32>::vlanes()), vx_load(S2 + i + 2 * VTraits<v_int32>::vlanes())), k1, v_muladd(vx_load(S1 + i + 2*VTraits<v_int32>::vlanes()), k0, d4)),
v_muladd(v_add(vx_load(S0 + i + 3 * VTraits<v_int32>::vlanes()), vx_load(S2 + i + 3 * VTraits<v_int32>::vlanes())), k1, v_muladd(vx_load(S1 + i + 3*VTraits<v_int32>::vlanes()), k0, d4))));
}
if( i <= width - VTraits<v_int16>::vlanes() )
{
v_store(dst + i, v_pack(v_muladd(v_add(vx_load(S0 + i), vx_load(S2 + i)), k1, v_muladd(vx_load(S1 + i), k0, d4)),
v_muladd(v_add(vx_load(S0 + i + VTraits<v_int32>::vlanes()), vx_load(S2 + i + VTraits<v_int32>::vlanes())), k1, v_muladd(vx_load(S1 + i + VTraits<v_int32>::vlanes()), k0, d4))));
i += VTraits<v_int16>::vlanes();
}
if( i <= width - VTraits<v_int32>::vlanes() )
{
v_pack_store(dst + i, v_muladd(v_add(vx_load(S0 + i), vx_load(S2 + i)), k1, v_muladd(vx_load(S1 + i), k0, d4)));
i += VTraits<v_int32>::vlanes();
}
}
#endif
else
{
v_float32 k0 = vx_setall_f32(ky[0]), k1 = vx_setall_f32(ky[1]);
for( ; i <= width - 2*VTraits<v_int16>::vlanes(); i += 2*VTraits<v_int16>::vlanes() )
{
v_store(dst + i, v_pack(v_round(v_muladd(v_cvt_f32(v_add(vx_load(S0 + i), vx_load(S2 + i))), k1, v_muladd(v_cvt_f32(vx_load(S1 + i)), k0, df4))),
v_round(v_muladd(v_cvt_f32(v_add(vx_load(S0 + i + VTraits<v_int32>::vlanes()), vx_load(S2 + i + VTraits<v_int32>::vlanes()))), k1, v_muladd(v_cvt_f32(vx_load(S1 + i + VTraits<v_int32>::vlanes())), k0, df4)))));
v_store(dst + i + VTraits<v_int16>::vlanes(), v_pack(v_round(v_muladd(v_cvt_f32(v_add(vx_load(S0 + i + 2 * VTraits<v_int32>::vlanes()), vx_load(S2 + i + 2 * VTraits<v_int32>::vlanes()))), k1, v_muladd(v_cvt_f32(vx_load(S1 + i + 2*VTraits<v_int32>::vlanes())), k0, df4))),
v_round(v_muladd(v_cvt_f32(v_add(vx_load(S0 + i + 3 * VTraits<v_int32>::vlanes()), vx_load(S2 + i + 3 * VTraits<v_int32>::vlanes()))), k1, v_muladd(v_cvt_f32(vx_load(S1 + i + 3*VTraits<v_int32>::vlanes())), k0, df4)))));
}
if( i <= width - VTraits<v_int16>::vlanes() )
{
v_store(dst + i, v_pack(v_round(v_muladd(v_cvt_f32(v_add(vx_load(S0 + i), vx_load(S2 + i))), k1, v_muladd(v_cvt_f32(vx_load(S1 + i)), k0, df4))),
v_round(v_muladd(v_cvt_f32(v_add(vx_load(S0 + i + VTraits<v_int32>::vlanes()), vx_load(S2 + i + VTraits<v_int32>::vlanes()))), k1, v_muladd(v_cvt_f32(vx_load(S1 + i + VTraits<v_int32>::vlanes())), k0, df4)))));
i += VTraits<v_int16>::vlanes();
}
if( i <= width - VTraits<v_int32>::vlanes() )
{
v_pack_store(dst + i, v_round(v_muladd(v_cvt_f32(v_add(vx_load(S0 + i), vx_load(S2 + i))), k1, v_muladd(v_cvt_f32(vx_load(S1 + i)), k0, df4))));
i += VTraits<v_int32>::vlanes();
}
}
}
else
{
if( fabs(ky[1]) == 1 && ky[1] == -ky[-1] )
{
if( ky[1] < 0 )
std::swap(S0, S2);
for( ; i <= width - 2*VTraits<v_int16>::vlanes(); i += 2*VTraits<v_int16>::vlanes() )
{
v_store(dst + i, v_add(v_pack(v_sub(vx_load(S2 + i), vx_load(S0 + i)), v_sub(vx_load(S2 + i + VTraits<v_int32>::vlanes()), vx_load(S0 + i + VTraits<v_int32>::vlanes()))), d8));
v_store(dst + i + VTraits<v_int16>::vlanes(), v_add(v_pack(v_sub(vx_load(S2 + i + 2 * VTraits<v_int32>::vlanes()), vx_load(S0 + i + 2 * VTraits<v_int32>::vlanes())), v_sub(vx_load(S2 + i + 3 * VTraits<v_int32>::vlanes()), vx_load(S0 + i + 3 * VTraits<v_int32>::vlanes()))), d8));
}
if( i <= width - VTraits<v_int16>::vlanes() )
{
v_store(dst + i, v_add(v_pack(v_sub(vx_load(S2 + i), vx_load(S0 + i)), v_sub(vx_load(S2 + i + VTraits<v_int32>::vlanes()), vx_load(S0 + i + VTraits<v_int32>::vlanes()))), d8));
i += VTraits<v_int16>::vlanes();
}
if( i <= width - VTraits<v_int32>::vlanes() )
{
v_pack_store(dst + i, v_add(v_sub(vx_load(S2 + i), vx_load(S0 + i)), vx_setall_s32(d)));
i += VTraits<v_int32>::vlanes();
}
}
else
{
v_float32 k1 = vx_setall_f32(ky[1]);
for( ; i <= width - 2*VTraits<v_int16>::vlanes(); i += 2*VTraits<v_int16>::vlanes() )
{
v_store(dst + i, v_pack(v_round(v_muladd(v_cvt_f32(v_sub(vx_load(S2 + i), vx_load(S0 + i))), k1, df4)),
v_round(v_muladd(v_cvt_f32(v_sub(vx_load(S2 + i + VTraits<v_int32>::vlanes()), vx_load(S0 + i + VTraits<v_int32>::vlanes()))), k1, df4))));
v_store(dst + i + VTraits<v_int16>::vlanes(), v_pack(v_round(v_muladd(v_cvt_f32(v_sub(vx_load(S2 + i + 2 * VTraits<v_int32>::vlanes()), vx_load(S0 + i + 2 * VTraits<v_int32>::vlanes()))), k1, df4)),
v_round(v_muladd(v_cvt_f32(v_sub(vx_load(S2 + i + 3 * VTraits<v_int32>::vlanes()), vx_load(S0 + i + 3 * VTraits<v_int32>::vlanes()))), k1, df4))));
}
if( i <= width - VTraits<v_int16>::vlanes() )
{
v_store(dst + i, v_pack(v_round(v_muladd(v_cvt_f32(v_sub(vx_load(S2 + i), vx_load(S0 + i))), k1, df4)),
v_round(v_muladd(v_cvt_f32(v_sub(vx_load(S2 + i + VTraits<v_int32>::vlanes()), vx_load(S0 + i + VTraits<v_int32>::vlanes()))), k1, df4))));
i += VTraits<v_int16>::vlanes();
}
if( i <= width - VTraits<v_int32>::vlanes() )
{
v_pack_store(dst + i, v_round(v_muladd(v_cvt_f32(v_sub(vx_load(S2 + i), vx_load(S0 + i))), k1, df4)));
i += VTraits<v_int32>::vlanes();
}
}
}
return i;
}
int symmetryType;
float delta;
Mat kernel;
};
/////////////////////////////////////// 16s //////////////////////////////////
struct RowVec_16s32f
{
RowVec_16s32f() {}
RowVec_16s32f( const Mat& _kernel )
{
kernel = _kernel;
}
int operator()(const uchar* _src, uchar* _dst, int width, int cn) const
{
CV_INSTRUMENT_REGION();
int i = 0, k, _ksize = kernel.rows + kernel.cols - 1;
float* dst = (float*)_dst;
const float* _kx = kernel.ptr<float>();
width *= cn;
for( ; i <= width - 2*VTraits<v_int16>::vlanes(); i += 2*VTraits<v_int16>::vlanes() )
{
const short* src = (const short*)_src + i;
v_float32 s0 = vx_setzero_f32();
v_float32 s1 = vx_setzero_f32();
v_float32 s2 = vx_setzero_f32();
v_float32 s3 = vx_setzero_f32();
for( k = 0; k < _ksize; k++, src += cn )
{
v_float32 f = vx_setall_f32(_kx[k]);
v_int16 xl = vx_load(src);
v_int16 xh = vx_load(src + VTraits<v_int16>::vlanes());
s0 = v_muladd(v_cvt_f32(v_expand_low(xl)), f, s0);
s1 = v_muladd(v_cvt_f32(v_expand_high(xl)), f, s1);
s2 = v_muladd(v_cvt_f32(v_expand_low(xh)), f, s2);
s3 = v_muladd(v_cvt_f32(v_expand_high(xh)), f, s3);
}
v_store(dst + i, s0);
v_store(dst + i + VTraits<v_float32>::vlanes(), s1);
v_store(dst + i + 2*VTraits<v_float32>::vlanes(), s2);
v_store(dst + i + 3*VTraits<v_float32>::vlanes(), s3);
}
if( i <= width - VTraits<v_int16>::vlanes() )
{
const short* src = (const short*)_src + i;
v_float32 s0 = vx_setzero_f32();
v_float32 s1 = vx_setzero_f32();
for( k = 0; k < _ksize; k++, src += cn )
{
v_float32 f = vx_setall_f32(_kx[k]);
v_int16 x = vx_load(src);
s0 = v_muladd(v_cvt_f32(v_expand_low(x)), f, s0);
s1 = v_muladd(v_cvt_f32(v_expand_high(x)), f, s1);
}
v_store(dst + i, s0);
v_store(dst + i + VTraits<v_float32>::vlanes(), s1);
i += VTraits<v_int16>::vlanes();
}
if( i <= width - VTraits<v_float32>::vlanes() )
{
const short* src = (const short*)_src + i;
v_float32 s0 = vx_setzero_f32();
for( k = 0; k < _ksize; k++, src += cn )
s0 = v_muladd(v_cvt_f32(vx_load_expand(src)), vx_setall_f32(_kx[k]), s0);
v_store(dst + i, s0);
i += VTraits<v_float32>::vlanes();
}
return i;
}
Mat kernel;
};
struct SymmColumnVec_32f16s
{
SymmColumnVec_32f16s() { symmetryType=0; delta = 0; }
SymmColumnVec_32f16s(const Mat& _kernel, int _symmetryType, int, double _delta)
{
symmetryType = _symmetryType;
kernel = _kernel;
delta = (float)_delta;
CV_Assert( (symmetryType & (KERNEL_SYMMETRICAL | KERNEL_ASYMMETRICAL)) != 0 );
}
int operator()(const uchar** _src, uchar* _dst, int width) const
{
CV_INSTRUMENT_REGION();
int _ksize = kernel.rows + kernel.cols - 1;
if( _ksize == 1 )
return 0;
int ksize2 = _ksize / 2;
const float* ky = kernel.ptr<float>() + ksize2;
int i = 0, k;
bool symmetrical = (symmetryType & KERNEL_SYMMETRICAL) != 0;
const float** src = (const float**)_src;
short* dst = (short*)_dst;
v_float32 d4 = vx_setall_f32(delta);
if( symmetrical )
{
v_float32 k0 = vx_setall_f32(ky[0]);
v_float32 k1 = vx_setall_f32(ky[1]);
for( ; i <= width - 2*VTraits<v_int16>::vlanes(); i += 2*VTraits<v_int16>::vlanes() )
{
v_float32 s0 = v_muladd(vx_load(src[0] + i), k0, d4);
v_float32 s1 = v_muladd(vx_load(src[0] + i + VTraits<v_float32>::vlanes()), k0, d4);
v_float32 s2 = v_muladd(vx_load(src[0] + i + 2*VTraits<v_float32>::vlanes()), k0, d4);
v_float32 s3 = v_muladd(vx_load(src[0] + i + 3*VTraits<v_float32>::vlanes()), k0, d4);
s0 = v_muladd(v_add(vx_load(src[1] + i), vx_load(src[-1] + i)), k1, s0);
s1 = v_muladd(v_add(vx_load(src[1] + i + VTraits<v_float32>::vlanes()), vx_load(src[-1] + i + VTraits<v_float32>::vlanes())), k1, s1);
s2 = v_muladd(v_add(vx_load(src[1] + i + 2 * VTraits<v_float32>::vlanes()), vx_load(src[-1] + i + 2 * VTraits<v_float32>::vlanes())), k1, s2);
s3 = v_muladd(v_add(vx_load(src[1] + i + 3 * VTraits<v_float32>::vlanes()), vx_load(src[-1] + i + 3 * VTraits<v_float32>::vlanes())), k1, s3);
for( k = 2; k <= ksize2; k++ )
{
v_float32 k2 = vx_setall_f32(ky[k]);
s0 = v_muladd(v_add(vx_load(src[k] + i), vx_load(src[-k] + i)), k2, s0);
s1 = v_muladd(v_add(vx_load(src[k] + i + VTraits<v_float32>::vlanes()), vx_load(src[-k] + i + VTraits<v_float32>::vlanes())), k2, s1);
s2 = v_muladd(v_add(vx_load(src[k] + i + 2 * VTraits<v_float32>::vlanes()), vx_load(src[-k] + i + 2 * VTraits<v_float32>::vlanes())), k2, s2);
s3 = v_muladd(v_add(vx_load(src[k] + i + 3 * VTraits<v_float32>::vlanes()), vx_load(src[-k] + i + 3 * VTraits<v_float32>::vlanes())), k2, s3);
}
v_store(dst + i, v_pack(v_round(s0), v_round(s1)));
v_store(dst + i + VTraits<v_int16>::vlanes(), v_pack(v_round(s2), v_round(s3)));
}
if( i <= width - VTraits<v_int16>::vlanes() )
{
v_float32 s0 = v_muladd(vx_load(src[0] + i), k0, d4);
v_float32 s1 = v_muladd(vx_load(src[0] + i + VTraits<v_float32>::vlanes()), k0, d4);
s0 = v_muladd(v_add(vx_load(src[1] + i), vx_load(src[-1] + i)), k1, s0);
s1 = v_muladd(v_add(vx_load(src[1] + i + VTraits<v_float32>::vlanes()), vx_load(src[-1] + i + VTraits<v_float32>::vlanes())), k1, s1);
for( k = 2; k <= ksize2; k++ )
{
v_float32 k2 = vx_setall_f32(ky[k]);
s0 = v_muladd(v_add(vx_load(src[k] + i), vx_load(src[-k] + i)), k2, s0);
s1 = v_muladd(v_add(vx_load(src[k] + i + VTraits<v_float32>::vlanes()), vx_load(src[-k] + i + VTraits<v_float32>::vlanes())), k2, s1);
}
v_store(dst + i, v_pack(v_round(s0), v_round(s1)));
i += VTraits<v_int16>::vlanes();
}
if( i <= width - VTraits<v_float32>::vlanes() )
{
v_float32 s0 = v_muladd(vx_load(src[0] + i), k0, d4);
s0 = v_muladd(v_add(vx_load(src[1] + i), vx_load(src[-1] + i)), k1, s0);
for( k = 2; k <= ksize2; k++ )
s0 = v_muladd(v_add(vx_load(src[k] + i), vx_load(src[-k] + i)), vx_setall_f32(ky[k]), s0);
v_pack_store(dst + i, v_round(s0));
i += VTraits<v_float32>::vlanes();
}
}
else
{
v_float32 k1 = vx_setall_f32(ky[1]);
for( ; i <= width - 2*VTraits<v_int16>::vlanes(); i += 2*VTraits<v_int16>::vlanes() )
{
v_float32 s0 = v_muladd(v_sub(vx_load(src[1] + i), vx_load(src[-1] + i)), k1, d4);
v_float32 s1 = v_muladd(v_sub(vx_load(src[1] + i + VTraits<v_float32>::vlanes()), vx_load(src[-1] + i + VTraits<v_float32>::vlanes())), k1, d4);
v_float32 s2 = v_muladd(v_sub(vx_load(src[1] + i + 2 * VTraits<v_float32>::vlanes()), vx_load(src[-1] + i + 2 * VTraits<v_float32>::vlanes())), k1, d4);
v_float32 s3 = v_muladd(v_sub(vx_load(src[1] + i + 3 * VTraits<v_float32>::vlanes()), vx_load(src[-1] + i + 3 * VTraits<v_float32>::vlanes())), k1, d4);
for( k = 2; k <= ksize2; k++ )
{
v_float32 k2 = vx_setall_f32(ky[k]);
s0 = v_muladd(v_sub(vx_load(src[k] + i), vx_load(src[-k] + i)), k2, s0);
s1 = v_muladd(v_sub(vx_load(src[k] + i + VTraits<v_float32>::vlanes()), vx_load(src[-k] + i + VTraits<v_float32>::vlanes())), k2, s1);
s2 = v_muladd(v_sub(vx_load(src[k] + i + 2 * VTraits<v_float32>::vlanes()), vx_load(src[-k] + i + 2 * VTraits<v_float32>::vlanes())), k2, s2);
s3 = v_muladd(v_sub(vx_load(src[k] + i + 3 * VTraits<v_float32>::vlanes()), vx_load(src[-k] + i + 3 * VTraits<v_float32>::vlanes())), k2, s3);
}
v_store(dst + i, v_pack(v_round(s0), v_round(s1)));
v_store(dst + i + VTraits<v_int16>::vlanes(), v_pack(v_round(s2), v_round(s3)));
}
if( i <= width - VTraits<v_int16>::vlanes() )
{
v_float32 s0 = v_muladd(v_sub(vx_load(src[1] + i), vx_load(src[-1] + i)), k1, d4);
v_float32 s1 = v_muladd(v_sub(vx_load(src[1] + i + VTraits<v_float32>::vlanes()), vx_load(src[-1] + i + VTraits<v_float32>::vlanes())), k1, d4);
for( k = 2; k <= ksize2; k++ )
{
v_float32 k2 = vx_setall_f32(ky[k]);
s0 = v_muladd(v_sub(vx_load(src[k] + i), vx_load(src[-k] + i)), k2, s0);
s1 = v_muladd(v_sub(vx_load(src[k] + i + VTraits<v_float32>::vlanes()), vx_load(src[-k] + i + VTraits<v_float32>::vlanes())), k2, s1);
}
v_store(dst + i, v_pack(v_round(s0), v_round(s1)));
i += VTraits<v_int16>::vlanes();
}
if( i <= width - VTraits<v_float32>::vlanes() )
{
v_float32 s0 = v_muladd(v_sub(vx_load(src[1] + i), vx_load(src[-1] + i)), k1, d4);
for( k = 2; k <= ksize2; k++ )
s0 = v_muladd(v_sub(vx_load(src[k] + i), vx_load(src[-k] + i)), vx_setall_f32(ky[k]), s0);
v_pack_store(dst + i, v_round(s0));
i += VTraits<v_float32>::vlanes();
}
}
return i;
}
int symmetryType;
float delta;
Mat kernel;
};
/////////////////////////////////////// 32f //////////////////////////////////
struct RowVec_32f
{
RowVec_32f()
{
#if defined USE_IPP_SEP_FILTERS
bufsz = -1;
#endif
}
RowVec_32f( const Mat& _kernel )
{
kernel = _kernel;
#if defined USE_IPP_SEP_FILTERS
bufsz = -1;
#endif
}
int operator()(const uchar* _src, uchar* _dst, int width, int cn) const
{
CV_INSTRUMENT_REGION();
#if defined USE_IPP_SEP_FILTERS
CV_IPP_CHECK()
{
int ret = ippiOperator(_src, _dst, width, cn);
if (ret > 0)
return ret;
}
#endif
int _ksize = kernel.rows + kernel.cols - 1;
CV_DbgAssert(_ksize > 0);
const float* src0 = (const float*)_src;
float* dst = (float*)_dst;
const float* _kx = kernel.ptr<float>();
int i = 0, k;
width *= cn;
#if CV_AVX
for (; i <= width - 8; i += 8)
{
const float* src = src0 + i;
__m256 f, x0;
__m256 s0 = _mm256_set1_ps(0.0f);
for (k = 0; k < _ksize; k++, src += cn)
{
f = _mm256_set1_ps(_kx[k]);
x0 = _mm256_loadu_ps(src);
#if CV_FMA3
s0 = _mm256_fmadd_ps(x0, f, s0);
#else
s0 = _mm256_add_ps(s0, _mm256_mul_ps(x0, f));
#endif
}
_mm256_storeu_ps(dst + i, s0);
}
#endif
v_float32 k0 = vx_setall_f32(_kx[0]);
for( ; i <= width - 4*VTraits<v_float32>::vlanes(); i += 4*VTraits<v_float32>::vlanes() )
{
const float* src = src0 + i;
v_float32 s0 = v_mul(vx_load(src), k0);
v_float32 s1 = v_mul(vx_load(src + VTraits<v_float32>::vlanes()), k0);
v_float32 s2 = v_mul(vx_load(src + 2 * VTraits<v_float32>::vlanes()), k0);
v_float32 s3 = v_mul(vx_load(src + 3 * VTraits<v_float32>::vlanes()), k0);
src += cn;
for( k = 1; k < _ksize; k++, src += cn )
{
v_float32 k1 = vx_setall_f32(_kx[k]);
s0 = v_muladd(vx_load(src), k1, s0);
s1 = v_muladd(vx_load(src + VTraits<v_float32>::vlanes()), k1, s1);
s2 = v_muladd(vx_load(src + 2*VTraits<v_float32>::vlanes()), k1, s2);
s3 = v_muladd(vx_load(src + 3*VTraits<v_float32>::vlanes()), k1, s3);
}
v_store(dst + i, s0);
v_store(dst + i + VTraits<v_float32>::vlanes(), s1);
v_store(dst + i + 2*VTraits<v_float32>::vlanes(), s2);
v_store(dst + i + 3*VTraits<v_float32>::vlanes(), s3);
}
if( i <= width - 2*VTraits<v_float32>::vlanes() )
{
const float* src = src0 + i;
v_float32 s0 = v_mul(vx_load(src), k0);
v_float32 s1 = v_mul(vx_load(src + VTraits<v_float32>::vlanes()), k0);
src += cn;
for( k = 1; k < _ksize; k++, src += cn )
{
v_float32 k1 = vx_setall_f32(_kx[k]);
s0 = v_muladd(vx_load(src), k1, s0);
s1 = v_muladd(vx_load(src + VTraits<v_float32>::vlanes()), k1, s1);
}
v_store(dst + i, s0);
v_store(dst + i + VTraits<v_float32>::vlanes(), s1);
i += 2*VTraits<v_float32>::vlanes();
}
if( i <= width - VTraits<v_float32>::vlanes() )
{
const float* src = src0 + i;
v_float32 s0 = v_mul(vx_load(src), k0);
src += cn;
for( k = 1; k < _ksize; k++, src += cn )
s0 = v_muladd(vx_load(src), vx_setall_f32(_kx[k]), s0);
v_store(dst + i, s0);
i += VTraits<v_float32>::vlanes();
}
return i;
}
Mat kernel;
#if defined USE_IPP_SEP_FILTERS
private:
mutable int bufsz;
int ippiOperator(const uchar* _src, uchar* _dst, int width, int cn) const
{
CV_INSTRUMENT_REGION_IPP();
int _ksize = kernel.rows + kernel.cols - 1;
if ((1 != cn && 3 != cn) || width < _ksize*8)
return 0;
const float* src = (const float*)_src;
float* dst = (float*)_dst;
const float* _kx = (const float*)kernel.data;
IppiSize roisz = { width, 1 };
if( bufsz < 0 )
{
if( (cn == 1 && ippiFilterRowBorderPipelineGetBufferSize_32f_C1R(roisz, _ksize, &bufsz) < 0) ||
(cn == 3 && ippiFilterRowBorderPipelineGetBufferSize_32f_C3R(roisz, _ksize, &bufsz) < 0))
return 0;
}
AutoBuffer<uchar> buf(bufsz + 64);
uchar* bufptr = alignPtr(buf.data(), 32);
int step = (int)(width*sizeof(dst[0])*cn);
float borderValue[] = {0.f, 0.f, 0.f};
// here is the trick. IPP needs border type and extrapolates the row. We did it already.
// So we pass anchor=0 and ignore the right tail of results since they are incorrect there.
if( (cn == 1 && CV_INSTRUMENT_FUN_IPP(ippiFilterRowBorderPipeline_32f_C1R, src, step, &dst, roisz, _kx, _ksize, 0,
ippBorderRepl, borderValue[0], bufptr) < 0) ||
(cn == 3 && CV_INSTRUMENT_FUN_IPP(ippiFilterRowBorderPipeline_32f_C3R, src, step, &dst, roisz, _kx, _ksize, 0,
ippBorderRepl, borderValue, bufptr) < 0))
{
setIppErrorStatus();
return 0;
}
CV_IMPL_ADD(CV_IMPL_IPP);
return width - _ksize + 1;
}
#endif
};
struct SymmRowSmallVec_32f
{
SymmRowSmallVec_32f() { symmetryType = 0; }
SymmRowSmallVec_32f( const Mat& _kernel, int _symmetryType )
{
kernel = _kernel;
symmetryType = _symmetryType;
}
int operator()(const uchar* _src, uchar* _dst, int width, int cn) const
{
CV_INSTRUMENT_REGION();
int i = 0, _ksize = kernel.rows + kernel.cols - 1;
if( _ksize == 1 )
return 0;
float* dst = (float*)_dst;
const float* src = (const float*)_src + (_ksize/2)*cn;
bool symmetrical = (symmetryType & KERNEL_SYMMETRICAL) != 0;
const float* kx = kernel.ptr<float>() + _ksize/2;
width *= cn;
if( symmetrical )
{
if( _ksize == 3 )
{
if( fabs(kx[0]) == 2 && kx[1] == 1 )
{
#if CV_FMA3 || CV_AVX2
v_float32 k0 = vx_setall_f32(kx[0]);
for( ; i <= width - VTraits<v_float32>::vlanes(); i += VTraits<v_float32>::vlanes(), src += VTraits<v_float32>::vlanes() )
v_store(dst + i, v_muladd(vx_load(src), k0, v_add(vx_load(src - cn), vx_load(src + cn))));
#else
if( kx[0] > 0 )
for( ; i <= width - VTraits<v_float32>::vlanes(); i += VTraits<v_float32>::vlanes(), src += VTraits<v_float32>::vlanes() )
{
v_float32 x = vx_load(src);
v_store(dst + i, v_add(vx_load(src - cn), vx_load(src + cn), x , x));
}
else
for( ; i <= width - VTraits<v_float32>::vlanes(); i += VTraits<v_float32>::vlanes(), src += VTraits<v_float32>::vlanes() )
{
v_float32 x = vx_load(src);
v_store(dst + i, v_sub(v_add(vx_load(src - cn), vx_load(src + cn)), v_add(x, x)));
}
#endif
}
else
{
v_float32 k0 = vx_setall_f32(kx[0]), k1 = vx_setall_f32(kx[1]);
for( ; i <= width - VTraits<v_float32>::vlanes(); i += VTraits<v_float32>::vlanes(), src += VTraits<v_float32>::vlanes() )
v_store(dst + i, v_muladd(vx_load(src), k0, v_mul(v_add(vx_load(src - cn), vx_load(src + cn)), k1)));
}
}
else if( _ksize == 5 )
{
if( kx[0] == -2 && kx[1] == 0 && kx[2] == 1 )
{
#if CV_FMA3 || CV_AVX2
v_float32 k0 = vx_setall_f32(-2);
for( ; i <= width - VTraits<v_float32>::vlanes(); i += VTraits<v_float32>::vlanes(), src += VTraits<v_float32>::vlanes() )
v_store(dst + i, v_muladd(vx_load(src), k0, v_add(vx_load(src - 2 * cn), vx_load(src + 2 * cn))));
#else
for( ; i <= width - VTraits<v_float32>::vlanes(); i += VTraits<v_float32>::vlanes(), src += VTraits<v_float32>::vlanes() )
{
v_float32 x = vx_load(src);
v_store(dst + i, v_sub(v_add(vx_load(src - 2*cn), vx_load(src + 2*cn)), v_add(x, x)));
}
#endif
}
else
{
v_float32 k0 = vx_setall_f32(kx[0]), k1 = vx_setall_f32(kx[1]), k2 = vx_setall_f32(kx[2]);
for( ; i <= width - VTraits<v_float32>::vlanes(); i += VTraits<v_float32>::vlanes(), src += VTraits<v_float32>::vlanes() )
v_store(dst + i, v_muladd(v_add(vx_load(src + 2 * cn), vx_load(src - 2 * cn)), k2, v_muladd(vx_load(src), k0, v_mul(v_add(vx_load(src - cn), vx_load(src + cn)), k1))));
}
}
}
else
{
if( _ksize == 3 )
{
if( kx[0] == 0 && kx[1] == 1 )
for( ; i <= width - VTraits<v_float32>::vlanes(); i += VTraits<v_float32>::vlanes(), src += VTraits<v_float32>::vlanes() )
v_store(dst + i, v_sub(vx_load(src + cn), vx_load(src - cn)));
else
{
v_float32 k1 = vx_setall_f32(kx[1]);
for( ; i <= width - VTraits<v_float32>::vlanes(); i += VTraits<v_float32>::vlanes(), src += VTraits<v_float32>::vlanes() )
v_store(dst + i, v_mul(v_sub(vx_load(src + cn), vx_load(src - cn)), k1));
}
}
else if( _ksize == 5 )
{
v_float32 k1 = vx_setall_f32(kx[1]), k2 = vx_setall_f32(kx[2]);
for ( ; i <= width - VTraits<v_float32>::vlanes(); i += VTraits<v_float32>::vlanes(), src += VTraits<v_float32>::vlanes() )
v_store(dst + i, v_muladd(v_sub(vx_load(src + 2 * cn), vx_load(src - 2 * cn)), k2, v_mul(v_sub(vx_load(src + cn), vx_load(src - cn)), k1)));
}
}
return i;
}
Mat kernel;
int symmetryType;
};
struct SymmColumnVec_32f
{
SymmColumnVec_32f() {
symmetryType=0;
delta = 0;
}
SymmColumnVec_32f(const Mat& _kernel, int _symmetryType, int, double _delta)
{
symmetryType = _symmetryType;
kernel = _kernel;
delta = (float)_delta;
CV_Assert( (symmetryType & (KERNEL_SYMMETRICAL | KERNEL_ASYMMETRICAL)) != 0 );
}
int operator()(const uchar** _src, uchar* _dst, int width) const
{
CV_INSTRUMENT_REGION();
int ksize2 = (kernel.rows + kernel.cols - 1)/2;
const float* ky = kernel.ptr<float>() + ksize2;
int i = 0, k;
bool symmetrical = (symmetryType & KERNEL_SYMMETRICAL) != 0;
const float** src = (const float**)_src;
float* dst = (float*)_dst;
if( symmetrical )
{
#if CV_AVX
{
const float *S, *S2;
const __m256 d8 = _mm256_set1_ps(delta);
for( ; i <= width - 16; i += 16 )
{
__m256 f = _mm256_set1_ps(ky[0]);
__m256 s0, s1;
__m256 x0;
S = src[0] + i;
s0 = _mm256_loadu_ps(S);
#if CV_FMA3
s0 = _mm256_fmadd_ps(s0, f, d8);
#else
s0 = _mm256_add_ps(_mm256_mul_ps(s0, f), d8);
#endif
s1 = _mm256_loadu_ps(S+8);
#if CV_FMA3
s1 = _mm256_fmadd_ps(s1, f, d8);
#else
s1 = _mm256_add_ps(_mm256_mul_ps(s1, f), d8);
#endif
for( k = 1; k <= ksize2; k++ )
{
S = src[k] + i;
S2 = src[-k] + i;
f = _mm256_set1_ps(ky[k]);
x0 = _mm256_add_ps(_mm256_loadu_ps(S), _mm256_loadu_ps(S2));
#if CV_FMA3
s0 = _mm256_fmadd_ps(x0, f, s0);
#else
s0 = _mm256_add_ps(s0, _mm256_mul_ps(x0, f));
#endif
x0 = _mm256_add_ps(_mm256_loadu_ps(S+8), _mm256_loadu_ps(S2+8));
#if CV_FMA3
s1 = _mm256_fmadd_ps(x0, f, s1);
#else
s1 = _mm256_add_ps(s1, _mm256_mul_ps(x0, f));
#endif
}
_mm256_storeu_ps(dst + i, s0);
_mm256_storeu_ps(dst + i + 8, s1);
}
}
#endif
const v_float32 d4 = vx_setall_f32(delta);
const v_float32 k0 = vx_setall_f32(ky[0]);
for( ; i <= width - 4*VTraits<v_float32>::vlanes(); i += 4*VTraits<v_float32>::vlanes() )
{
v_float32 s0 = v_muladd(vx_load(src[0] + i), k0, d4);
v_float32 s1 = v_muladd(vx_load(src[0] + i + VTraits<v_float32>::vlanes()), k0, d4);
v_float32 s2 = v_muladd(vx_load(src[0] + i + 2*VTraits<v_float32>::vlanes()), k0, d4);
v_float32 s3 = v_muladd(vx_load(src[0] + i + 3*VTraits<v_float32>::vlanes()), k0, d4);
for( k = 1; k <= ksize2; k++ )
{
v_float32 k1 = vx_setall_f32(ky[k]);
s0 = v_muladd(v_add(vx_load(src[k] + i), vx_load(src[-k] + i)), k1, s0);
s1 = v_muladd(v_add(vx_load(src[k] + i + VTraits<v_float32>::vlanes()), vx_load(src[-k] + i + VTraits<v_float32>::vlanes())), k1, s1);
s2 = v_muladd(v_add(vx_load(src[k] + i + 2 * VTraits<v_float32>::vlanes()), vx_load(src[-k] + i + 2 * VTraits<v_float32>::vlanes())), k1, s2);
s3 = v_muladd(v_add(vx_load(src[k] + i + 3 * VTraits<v_float32>::vlanes()), vx_load(src[-k] + i + 3 * VTraits<v_float32>::vlanes())), k1, s3);
}
v_store(dst + i, s0);
v_store(dst + i + VTraits<v_float32>::vlanes(), s1);
v_store(dst + i + 2*VTraits<v_float32>::vlanes(), s2);
v_store(dst + i + 3*VTraits<v_float32>::vlanes(), s3);
}
if( i <= width - 2*VTraits<v_float32>::vlanes() )
{
v_float32 s0 = v_muladd(vx_load(src[0] + i), k0, d4);
v_float32 s1 = v_muladd(vx_load(src[0] + i + VTraits<v_float32>::vlanes()), k0, d4);
for( k = 1; k <= ksize2; k++ )
{
v_float32 k1 = vx_setall_f32(ky[k]);
s0 = v_muladd(v_add(vx_load(src[k] + i), vx_load(src[-k] + i)), k1, s0);
s1 = v_muladd(v_add(vx_load(src[k] + i + VTraits<v_float32>::vlanes()), vx_load(src[-k] + i + VTraits<v_float32>::vlanes())), k1, s1);
}
v_store(dst + i, s0);
v_store(dst + i + VTraits<v_float32>::vlanes(), s1);
i += 2*VTraits<v_float32>::vlanes();
}
if( i <= width - VTraits<v_float32>::vlanes() )
{
v_float32 s0 = v_muladd(vx_load(src[0] + i), k0, d4);
for( k = 1; k <= ksize2; k++ )
s0 = v_muladd(v_add(vx_load(src[k] + i), vx_load(src[-k] + i)), vx_setall_f32(ky[k]), s0);
v_store(dst + i, s0);
i += VTraits<v_float32>::vlanes();
}
}
else
{
CV_DbgAssert(ksize2 > 0);
#if CV_AVX
{
const float *S2;
const __m256 d8 = _mm256_set1_ps(delta);
for (; i <= width - 16; i += 16)
{
__m256 f, s0 = d8, s1 = d8;
__m256 x0;
for (k = 1; k <= ksize2; k++)
{
const float *S = src[k] + i;
S2 = src[-k] + i;
f = _mm256_set1_ps(ky[k]);
x0 = _mm256_sub_ps(_mm256_loadu_ps(S), _mm256_loadu_ps(S2));
#if CV_FMA3
s0 = _mm256_fmadd_ps(x0, f, s0);
#else
s0 = _mm256_add_ps(s0, _mm256_mul_ps(x0, f));
#endif
x0 = _mm256_sub_ps(_mm256_loadu_ps(S + 8), _mm256_loadu_ps(S2 + 8));
#if CV_FMA3
s1 = _mm256_fmadd_ps(x0, f, s1);
#else
s1 = _mm256_add_ps(s1, _mm256_mul_ps(x0, f));
#endif
}
_mm256_storeu_ps(dst + i, s0);
_mm256_storeu_ps(dst + i + 8, s1);
}
}
#endif
const v_float32 d4 = vx_setall_f32(delta);
const v_float32 k1 = vx_setall_f32(ky[1]);
for( ; i <= width - 4*VTraits<v_float32>::vlanes(); i += 4*VTraits<v_float32>::vlanes() )
{
v_float32 s0 = v_muladd(v_sub(vx_load(src[1] + i), vx_load(src[-1] + i)), k1, d4);
v_float32 s1 = v_muladd(v_sub(vx_load(src[1] + i + VTraits<v_float32>::vlanes()), vx_load(src[-1] + i + VTraits<v_float32>::vlanes())), k1, d4);
v_float32 s2 = v_muladd(v_sub(vx_load(src[1] + i + 2 * VTraits<v_float32>::vlanes()), vx_load(src[-1] + i + 2 * VTraits<v_float32>::vlanes())), k1, d4);
v_float32 s3 = v_muladd(v_sub(vx_load(src[1] + i + 3 * VTraits<v_float32>::vlanes()), vx_load(src[-1] + i + 3 * VTraits<v_float32>::vlanes())), k1, d4);
for( k = 2; k <= ksize2; k++ )
{
v_float32 k2 = vx_setall_f32(ky[k]);
s0 = v_muladd(v_sub(vx_load(src[k] + i), vx_load(src[-k] + i)), k2, s0);
s1 = v_muladd(v_sub(vx_load(src[k] + i + VTraits<v_float32>::vlanes()), vx_load(src[-k] + i + VTraits<v_float32>::vlanes())), k2, s1);
s2 = v_muladd(v_sub(vx_load(src[k] + i + 2 * VTraits<v_float32>::vlanes()), vx_load(src[-k] + i + 2 * VTraits<v_float32>::vlanes())), k2, s2);
s3 = v_muladd(v_sub(vx_load(src[k] + i + 3 * VTraits<v_float32>::vlanes()), vx_load(src[-k] + i + 3 * VTraits<v_float32>::vlanes())), k2, s3);
}
v_store(dst + i, s0);
v_store(dst + i + VTraits<v_float32>::vlanes(), s1);
v_store(dst + i + 2*VTraits<v_float32>::vlanes(), s2);
v_store(dst + i + 3*VTraits<v_float32>::vlanes(), s3);
}
if( i <= width - 2*VTraits<v_float32>::vlanes() )
{
v_float32 s0 = v_muladd(v_sub(vx_load(src[1] + i), vx_load(src[-1] + i)), k1, d4);
v_float32 s1 = v_muladd(v_sub(vx_load(src[1] + i + VTraits<v_float32>::vlanes()), vx_load(src[-1] + i + VTraits<v_float32>::vlanes())), k1, d4);
for( k = 2; k <= ksize2; k++ )
{
v_float32 k2 = vx_setall_f32(ky[k]);
s0 = v_muladd(v_sub(vx_load(src[k] + i), vx_load(src[-k] + i)), k2, s0);
s1 = v_muladd(v_sub(vx_load(src[k] + i + VTraits<v_float32>::vlanes()), vx_load(src[-k] + i + VTraits<v_float32>::vlanes())), k2, s1);
}
v_store(dst + i, s0);
v_store(dst + i + VTraits<v_float32>::vlanes(), s1);
i += 2*VTraits<v_float32>::vlanes();
}
if( i <= width - VTraits<v_float32>::vlanes() )
{
v_float32 s0 = v_muladd(v_sub(vx_load(src[1] + i), vx_load(src[-1] + i)), k1, d4);
for( k = 2; k <= ksize2; k++ )
s0 = v_muladd(v_sub(vx_load(src[k] + i), vx_load(src[-k] + i)), vx_setall_f32(ky[k]), s0);
v_store(dst + i, s0);
i += VTraits<v_float32>::vlanes();
}
}
return i;
}
int symmetryType;
float delta;
Mat kernel;
};
struct SymmColumnSmallVec_32f
{
SymmColumnSmallVec_32f() { symmetryType=0; delta = 0; }
SymmColumnSmallVec_32f(const Mat& _kernel, int _symmetryType, int, double _delta)
{
symmetryType = _symmetryType;
kernel = _kernel;
delta = (float)_delta;
CV_Assert( (symmetryType & (KERNEL_SYMMETRICAL | KERNEL_ASYMMETRICAL)) != 0 );
}
int operator()(const uchar** _src, uchar* _dst, int width) const
{
CV_INSTRUMENT_REGION();
int ksize2 = (kernel.rows + kernel.cols - 1)/2;
const float* ky = kernel.ptr<float>() + ksize2;
int i = 0;
bool symmetrical = (symmetryType & KERNEL_SYMMETRICAL) != 0;
const float** src = (const float**)_src;
const float *S0 = src[-1], *S1 = src[0], *S2 = src[1];
float* dst = (float*)_dst;
v_float32 d4 = vx_setall_f32(delta);
if( symmetrical )
{
if( fabs(ky[0]) == 2 && ky[1] == 1 )
{
#if CV_FMA3 || CV_AVX2
v_float32 k0 = vx_setall_f32(ky[0]);
for( ; i <= width - VTraits<v_float32>::vlanes(); i += VTraits<v_float32>::vlanes() )
v_store(dst + i, v_muladd(vx_load(S1 + i), k0, v_add(v_add(vx_load(S0 + i), vx_load(S2 + i)), d4)));
#else
if(ky[0] > 0)
for( ; i <= width - VTraits<v_float32>::vlanes(); i += VTraits<v_float32>::vlanes() )
{
v_float32 x = vx_load(S1 + i);
v_store(dst + i, v_add(vx_load(S0 + i), vx_load(S2 + i), d4, x, x));
}
else
for( ; i <= width - VTraits<v_float32>::vlanes(); i += VTraits<v_float32>::vlanes() )
{
v_float32 x = vx_load(S1 + i);
v_store(dst + i, v_sub(v_add(vx_load(S0 + i), vx_load(S2 + i), d4), v_add(x, x)));
}
#endif
}
else
{
v_float32 k0 = vx_setall_f32(ky[0]), k1 = vx_setall_f32(ky[1]);
for ( ; i <= width - VTraits<v_float32>::vlanes(); i += VTraits<v_float32>::vlanes() )
v_store(dst + i, v_muladd(v_add(vx_load(S0 + i), vx_load(S2 + i)), k1, v_muladd(vx_load(S1 + i), k0, d4)));
}
}
else
{
if( fabs(ky[1]) == 1 && ky[1] == -ky[-1] )
{
if( ky[1] < 0 )
std::swap(S0, S2);
for ( ; i <= width - VTraits<v_float32>::vlanes(); i += VTraits<v_float32>::vlanes() )
v_store(dst + i, v_add(v_sub(vx_load(S2 + i), vx_load(S0 + i)), d4));
}
else
{
v_float32 k1 = vx_setall_f32(ky[1]);
for ( ; i <= width - VTraits<v_float32>::vlanes(); i += VTraits<v_float32>::vlanes() )
v_store(dst + i, v_muladd(v_sub(vx_load(S2 + i), vx_load(S0 + i)), k1, d4));
}
}
return i;
}
int symmetryType;
float delta;
Mat kernel;
};
/////////////////////////////// non-separable filters ///////////////////////////////
///////////////////////////////// 8u<->8u, 8u<->16s /////////////////////////////////
struct FilterVec_8u
{
FilterVec_8u() { delta = 0; _nz = 0; }
FilterVec_8u(const Mat& _kernel, int _bits, double _delta)
{
Mat kernel;
_kernel.convertTo(kernel, CV_32F, 1./(1 << _bits), 0);
delta = (float)(_delta/(1 << _bits));
std::vector<Point> coords;
preprocess2DKernel(kernel, coords, coeffs);
_nz = (int)coords.size();
}
int operator()(const uchar** src, uchar* dst, int width) const
{
CV_INSTRUMENT_REGION();
CV_DbgAssert(_nz > 0);
const float* kf = (const float*)&coeffs[0];
int i = 0, k, nz = _nz;
v_float32 d4 = vx_setall_f32(delta);
v_float32 f0 = vx_setall_f32(kf[0]);
for( ; i <= width - VTraits<v_uint8>::vlanes(); i += VTraits<v_uint8>::vlanes() )
{
v_uint16 xl, xh;
v_expand(vx_load(src[0] + i), xl, xh);
v_uint32 x0, x1, x2, x3;
v_expand(xl, x0, x1);
v_expand(xh, x2, x3);
v_float32 s0 = v_muladd(v_cvt_f32(v_reinterpret_as_s32(x0)), f0, d4);
v_float32 s1 = v_muladd(v_cvt_f32(v_reinterpret_as_s32(x1)), f0, d4);
v_float32 s2 = v_muladd(v_cvt_f32(v_reinterpret_as_s32(x2)), f0, d4);
v_float32 s3 = v_muladd(v_cvt_f32(v_reinterpret_as_s32(x3)), f0, d4);
for( k = 1; k < nz; k++ )
{
v_float32 f = vx_setall_f32(kf[k]);
v_expand(vx_load(src[k] + i), xl, xh);
v_expand(xl, x0, x1);
v_expand(xh, x2, x3);
s0 = v_muladd(v_cvt_f32(v_reinterpret_as_s32(x0)), f, s0);
s1 = v_muladd(v_cvt_f32(v_reinterpret_as_s32(x1)), f, s1);
s2 = v_muladd(v_cvt_f32(v_reinterpret_as_s32(x2)), f, s2);
s3 = v_muladd(v_cvt_f32(v_reinterpret_as_s32(x3)), f, s3);
}
v_store(dst + i, v_pack_u(v_pack(v_round(s0), v_round(s1)), v_pack(v_round(s2), v_round(s3))));
}
if( i <= width - VTraits<v_uint16>::vlanes() )
{
v_uint32 x0, x1;
v_expand(vx_load_expand(src[0] + i), x0, x1);
v_float32 s0 = v_muladd(v_cvt_f32(v_reinterpret_as_s32(x0)), f0, d4);
v_float32 s1 = v_muladd(v_cvt_f32(v_reinterpret_as_s32(x1)), f0, d4);
for( k = 1; k < nz; k++ )
{
v_float32 f = vx_setall_f32(kf[k]);
v_expand(vx_load_expand(src[k] + i), x0, x1);
s0 = v_muladd(v_cvt_f32(v_reinterpret_as_s32(x0)), f, s0);
s1 = v_muladd(v_cvt_f32(v_reinterpret_as_s32(x1)), f, s1);
}
v_pack_u_store(dst + i, v_pack(v_round(s0), v_round(s1)));
i += VTraits<v_uint16>::vlanes();
}
return i;
}
int _nz;
std::vector<uchar> coeffs;
float delta;
};
struct FilterVec_8u16s
{
FilterVec_8u16s() { delta = 0; _nz = 0; }
FilterVec_8u16s(const Mat& _kernel, int _bits, double _delta)
{
Mat kernel;
_kernel.convertTo(kernel, CV_32F, 1./(1 << _bits), 0);
delta = (float)(_delta/(1 << _bits));
std::vector<Point> coords;
preprocess2DKernel(kernel, coords, coeffs);
_nz = (int)coords.size();
}
int operator()(const uchar** src, uchar* _dst, int width) const
{
CV_INSTRUMENT_REGION();
CV_DbgAssert(_nz > 0);
const float* kf = (const float*)&coeffs[0];
short* dst = (short*)_dst;
int i = 0, k, nz = _nz;
v_float32 d4 = vx_setall_f32(delta);
v_float32 f0 = vx_setall_f32(kf[0]);
for( ; i <= width - VTraits<v_uint8>::vlanes(); i += VTraits<v_uint8>::vlanes() )
{
v_uint16 xl, xh;
v_expand(vx_load(src[0] + i), xl, xh);
v_float32 s0 = v_muladd(v_cvt_f32(v_reinterpret_as_s32(v_expand_low(xl))), f0, d4);
v_float32 s1 = v_muladd(v_cvt_f32(v_reinterpret_as_s32(v_expand_high(xl))), f0, d4);
v_float32 s2 = v_muladd(v_cvt_f32(v_reinterpret_as_s32(v_expand_low(xh))), f0, d4);
v_float32 s3 = v_muladd(v_cvt_f32(v_reinterpret_as_s32(v_expand_high(xh))), f0, d4);
for( k = 1; k < nz; k++ )
{
v_float32 f = vx_setall_f32(kf[k]);
v_expand(vx_load(src[k] + i), xl, xh);
s0 = v_muladd(v_cvt_f32(v_reinterpret_as_s32(v_expand_low(xl))), f, s0);
s1 = v_muladd(v_cvt_f32(v_reinterpret_as_s32(v_expand_high(xl))), f, s1);
s2 = v_muladd(v_cvt_f32(v_reinterpret_as_s32(v_expand_low(xh))), f, s2);
s3 = v_muladd(v_cvt_f32(v_reinterpret_as_s32(v_expand_high(xh))), f, s3);
}
v_store(dst + i, v_pack(v_round(s0), v_round(s1)));
v_store(dst + i + VTraits<v_int16>::vlanes(), v_pack(v_round(s2), v_round(s3)));
}
if( i <= width - VTraits<v_uint16>::vlanes() )
{
v_uint16 x = vx_load_expand(src[0] + i);
v_float32 s0 = v_muladd(v_cvt_f32(v_reinterpret_as_s32(v_expand_low(x))), f0, d4);
v_float32 s1 = v_muladd(v_cvt_f32(v_reinterpret_as_s32(v_expand_high(x))), f0, d4);
for( k = 1; k < nz; k++ )
{
v_float32 f = vx_setall_f32(kf[k]);
x = vx_load_expand(src[k] + i);
s0 = v_muladd(v_cvt_f32(v_reinterpret_as_s32(v_expand_low(x))), f, s0);
s1 = v_muladd(v_cvt_f32(v_reinterpret_as_s32(v_expand_high(x))), f, s1);
}
v_store(dst + i, v_pack(v_round(s0), v_round(s1)));
i += VTraits<v_uint16>::vlanes();
}
if( i <= width - VTraits<v_int32>::vlanes() )
{
v_float32 s0 = v_muladd(v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(src[0] + i))), f0, d4);
for( k = 1; k < nz; k++ )
s0 = v_muladd(v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(src[k] + i))), vx_setall_f32(kf[k]), s0);
v_pack_store(dst + i, v_round(s0));
i += VTraits<v_int32>::vlanes();
}
return i;
}
int _nz;
std::vector<uchar> coeffs;
float delta;
};
struct FilterVec_32f
{
FilterVec_32f() { delta = 0; _nz = 0; }
FilterVec_32f(const Mat& _kernel, int, double _delta)
{
delta = (float)_delta;
std::vector<Point> coords;
preprocess2DKernel(_kernel, coords, coeffs);
_nz = (int)coords.size();
}
int operator()(const uchar** _src, uchar* _dst, int width) const
{
CV_INSTRUMENT_REGION();
const float* kf = (const float*)&coeffs[0];
const float** src = (const float**)_src;
float* dst = (float*)_dst;
int i = 0, k, nz = _nz;
v_float32 d4 = vx_setall_f32(delta);
v_float32 f0 = vx_setall_f32(kf[0]);
for( ; i <= width - 4*VTraits<v_float32>::vlanes(); i += 4*VTraits<v_float32>::vlanes() )
{
v_float32 s0 = v_muladd(vx_load(src[0] + i), f0, d4);
v_float32 s1 = v_muladd(vx_load(src[0] + i + VTraits<v_float32>::vlanes()), f0, d4);
v_float32 s2 = v_muladd(vx_load(src[0] + i + 2*VTraits<v_float32>::vlanes()), f0, d4);
v_float32 s3 = v_muladd(vx_load(src[0] + i + 3*VTraits<v_float32>::vlanes()), f0, d4);
for( k = 1; k < nz; k++ )
{
v_float32 f1 = vx_setall_f32(kf[k]);
s0 = v_muladd(vx_load(src[k] + i), f1, s0);
s1 = v_muladd(vx_load(src[k] + i + VTraits<v_float32>::vlanes()), f1, s1);
s2 = v_muladd(vx_load(src[k] + i + 2*VTraits<v_float32>::vlanes()), f1, s2);
s3 = v_muladd(vx_load(src[k] + i + 3*VTraits<v_float32>::vlanes()), f1, s3);
}
v_store(dst + i, s0);
v_store(dst + i + VTraits<v_float32>::vlanes(), s1);
v_store(dst + i + 2*VTraits<v_float32>::vlanes(), s2);
v_store(dst + i + 3*VTraits<v_float32>::vlanes(), s3);
}
if( i <= width - 2*VTraits<v_float32>::vlanes() )
{
v_float32 s0 = v_muladd(vx_load(src[0] + i), f0, d4);
v_float32 s1 = v_muladd(vx_load(src[0] + i + VTraits<v_float32>::vlanes()), f0, d4);
for( k = 1; k < nz; k++ )
{
v_float32 f1 = vx_setall_f32(kf[k]);
s0 = v_muladd(vx_load(src[k] + i), f1, s0);
s1 = v_muladd(vx_load(src[k] + i + VTraits<v_float32>::vlanes()), f1, s1);
}
v_store(dst + i, s0);
v_store(dst + i + VTraits<v_float32>::vlanes(), s1);
i += 2*VTraits<v_float32>::vlanes();
}
if( i <= width - VTraits<v_float32>::vlanes() )
{
v_float32 s0 = v_muladd(vx_load(src[0] + i), f0, d4);
for( k = 1; k < nz; k++ )
s0 = v_muladd(vx_load(src[k] + i), vx_setall_f32(kf[k]), s0);
v_store(dst + i, s0);
i += VTraits<v_float32>::vlanes();
}
return i;
}
int _nz;
std::vector<uchar> coeffs;
float delta;
};
#else
typedef RowNoVec RowVec_8u32s;
typedef RowNoVec RowVec_8u32f;
typedef RowNoVec RowVec_16s32f;
typedef RowNoVec RowVec_32f;
typedef SymmRowSmallNoVec SymmRowSmallVec_8u32s;
typedef SymmRowSmallNoVec SymmRowSmallVec_32f;
typedef ColumnNoVec SymmColumnVec_32s8u;
typedef ColumnNoVec SymmColumnVec_32f8u;
typedef ColumnNoVec SymmColumnVec_32f16s;
typedef ColumnNoVec SymmColumnVec_32f;
typedef SymmColumnSmallNoVec SymmColumnSmallVec_32s16s;
typedef SymmColumnSmallNoVec SymmColumnSmallVec_32f;
typedef FilterNoVec FilterVec_8u;
typedef FilterNoVec FilterVec_8u16s;
typedef FilterNoVec FilterVec_32f;
#endif
template<typename ST, typename DT, class VecOp> struct RowFilter : public BaseRowFilter
{
RowFilter( const Mat& _kernel, int _anchor, const VecOp& _vecOp=VecOp() )
{
if( _kernel.isContinuous() )
kernel = _kernel;
else
_kernel.copyTo(kernel);
anchor = _anchor;
ksize = kernel.rows + kernel.cols - 1;
CV_Assert( kernel.type() == DataType<DT>::type &&
(kernel.rows == 1 || kernel.cols == 1));
vecOp = _vecOp;
}
void operator()(const uchar* src, uchar* dst, int width, int cn) CV_OVERRIDE
{
CV_INSTRUMENT_REGION();
int _ksize = ksize;
const DT* kx = kernel.ptr<DT>();
const ST* S;
DT* D = (DT*)dst;
int i, k;
i = vecOp(src, dst, width, cn);
width *= cn;
#if CV_ENABLE_UNROLLED
for( ; i <= width - 4; i += 4 )
{
S = (const ST*)src + i;
DT f = kx[0];
DT s0 = f*S[0], s1 = f*S[1], s2 = f*S[2], s3 = f*S[3];
for( k = 1; k < _ksize; k++ )
{
S += cn;
f = kx[k];
s0 += f*S[0]; s1 += f*S[1];
s2 += f*S[2]; s3 += f*S[3];
}
D[i] = s0; D[i+1] = s1;
D[i+2] = s2; D[i+3] = s3;
}
#endif
for( ; i < width; i++ )
{
S = (const ST*)src + i;
DT s0 = kx[0]*S[0];
for( k = 1; k < _ksize; k++ )
{
S += cn;
s0 += kx[k]*S[0];
}
D[i] = s0;
}
}
Mat kernel;
VecOp vecOp;
};
template<typename ST, typename DT, class VecOp> struct SymmRowSmallFilter :
public RowFilter<ST, DT, VecOp>
{
SymmRowSmallFilter( const Mat& _kernel, int _anchor, int _symmetryType,
const VecOp& _vecOp = VecOp())
: RowFilter<ST, DT, VecOp>( _kernel, _anchor, _vecOp )
{
symmetryType = _symmetryType;
CV_Assert( (symmetryType & (KERNEL_SYMMETRICAL | KERNEL_ASYMMETRICAL)) != 0 && this->ksize <= 5 );
}
void operator()(const uchar* src, uchar* dst, int width, int cn) CV_OVERRIDE
{
CV_INSTRUMENT_REGION();
int ksize2 = this->ksize/2, ksize2n = ksize2*cn;
const DT* kx = this->kernel.template ptr<DT>() + ksize2;
bool symmetrical = (this->symmetryType & KERNEL_SYMMETRICAL) != 0;
DT* D = (DT*)dst;
int i = this->vecOp(src, dst, width, cn), j, k;
const ST* S = (const ST*)src + i + ksize2n;
width *= cn;
if( symmetrical )
{
if( this->ksize == 1 && kx[0] == 1 )
{
for( ; i <= width - 2; i += 2 )
{
DT s0 = S[i], s1 = S[i+1];
D[i] = s0; D[i+1] = s1;
}
S += i;
}
else if( this->ksize == 3 )
{
if( kx[0] == 2 && kx[1] == 1 )
for( ; i <= width - 2; i += 2, S += 2 )
{
DT s0 = S[-cn] + S[0]*2 + S[cn], s1 = S[1-cn] + S[1]*2 + S[1+cn];
D[i] = s0; D[i+1] = s1;
}
else if( kx[0] == -2 && kx[1] == 1 )
for( ; i <= width - 2; i += 2, S += 2 )
{
DT s0 = S[-cn] - S[0]*2 + S[cn], s1 = S[1-cn] - S[1]*2 + S[1+cn];
D[i] = s0; D[i+1] = s1;
}
else
{
DT k0 = kx[0], k1 = kx[1];
for( ; i <= width - 2; i += 2, S += 2 )
{
DT s0 = S[0]*k0 + (S[-cn] + S[cn])*k1, s1 = S[1]*k0 + (S[1-cn] + S[1+cn])*k1;
D[i] = s0; D[i+1] = s1;
}
}
}
else if( this->ksize == 5 )
{
DT k0 = kx[0], k1 = kx[1], k2 = kx[2];
if( k0 == -2 && k1 == 0 && k2 == 1 )
for( ; i <= width - 2; i += 2, S += 2 )
{
DT s0 = -2*S[0] + S[-cn*2] + S[cn*2];
DT s1 = -2*S[1] + S[1-cn*2] + S[1+cn*2];
D[i] = s0; D[i+1] = s1;
}
else
for( ; i <= width - 2; i += 2, S += 2 )
{
DT s0 = S[0]*k0 + (S[-cn] + S[cn])*k1 + (S[-cn*2] + S[cn*2])*k2;
DT s1 = S[1]*k0 + (S[1-cn] + S[1+cn])*k1 + (S[1-cn*2] + S[1+cn*2])*k2;
D[i] = s0; D[i+1] = s1;
}
}
for( ; i < width; i++, S++ )
{
DT s0 = kx[0]*S[0];
for( k = 1, j = cn; k <= ksize2; k++, j += cn )
s0 += kx[k]*(S[j] + S[-j]);
D[i] = s0;
}
}
else
{
if( this->ksize == 3 )
{
if( kx[0] == 0 && kx[1] == 1 )
for( ; i <= width - 2; i += 2, S += 2 )
{
DT s0 = S[cn] - S[-cn], s1 = S[1+cn] - S[1-cn];
D[i] = s0; D[i+1] = s1;
}
else
{
DT k1 = kx[1];
for( ; i <= width - 2; i += 2, S += 2 )
{
DT s0 = (S[cn] - S[-cn])*k1, s1 = (S[1+cn] - S[1-cn])*k1;
D[i] = s0; D[i+1] = s1;
}
}
}
else if( this->ksize == 5 )
{
DT k1 = kx[1], k2 = kx[2];
for( ; i <= width - 2; i += 2, S += 2 )
{
DT s0 = (S[cn] - S[-cn])*k1 + (S[cn*2] - S[-cn*2])*k2;
DT s1 = (S[1+cn] - S[1-cn])*k1 + (S[1+cn*2] - S[1-cn*2])*k2;
D[i] = s0; D[i+1] = s1;
}
}
for( ; i < width; i++, S++ )
{
DT s0 = kx[0]*S[0];
for( k = 1, j = cn; k <= ksize2; k++, j += cn )
s0 += kx[k]*(S[j] - S[-j]);
D[i] = s0;
}
}
}
int symmetryType;
};
template<class CastOp, class VecOp> struct ColumnFilter : public BaseColumnFilter
{
typedef typename CastOp::type1 ST;
typedef typename CastOp::rtype DT;
ColumnFilter( const Mat& _kernel, int _anchor,
double _delta, const CastOp& _castOp=CastOp(),
const VecOp& _vecOp=VecOp() )
{
if( _kernel.isContinuous() )
kernel = _kernel;
else
_kernel.copyTo(kernel);
anchor = _anchor;
ksize = kernel.rows + kernel.cols - 1;
delta = saturate_cast<ST>(_delta);
castOp0 = _castOp;
vecOp = _vecOp;
CV_Assert( kernel.type() == DataType<ST>::type &&
(kernel.rows == 1 || kernel.cols == 1));
}
void operator()(const uchar** src, uchar* dst, int dststep, int count, int width) CV_OVERRIDE
{
CV_INSTRUMENT_REGION();
const ST* ky = kernel.template ptr<ST>();
ST _delta = delta;
int _ksize = ksize;
int i, k;
CastOp castOp = castOp0;
for( ; count--; dst += dststep, src++ )
{
DT* D = (DT*)dst;
i = vecOp(src, dst, width);
#if CV_ENABLE_UNROLLED
for( ; i <= width - 4; i += 4 )
{
ST f = ky[0];
const ST* S = (const ST*)src[0] + i;
ST s0 = f*S[0] + _delta, s1 = f*S[1] + _delta,
s2 = f*S[2] + _delta, s3 = f*S[3] + _delta;
for( k = 1; k < _ksize; k++ )
{
S = (const ST*)src[k] + i; f = ky[k];
s0 += f*S[0]; s1 += f*S[1];
s2 += f*S[2]; s3 += f*S[3];
}
D[i] = castOp(s0); D[i+1] = castOp(s1);
D[i+2] = castOp(s2); D[i+3] = castOp(s3);
}
#endif
for( ; i < width; i++ )
{
ST s0 = ky[0]*((const ST*)src[0])[i] + _delta;
for( k = 1; k < _ksize; k++ )
s0 += ky[k]*((const ST*)src[k])[i];
D[i] = castOp(s0);
}
}
}
Mat kernel;
CastOp castOp0;
VecOp vecOp;
ST delta;
};
template<class CastOp, class VecOp> struct SymmColumnFilter : public ColumnFilter<CastOp, VecOp>
{
typedef typename CastOp::type1 ST;
typedef typename CastOp::rtype DT;
SymmColumnFilter( const Mat& _kernel, int _anchor,
double _delta, int _symmetryType,
const CastOp& _castOp=CastOp(),
const VecOp& _vecOp=VecOp())
: ColumnFilter<CastOp, VecOp>( _kernel, _anchor, _delta, _castOp, _vecOp )
{
symmetryType = _symmetryType;
CV_Assert( (symmetryType & (KERNEL_SYMMETRICAL | KERNEL_ASYMMETRICAL)) != 0 );
}
void operator()(const uchar** src, uchar* dst, int dststep, int count, int width) CV_OVERRIDE
{
CV_INSTRUMENT_REGION();
int ksize2 = this->ksize/2;
const ST* ky = this->kernel.template ptr<ST>() + ksize2;
int i, k;
bool symmetrical = (symmetryType & KERNEL_SYMMETRICAL) != 0;
ST _delta = this->delta;
CastOp castOp = this->castOp0;
src += ksize2;
if( symmetrical )
{
for( ; count--; dst += dststep, src++ )
{
DT* D = (DT*)dst;
i = (this->vecOp)(src, dst, width);
#if CV_ENABLE_UNROLLED
for( ; i <= width - 4; i += 4 )
{
ST f = ky[0];
const ST* S = (const ST*)src[0] + i, *S2;
ST s0 = f*S[0] + _delta, s1 = f*S[1] + _delta,
s2 = f*S[2] + _delta, s3 = f*S[3] + _delta;
for( k = 1; k <= ksize2; k++ )
{
S = (const ST*)src[k] + i;
S2 = (const ST*)src[-k] + i;
f = ky[k];
s0 += f*(S[0] + S2[0]);
s1 += f*(S[1] + S2[1]);
s2 += f*(S[2] + S2[2]);
s3 += f*(S[3] + S2[3]);
}
D[i] = castOp(s0); D[i+1] = castOp(s1);
D[i+2] = castOp(s2); D[i+3] = castOp(s3);
}
#endif
for( ; i < width; i++ )
{
ST s0 = ky[0]*((const ST*)src[0])[i] + _delta;
for( k = 1; k <= ksize2; k++ )
s0 += ky[k]*(((const ST*)src[k])[i] + ((const ST*)src[-k])[i]);
D[i] = castOp(s0);
}
}
}
else
{
for( ; count--; dst += dststep, src++ )
{
DT* D = (DT*)dst;
i = this->vecOp(src, dst, width);
#if CV_ENABLE_UNROLLED
for( ; i <= width - 4; i += 4 )
{
ST f = ky[0];
const ST *S, *S2;
ST s0 = _delta, s1 = _delta, s2 = _delta, s3 = _delta;
for( k = 1; k <= ksize2; k++ )
{
S = (const ST*)src[k] + i;
S2 = (const ST*)src[-k] + i;
f = ky[k];
s0 += f*(S[0] - S2[0]);
s1 += f*(S[1] - S2[1]);
s2 += f*(S[2] - S2[2]);
s3 += f*(S[3] - S2[3]);
}
D[i] = castOp(s0); D[i+1] = castOp(s1);
D[i+2] = castOp(s2); D[i+3] = castOp(s3);
}
#endif
for( ; i < width; i++ )
{
ST s0 = _delta;
for( k = 1; k <= ksize2; k++ )
s0 += ky[k]*(((const ST*)src[k])[i] - ((const ST*)src[-k])[i]);
D[i] = castOp(s0);
}
}
}
}
int symmetryType;
};
template<class CastOp, class VecOp>
struct SymmColumnSmallFilter : public SymmColumnFilter<CastOp, VecOp>
{
typedef typename CastOp::type1 ST;
typedef typename CastOp::rtype DT;
SymmColumnSmallFilter( const Mat& _kernel, int _anchor,
double _delta, int _symmetryType,
const CastOp& _castOp=CastOp(),
const VecOp& _vecOp=VecOp())
: SymmColumnFilter<CastOp, VecOp>( _kernel, _anchor, _delta, _symmetryType, _castOp, _vecOp )
{
CV_Assert( this->ksize == 3 );
}
void operator()(const uchar** src, uchar* dst, int dststep, int count, int width) CV_OVERRIDE
{
CV_INSTRUMENT_REGION();
int ksize2 = this->ksize/2;
const ST* ky = this->kernel.template ptr<ST>() + ksize2;
int i;
bool symmetrical = (this->symmetryType & KERNEL_SYMMETRICAL) != 0;
bool is_1_2_1 = ky[0] == 2 && ky[1] == 1;
bool is_1_m2_1 = ky[0] == -2 && ky[1] == 1;
bool is_m1_0_1 = ky[0] == 0 && (ky[1] == 1 || ky[1] == -1);
ST f0 = ky[0], f1 = ky[1];
ST _delta = this->delta;
CastOp castOp = this->castOp0;
src += ksize2;
for( ; count--; dst += dststep, src++ )
{
DT* D = (DT*)dst;
i = (this->vecOp)(src, dst, width);
const ST* S0 = (const ST*)src[-1];
const ST* S1 = (const ST*)src[0];
const ST* S2 = (const ST*)src[1];
if( symmetrical )
{
if( is_1_2_1 )
{
#if CV_ENABLE_UNROLLED
for( ; i <= width - 4; i += 4 )
{
ST s0 = S0[i] + S1[i]*2 + S2[i] + _delta;
ST s1 = S0[i+1] + S1[i+1]*2 + S2[i+1] + _delta;
D[i] = castOp(s0);
D[i+1] = castOp(s1);
s0 = S0[i+2] + S1[i+2]*2 + S2[i+2] + _delta;
s1 = S0[i+3] + S1[i+3]*2 + S2[i+3] + _delta;
D[i+2] = castOp(s0);
D[i+3] = castOp(s1);
}
#endif
for( ; i < width; i ++ )
{
ST s0 = S0[i] + S1[i]*2 + S2[i] + _delta;
D[i] = castOp(s0);
}
}
else if( is_1_m2_1 )
{
#if CV_ENABLE_UNROLLED
for( ; i <= width - 4; i += 4 )
{
ST s0 = S0[i] - S1[i]*2 + S2[i] + _delta;
ST s1 = S0[i+1] - S1[i+1]*2 + S2[i+1] + _delta;
D[i] = castOp(s0);
D[i+1] = castOp(s1);
s0 = S0[i+2] - S1[i+2]*2 + S2[i+2] + _delta;
s1 = S0[i+3] - S1[i+3]*2 + S2[i+3] + _delta;
D[i+2] = castOp(s0);
D[i+3] = castOp(s1);
}
#endif
for( ; i < width; i ++ )
{
ST s0 = S0[i] - S1[i]*2 + S2[i] + _delta;
D[i] = castOp(s0);
}
}
else
{
#if CV_ENABLE_UNROLLED
for( ; i <= width - 4; i += 4 )
{
ST s0 = (S0[i] + S2[i])*f1 + S1[i]*f0 + _delta;
ST s1 = (S0[i+1] + S2[i+1])*f1 + S1[i+1]*f0 + _delta;
D[i] = castOp(s0);
D[i+1] = castOp(s1);
s0 = (S0[i+2] + S2[i+2])*f1 + S1[i+2]*f0 + _delta;
s1 = (S0[i+3] + S2[i+3])*f1 + S1[i+3]*f0 + _delta;
D[i+2] = castOp(s0);
D[i+3] = castOp(s1);
}
#endif
for( ; i < width; i ++ )
{
ST s0 = (S0[i] + S2[i])*f1 + S1[i]*f0 + _delta;
D[i] = castOp(s0);
}
}
}
else
{
if( is_m1_0_1 )
{
if( f1 < 0 )
std::swap(S0, S2);
#if CV_ENABLE_UNROLLED
for( ; i <= width - 4; i += 4 )
{
ST s0 = S2[i] - S0[i] + _delta;
ST s1 = S2[i+1] - S0[i+1] + _delta;
D[i] = castOp(s0);
D[i+1] = castOp(s1);
s0 = S2[i+2] - S0[i+2] + _delta;
s1 = S2[i+3] - S0[i+3] + _delta;
D[i+2] = castOp(s0);
D[i+3] = castOp(s1);
}
#endif
for( ; i < width; i ++ )
{
ST s0 = S2[i] - S0[i] + _delta;
D[i] = castOp(s0);
}
if( f1 < 0 )
std::swap(S0, S2);
}
else
{
#if CV_ENABLE_UNROLLED
for( ; i <= width - 4; i += 4 )
{
ST s0 = (S2[i] - S0[i])*f1 + _delta;
ST s1 = (S2[i+1] - S0[i+1])*f1 + _delta;
D[i] = castOp(s0);
D[i+1] = castOp(s1);
s0 = (S2[i+2] - S0[i+2])*f1 + _delta;
s1 = (S2[i+3] - S0[i+3])*f1 + _delta;
D[i+2] = castOp(s0);
D[i+3] = castOp(s1);
}
#endif
for( ; i < width; i++ )
D[i] = castOp((S2[i] - S0[i])*f1 + _delta);
}
}
}
}
};
template<typename ST, typename DT> struct Cast
{
typedef ST type1;
typedef DT rtype;
DT operator()(ST val) const { return saturate_cast<DT>(val); }
};
template<typename ST, typename DT, int bits> struct FixedPtCast
{
typedef ST type1;
typedef DT rtype;
enum { SHIFT = bits, DELTA = 1 << (bits-1) };
DT operator()(ST val) const { return saturate_cast<DT>((val + DELTA)>>SHIFT); }
};
template<typename ST, typename DT> struct FixedPtCastEx
{
typedef ST type1;
typedef DT rtype;
FixedPtCastEx() : SHIFT(0), DELTA(0) {}
FixedPtCastEx(int bits) : SHIFT(bits), DELTA(bits ? 1 << (bits-1) : 0) {}
DT operator()(ST val) const { return saturate_cast<DT>((val + DELTA)>>SHIFT); }
int SHIFT, DELTA;
};
Ptr<BaseRowFilter> getLinearRowFilter(
int srcType, int bufType,
const Mat& kernel, int anchor,
int symmetryType)
{
CV_INSTRUMENT_REGION();
int sdepth = CV_MAT_DEPTH(srcType), ddepth = CV_MAT_DEPTH(bufType);
int cn = CV_MAT_CN(srcType);
CV_Assert( cn == CV_MAT_CN(bufType) &&
ddepth >= std::max(sdepth, CV_32S) &&
kernel.type() == ddepth );
int ksize = kernel.rows + kernel.cols - 1;
if( (symmetryType & (KERNEL_SYMMETRICAL|KERNEL_ASYMMETRICAL)) != 0 && ksize <= 5 )
{
if( sdepth == CV_8U && ddepth == CV_32S )
return makePtr<SymmRowSmallFilter<uchar, int, SymmRowSmallVec_8u32s> >
(kernel, anchor, symmetryType, SymmRowSmallVec_8u32s(kernel, symmetryType));
if( sdepth == CV_32F && ddepth == CV_32F )
return makePtr<SymmRowSmallFilter<float, float, SymmRowSmallVec_32f> >
(kernel, anchor, symmetryType, SymmRowSmallVec_32f(kernel, symmetryType));
}
if( sdepth == CV_8U && ddepth == CV_32S )
return makePtr<RowFilter<uchar, int, RowVec_8u32s> >
(kernel, anchor, RowVec_8u32s(kernel));
if( sdepth == CV_8U && ddepth == CV_32F )
return makePtr<RowFilter<uchar, float, RowVec_8u32f> >
(kernel, anchor, RowVec_8u32f(kernel));
if( sdepth == CV_8U && ddepth == CV_64F )
return makePtr<RowFilter<uchar, double, RowNoVec> >(kernel, anchor);
if( sdepth == CV_16U && ddepth == CV_32F )
return makePtr<RowFilter<ushort, float, RowNoVec> >(kernel, anchor);
if( sdepth == CV_16U && ddepth == CV_64F )
return makePtr<RowFilter<ushort, double, RowNoVec> >(kernel, anchor);
if( sdepth == CV_16S && ddepth == CV_32F )
return makePtr<RowFilter<short, float, RowVec_16s32f> >
(kernel, anchor, RowVec_16s32f(kernel));
if( sdepth == CV_16S && ddepth == CV_64F )
return makePtr<RowFilter<short, double, RowNoVec> >(kernel, anchor);
if( sdepth == CV_32F && ddepth == CV_32F )
return makePtr<RowFilter<float, float, RowVec_32f> >
(kernel, anchor, RowVec_32f(kernel));
if( sdepth == CV_32F && ddepth == CV_64F )
return makePtr<RowFilter<float, double, RowNoVec> >(kernel, anchor);
if( sdepth == CV_64F && ddepth == CV_64F )
return makePtr<RowFilter<double, double, RowNoVec> >(kernel, anchor);
CV_Error_( cv::Error::StsNotImplemented,
("Unsupported combination of source format (=%d), and buffer format (=%d)",
srcType, bufType));
}
Ptr<BaseColumnFilter> getLinearColumnFilter(
int bufType, int dstType,
const Mat& kernel, int anchor,
int symmetryType, double delta,
int bits)
{
CV_INSTRUMENT_REGION();
int sdepth = CV_MAT_DEPTH(bufType), ddepth = CV_MAT_DEPTH(dstType);
int cn = CV_MAT_CN(dstType);
CV_Assert( cn == CV_MAT_CN(bufType) &&
sdepth >= std::max(ddepth, CV_32S) &&
kernel.type() == sdepth );
if( !(symmetryType & (KERNEL_SYMMETRICAL|KERNEL_ASYMMETRICAL)) )
{
if( ddepth == CV_8U && sdepth == CV_32S )
return makePtr<ColumnFilter<FixedPtCastEx<int, uchar>, ColumnNoVec> >
(kernel, anchor, delta, FixedPtCastEx<int, uchar>(bits));
if( ddepth == CV_8U && sdepth == CV_32F )
return makePtr<ColumnFilter<Cast<float, uchar>, ColumnNoVec> >(kernel, anchor, delta);
if( ddepth == CV_8U && sdepth == CV_64F )
return makePtr<ColumnFilter<Cast<double, uchar>, ColumnNoVec> >(kernel, anchor, delta);
if( ddepth == CV_16U && sdepth == CV_32F )
return makePtr<ColumnFilter<Cast<float, ushort>, ColumnNoVec> >(kernel, anchor, delta);
if( ddepth == CV_16U && sdepth == CV_64F )
return makePtr<ColumnFilter<Cast<double, ushort>, ColumnNoVec> >(kernel, anchor, delta);
if( ddepth == CV_16S && sdepth == CV_32F )
return makePtr<ColumnFilter<Cast<float, short>, ColumnNoVec> >(kernel, anchor, delta);
if( ddepth == CV_16S && sdepth == CV_64F )
return makePtr<ColumnFilter<Cast<double, short>, ColumnNoVec> >(kernel, anchor, delta);
if( ddepth == CV_32F && sdepth == CV_32F )
return makePtr<ColumnFilter<Cast<float, float>, ColumnNoVec> >(kernel, anchor, delta);
if( ddepth == CV_64F && sdepth == CV_64F )
return makePtr<ColumnFilter<Cast<double, double>, ColumnNoVec> >(kernel, anchor, delta);
}
else
{
int ksize = kernel.rows + kernel.cols - 1;
if( ksize == 3 )
{
if( ddepth == CV_8U && sdepth == CV_32S )
return makePtr<SymmColumnSmallFilter<
FixedPtCastEx<int, uchar>, SymmColumnVec_32s8u> >
(kernel, anchor, delta, symmetryType, FixedPtCastEx<int, uchar>(bits),
SymmColumnVec_32s8u(kernel, symmetryType, bits, delta));
if( ddepth == CV_16S && sdepth == CV_32S && bits == 0 )
return makePtr<SymmColumnSmallFilter<Cast<int, short>,
SymmColumnSmallVec_32s16s> >(kernel, anchor, delta, symmetryType,
Cast<int, short>(), SymmColumnSmallVec_32s16s(kernel, symmetryType, bits, delta));
if( ddepth == CV_32F && sdepth == CV_32F )
return makePtr<SymmColumnSmallFilter<
Cast<float, float>,SymmColumnSmallVec_32f> >
(kernel, anchor, delta, symmetryType, Cast<float, float>(),
SymmColumnSmallVec_32f(kernel, symmetryType, 0, delta));
}
if( ddepth == CV_8U && sdepth == CV_32S )
return makePtr<SymmColumnFilter<FixedPtCastEx<int, uchar>, SymmColumnVec_32s8u> >
(kernel, anchor, delta, symmetryType, FixedPtCastEx<int, uchar>(bits),
SymmColumnVec_32s8u(kernel, symmetryType, bits, delta));
if( ddepth == CV_8U && sdepth == CV_32F )
return makePtr<SymmColumnFilter<Cast<float, uchar>, SymmColumnVec_32f8u> >
(kernel, anchor, delta, symmetryType, Cast<float, uchar>(),
SymmColumnVec_32f8u(kernel, symmetryType, 0, delta));
if( ddepth == CV_8U && sdepth == CV_64F )
return makePtr<SymmColumnFilter<Cast<double, uchar>, ColumnNoVec> >
(kernel, anchor, delta, symmetryType);
if( ddepth == CV_16U && sdepth == CV_32F )
return makePtr<SymmColumnFilter<Cast<float, ushort>, ColumnNoVec> >
(kernel, anchor, delta, symmetryType);
if( ddepth == CV_16U && sdepth == CV_64F )
return makePtr<SymmColumnFilter<Cast<double, ushort>, ColumnNoVec> >
(kernel, anchor, delta, symmetryType);
if( ddepth == CV_16S && sdepth == CV_32S )
return makePtr<SymmColumnFilter<Cast<int, short>, ColumnNoVec> >
(kernel, anchor, delta, symmetryType);
if( ddepth == CV_16S && sdepth == CV_32F )
return makePtr<SymmColumnFilter<Cast<float, short>, SymmColumnVec_32f16s> >
(kernel, anchor, delta, symmetryType, Cast<float, short>(),
SymmColumnVec_32f16s(kernel, symmetryType, 0, delta));
if( ddepth == CV_16S && sdepth == CV_64F )
return makePtr<SymmColumnFilter<Cast<double, short>, ColumnNoVec> >
(kernel, anchor, delta, symmetryType);
if( ddepth == CV_32F && sdepth == CV_32F )
return makePtr<SymmColumnFilter<Cast<float, float>, SymmColumnVec_32f> >
(kernel, anchor, delta, symmetryType, Cast<float, float>(),
SymmColumnVec_32f(kernel, symmetryType, 0, delta));
if( ddepth == CV_64F && sdepth == CV_64F )
return makePtr<SymmColumnFilter<Cast<double, double>, ColumnNoVec> >
(kernel, anchor, delta, symmetryType);
}
CV_Error_( cv::Error::StsNotImplemented,
("Unsupported combination of buffer format (=%d), and destination format (=%d)",
bufType, dstType));
}
template<typename ST, class CastOp, class VecOp> struct Filter2D : public BaseFilter
{
typedef typename CastOp::type1 KT;
typedef typename CastOp::rtype DT;
Filter2D( const Mat& _kernel, Point _anchor,
double _delta, const CastOp& _castOp=CastOp(),
const VecOp& _vecOp=VecOp() )
{
anchor = _anchor;
ksize = _kernel.size();
delta = saturate_cast<KT>(_delta);
castOp0 = _castOp;
vecOp = _vecOp;
CV_Assert( _kernel.type() == DataType<KT>::type );
preprocess2DKernel( _kernel, coords, coeffs );
ptrs.resize( coords.size() );
}
void operator()(const uchar** src, uchar* dst, int dststep, int count, int width, int cn) CV_OVERRIDE
{
KT _delta = delta;
const Point* pt = &coords[0];
const KT* kf = (const KT*)&coeffs[0];
const ST** kp = (const ST**)&ptrs[0];
int i, k, nz = (int)coords.size();
CastOp castOp = castOp0;
width *= cn;
for( ; count > 0; count--, dst += dststep, src++ )
{
DT* D = (DT*)dst;
for( k = 0; k < nz; k++ )
kp[k] = (const ST*)src[pt[k].y] + pt[k].x*cn;
i = vecOp((const uchar**)kp, dst, width);
#if CV_ENABLE_UNROLLED
for( ; i <= width - 4; i += 4 )
{
KT s0 = _delta, s1 = _delta, s2 = _delta, s3 = _delta;
for( k = 0; k < nz; k++ )
{
const ST* sptr = kp[k] + i;
KT f = kf[k];
s0 += f*sptr[0];
s1 += f*sptr[1];
s2 += f*sptr[2];
s3 += f*sptr[3];
}
D[i] = castOp(s0); D[i+1] = castOp(s1);
D[i+2] = castOp(s2); D[i+3] = castOp(s3);
}
#endif
for( ; i < width; i++ )
{
KT s0 = _delta;
for( k = 0; k < nz; k++ )
s0 += kf[k]*kp[k][i];
D[i] = castOp(s0);
}
}
}
std::vector<Point> coords;
std::vector<uchar> coeffs;
std::vector<uchar*> ptrs;
KT delta;
CastOp castOp0;
VecOp vecOp;
};
Ptr<BaseFilter> getLinearFilter(
int srcType, int dstType,
const Mat& _kernel, Point anchor,
double delta, int bits)
{
CV_INSTRUMENT_REGION();
int sdepth = CV_MAT_DEPTH(srcType), ddepth = CV_MAT_DEPTH(dstType);
int cn = CV_MAT_CN(srcType), kdepth = _kernel.depth();
CV_Assert( cn == CV_MAT_CN(dstType) && ddepth >= sdepth );
anchor = normalizeAnchor(anchor, _kernel.size());
/*if( sdepth == CV_8U && ddepth == CV_8U && kdepth == CV_32S )
return makePtr<Filter2D<uchar, FixedPtCastEx<int, uchar>, FilterVec_8u> >
(_kernel, anchor, delta, FixedPtCastEx<int, uchar>(bits),
FilterVec_8u(_kernel, bits, delta));
if( sdepth == CV_8U && ddepth == CV_16S && kdepth == CV_32S )
return makePtr<Filter2D<uchar, FixedPtCastEx<int, short>, FilterVec_8u16s> >
(_kernel, anchor, delta, FixedPtCastEx<int, short>(bits),
FilterVec_8u16s(_kernel, bits, delta));*/
kdepth = sdepth == CV_64F || ddepth == CV_64F ? CV_64F : CV_32F;
Mat kernel;
if( _kernel.type() == kdepth )
kernel = _kernel;
else
_kernel.convertTo(kernel, kdepth, _kernel.type() == CV_32S ? 1./(1 << bits) : 1.);
if( sdepth == CV_8U && ddepth == CV_8U )
return makePtr<Filter2D<uchar, Cast<float, uchar>, FilterVec_8u> >
(kernel, anchor, delta, Cast<float, uchar>(), FilterVec_8u(kernel, 0, delta));
if( sdepth == CV_8U && ddepth == CV_16U )
return makePtr<Filter2D<uchar,
Cast<float, ushort>, FilterNoVec> >(kernel, anchor, delta);
if( sdepth == CV_8U && ddepth == CV_16S )
return makePtr<Filter2D<uchar, Cast<float, short>, FilterVec_8u16s> >
(kernel, anchor, delta, Cast<float, short>(), FilterVec_8u16s(kernel, 0, delta));
if( sdepth == CV_8U && ddepth == CV_32F )
return makePtr<Filter2D<uchar,
Cast<float, float>, FilterNoVec> >(kernel, anchor, delta);
if( sdepth == CV_8U && ddepth == CV_64F )
return makePtr<Filter2D<uchar,
Cast<double, double>, FilterNoVec> >(kernel, anchor, delta);
if( sdepth == CV_16U && ddepth == CV_16U )
return makePtr<Filter2D<ushort,
Cast<float, ushort>, FilterNoVec> >(kernel, anchor, delta);
if( sdepth == CV_16U && ddepth == CV_32F )
return makePtr<Filter2D<ushort,
Cast<float, float>, FilterNoVec> >(kernel, anchor, delta);
if( sdepth == CV_16U && ddepth == CV_64F )
return makePtr<Filter2D<ushort,
Cast<double, double>, FilterNoVec> >(kernel, anchor, delta);
if( sdepth == CV_16S && ddepth == CV_16S )
return makePtr<Filter2D<short,
Cast<float, short>, FilterNoVec> >(kernel, anchor, delta);
if( sdepth == CV_16S && ddepth == CV_32F )
return makePtr<Filter2D<short,
Cast<float, float>, FilterNoVec> >(kernel, anchor, delta);
if( sdepth == CV_16S && ddepth == CV_64F )
return makePtr<Filter2D<short,
Cast<double, double>, FilterNoVec> >(kernel, anchor, delta);
if( sdepth == CV_32F && ddepth == CV_32F )
return makePtr<Filter2D<float, Cast<float, float>, FilterVec_32f> >
(kernel, anchor, delta, Cast<float, float>(), FilterVec_32f(kernel, 0, delta));
if( sdepth == CV_64F && ddepth == CV_64F )
return makePtr<Filter2D<double,
Cast<double, double>, FilterNoVec> >(kernel, anchor, delta);
CV_Error_( cv::Error::StsNotImplemented,
("Unsupported combination of source format (=%d), and destination format (=%d)",
srcType, dstType));
}
#endif
CV_CPU_OPTIMIZATION_NAMESPACE_END
} // namespace
/*M///////////////////////////////////////////////////////////////////////////////////////
//
// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
// By downloading, copying, installing or using the software you agree to this license.
// If you do not agree to this license, do not download, install,
// copy or use the software.
//
//
// License Agreement
// For Open Source Computer Vision Library
//
// Copyright (C) 2000-2008, Intel Corporation, all rights reserved.
// Copyright (C) 2009, Willow Garage Inc., all rights reserved.
// Third party copyrights are property of their respective owners.
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
// * Redistribution's of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
//
// * Redistribution's in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// * The name of the copyright holders may not be used to endorse or promote products
// derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors "as is" and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the Intel Corporation or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/
/****************************************************************************************\
Base Image Filter
\****************************************************************************************/
namespace cv {
CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN
// forward declarations
int FilterEngine__start(FilterEngine& this_, const Size &_wholeSize, const Size &sz, const Point &ofs);
int FilterEngine__proceed(FilterEngine& this_, const uchar* src, int srcstep, int count,
uchar* dst, int dststep);
void FilterEngine__apply(FilterEngine& this_, const Mat& src, Mat& dst, const Size& wsz, const Point& ofs);
Ptr<BaseRowFilter> getLinearRowFilter(
int srcType, int bufType,
const Mat& kernel, int anchor,
int symmetryType);
Ptr<BaseColumnFilter> getLinearColumnFilter(
int bufType, int dstType,
const Mat& kernel, int anchor,
int symmetryType, double delta,
int bits);
Ptr<BaseFilter> getLinearFilter(
int srcType, int dstType,
const Mat& filter_kernel, Point anchor,
double delta, int bits);
int FilterEngine__start(FilterEngine& this_, const Size &_wholeSize, const Size &sz, const Point &ofs)
{
CV_INSTRUMENT_REGION();
int i, j;
this_.wholeSize = _wholeSize;
this_.roi = Rect(ofs, sz);
CV_Assert( this_.roi.x >= 0 && this_.roi.y >= 0 && this_.roi.width >= 0 && this_.roi.height >= 0 &&
this_.roi.x + this_.roi.width <= this_.wholeSize.width &&