test / modules /dnn /src /layers /cpu_kernels /conv_block.simd.hpp
tgollogly's picture
Upload 7492 files
dffe123 verified
Raw History Blame Contribute Delete
27 kB
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
#include "opencv2/core/hal/intrin.hpp"
namespace cv {
namespace dnn {
CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN
void convBlock_F32(int np, const float* a, const float* b, float* c, int ldc, bool init_c, int width, const int convMR, const int convNR);
// FP 16 branch.
void convBlock_F16(int np, const char * _a, const char * _b, char * _c, int ldc, bool init_c, int width,
const int convMR_fp16, const int convNR_fp16);
void convBlockMR1_F16(int np, const char* _a, const char* _b, float *c, const float _bias, bool init_c,
const float minval, const float maxval, bool ifMinMaxAct, const int width, const int convNR_FP16);
#if !defined(CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY)
#if CV_AVX
#if !CV_FMA3 // AVX workaround
#undef _mm256_fmadd_ps
#define _mm256_fmadd_ps(a, b, c) _mm256_add_ps(c, _mm256_mul_ps(a, b))
#endif
void convBlock_F32(int np, const float* a, const float* b, float* c, int ldc, bool init_c, int width, const int convMR, const int convNR)
{
CV_Assert(convMR == 4 && convNR == 24);
__m256 c00 = _mm256_set1_ps(0.f), c01 = c00, c02 = c00;
__m256 c10 = c00, c11 = c00, c12 = c00;
__m256 c20 = c00, c21 = c00, c22 = c00;
__m256 c30 = c00, c31 = c00, c32 = c00;
__m256 a0 = _mm256_setzero_ps(), a1 = _mm256_setzero_ps();
__m256 b0 = _mm256_setzero_ps(), b1 = _mm256_setzero_ps(), b2 = _mm256_setzero_ps();
if (width > 16)
{
for (int p = 0; p < np; p++, a += convMR, b += convNR)
{
a0 = _mm256_set1_ps(a[0]), a1 = _mm256_set1_ps(a[1]);
b0 = _mm256_load_ps(b), b1 = _mm256_load_ps(b + 8), b2 = _mm256_load_ps(b + 16);
c00 = _mm256_fmadd_ps(b0, a0, c00);
c01 = _mm256_fmadd_ps(b1, a0, c01);
c02 = _mm256_fmadd_ps(b2, a0, c02);
c10 = _mm256_fmadd_ps(b0, a1, c10);
c11 = _mm256_fmadd_ps(b1, a1, c11);
c12 = _mm256_fmadd_ps(b2, a1, c12);
a0 = _mm256_set1_ps(a[2]), a1 = _mm256_set1_ps(a[3]);
c20 = _mm256_fmadd_ps(b0, a0, c20);
c21 = _mm256_fmadd_ps(b1, a0, c21);
c22 = _mm256_fmadd_ps(b2, a0, c22);
c30 = _mm256_fmadd_ps(b0, a1, c30);
c31 = _mm256_fmadd_ps(b1, a1, c31);
c32 = _mm256_fmadd_ps(b2, a1, c32);
}
}
else if (width > 8)
{
for (int p = 0; p < np; p++, a += convMR, b += convNR)
{
a0 = _mm256_set1_ps(a[0]), a1 = _mm256_set1_ps(a[1]);
b0 = _mm256_load_ps(b), b1 = _mm256_load_ps(b + 8);
c00 = _mm256_fmadd_ps(b0, a0, c00);
c01 = _mm256_fmadd_ps(b1, a0, c01);
c10 = _mm256_fmadd_ps(b0, a1, c10);
c11 = _mm256_fmadd_ps(b1, a1, c11);
a0 = _mm256_set1_ps(a[2]), a1 = _mm256_set1_ps(a[3]);
c20 = _mm256_fmadd_ps(b0, a0, c20);
c21 = _mm256_fmadd_ps(b1, a0, c21);
c30 = _mm256_fmadd_ps(b0, a1, c30);
c31 = _mm256_fmadd_ps(b1, a1, c31);
}
}
else
{
for (int p = 0; p < np; p++, a += convMR, b += convNR)
{
a0 = _mm256_set1_ps(a[0]), a1 = _mm256_set1_ps(a[1]);
b0 = _mm256_load_ps(b);
c00 = _mm256_fmadd_ps(b0, a0, c00);
c10 = _mm256_fmadd_ps(b0, a1, c10);
a0 = _mm256_set1_ps(a[2]), a1 = _mm256_set1_ps(a[3]);
c20 = _mm256_fmadd_ps(b0, a0, c20);
c30 = _mm256_fmadd_ps(b0, a1, c30);
}
}
if (!init_c)
{
c00 = _mm256_add_ps(c00, _mm256_load_ps(c));
c01 = _mm256_add_ps(c01, _mm256_load_ps(c + 8));
c02 = _mm256_add_ps(c02, _mm256_load_ps(c + 16));
c10 = _mm256_add_ps(c10, _mm256_load_ps(c + ldc));
c11 = _mm256_add_ps(c11, _mm256_load_ps(c + ldc + 8));
c12 = _mm256_add_ps(c12, _mm256_load_ps(c + ldc + 16));
c20 = _mm256_add_ps(c20, _mm256_load_ps(c + ldc*2));
c21 = _mm256_add_ps(c21, _mm256_load_ps(c + ldc*2 + 8));
c22 = _mm256_add_ps(c22, _mm256_load_ps(c + ldc*2 + 16));
c30 = _mm256_add_ps(c30, _mm256_load_ps(c + ldc*3));
c31 = _mm256_add_ps(c31, _mm256_load_ps(c + ldc*3 + 8));
c32 = _mm256_add_ps(c32, _mm256_load_ps(c + ldc*3 + 16));
}
_mm256_storeu_ps(c, c00), _mm256_storeu_ps(c+8, c01), _mm256_storeu_ps(c+16, c02);
_mm256_storeu_ps(c + ldc, c10), _mm256_storeu_ps(c + ldc + 8, c11), _mm256_storeu_ps(c + ldc + 16, c12);
_mm256_storeu_ps(c + ldc*2, c20), _mm256_storeu_ps(c + ldc*2 + 8, c21), _mm256_storeu_ps(c + ldc*2 + 16, c22);
_mm256_storeu_ps(c + ldc*3, c30), _mm256_storeu_ps(c + ldc*3 + 8, c31), _mm256_storeu_ps(c + ldc*3 + 16, c32);
_mm256_zeroupper();
}
#endif
#if CV_NEON
void convBlock_F32(int np, const float* a, const float* b, float* c, int ldc, bool init_c, int width, const int convMR, const int convNR)
{
#if CV_NEON_AARCH64
if (convMR == 4 && convNR == 28) // AARCH64
{
float32x4_t c00 = vdupq_n_f32(0.f), c01 = c00, c02 = c00, c03 = c00, c04 = c00, c05 = c00, c06 = c00;
float32x4_t c10 = vdupq_n_f32(0.f), c11 = c10, c12 = c10, c13 = c10, c14 = c10, c15 = c10, c16 = c10;
float32x4_t c20 = vdupq_n_f32(0.f), c21 = c20, c22 = c20, c23 = c20, c24 = c20, c25 = c20, c26 = c20;
float32x4_t c30 = vdupq_n_f32(0.f), c31 = c30, c32 = c30, c33 = c30, c34 = c30, c35 = c30, c36 = c30;
if (width > 16)
{
for( int p = 0; p < np; p++, a += convMR, b += convNR )
{
float32x4_t a0 = vld1q_f32(a), b0, b1, b2;
b0 = vld1q_f32(b); b1 = vld1q_f32(b + 4); b2 = vld1q_f32(b + 8);
c00 = vfmaq_laneq_f32(c00, b0, a0, 0);
c01 = vfmaq_laneq_f32(c01, b1, a0, 0);
c02 = vfmaq_laneq_f32(c02, b2, a0, 0);
c10 = vfmaq_laneq_f32(c10, b0, a0, 1);
c11 = vfmaq_laneq_f32(c11, b1, a0, 1);
c12 = vfmaq_laneq_f32(c12, b2, a0, 1);
c20 = vfmaq_laneq_f32(c20, b0, a0, 2);
c21 = vfmaq_laneq_f32(c21, b1, a0, 2);
c22 = vfmaq_laneq_f32(c22, b2, a0, 2);
c30 = vfmaq_laneq_f32(c30, b0, a0, 3);
c31 = vfmaq_laneq_f32(c31, b1, a0, 3);
c32 = vfmaq_laneq_f32(c32, b2, a0, 3);
b0 = vld1q_f32(b + 12); b1 = vld1q_f32(b + 16); b2 = vld1q_f32(b + 20);
c03 = vfmaq_laneq_f32(c03, b0, a0, 0);
c04 = vfmaq_laneq_f32(c04, b1, a0, 0);
c05 = vfmaq_laneq_f32(c05, b2, a0, 0);
c13 = vfmaq_laneq_f32(c13, b0, a0, 1);
c14 = vfmaq_laneq_f32(c14, b1, a0, 1);
c15 = vfmaq_laneq_f32(c15, b2, a0, 1);
c23 = vfmaq_laneq_f32(c23, b0, a0, 2);
c24 = vfmaq_laneq_f32(c24, b1, a0, 2);
c25 = vfmaq_laneq_f32(c25, b2, a0, 2);
c33 = vfmaq_laneq_f32(c33, b0, a0, 3);
c34 = vfmaq_laneq_f32(c34, b1, a0, 3);
c35 = vfmaq_laneq_f32(c35, b2, a0, 3);
b0 = vld1q_f32(b + 24);
c06 = vfmaq_laneq_f32(c06, b0, a0, 0);
c16 = vfmaq_laneq_f32(c16, b0, a0, 1);
c26 = vfmaq_laneq_f32(c26, b0, a0, 2);
c36 = vfmaq_laneq_f32(c36, b0, a0, 3);
}
}
else if (width > 8)
{
for( int p = 0; p < np; p++, a += convMR, b += convNR )
{
float32x4_t a0 = vld1q_f32(a), b0, b1, b2;
b0 = vld1q_f32(b); b1 = vld1q_f32(b + 4); b2 = vld1q_f32(b + 8);
c00 = vfmaq_laneq_f32(c00, b0, a0, 0);
c01 = vfmaq_laneq_f32(c01, b1, a0, 0);
c02 = vfmaq_laneq_f32(c02, b2, a0, 0);
c10 = vfmaq_laneq_f32(c10, b0, a0, 1);
c11 = vfmaq_laneq_f32(c11, b1, a0, 1);
c12 = vfmaq_laneq_f32(c12, b2, a0, 1);
c20 = vfmaq_laneq_f32(c20, b0, a0, 2);
c21 = vfmaq_laneq_f32(c21, b1, a0, 2);
c22 = vfmaq_laneq_f32(c22, b2, a0, 2);
c30 = vfmaq_laneq_f32(c30, b0, a0, 3);
c31 = vfmaq_laneq_f32(c31, b1, a0, 3);
c32 = vfmaq_laneq_f32(c32, b2, a0, 3);
b0 = vld1q_f32(b + 12);
c03 = vfmaq_laneq_f32(c03, b0, a0, 0);
c13 = vfmaq_laneq_f32(c13, b0, a0, 1);
c23 = vfmaq_laneq_f32(c23, b0, a0, 2);
c33 = vfmaq_laneq_f32(c33, b0, a0, 3);
}
}
else if (width > 4)
{
for( int p = 0; p < np; p++, a += convMR, b += convNR )
{
float32x4_t a0 = vld1q_f32(a), b0, b1;
b0 = vld1q_f32(b); b1 = vld1q_f32(b + 4);
c00 = vfmaq_laneq_f32(c00, b0, a0, 0);
c01 = vfmaq_laneq_f32(c01, b1, a0, 0);
c10 = vfmaq_laneq_f32(c10, b0, a0, 1);
c11 = vfmaq_laneq_f32(c11, b1, a0, 1);
c20 = vfmaq_laneq_f32(c20, b0, a0, 2);
c21 = vfmaq_laneq_f32(c21, b1, a0, 2);
c30 = vfmaq_laneq_f32(c30, b0, a0, 3);
c31 = vfmaq_laneq_f32(c31, b1, a0, 3);
}
}
else
{
for( int p = 0; p < np; p++, a += convMR, b += convNR )
{
float32x4_t a0 = vld1q_f32(a), b0;
b0 = vld1q_f32(b);
c00 = vfmaq_laneq_f32(c00, b0, a0, 0);
c10 = vfmaq_laneq_f32(c10, b0, a0, 1);
c20 = vfmaq_laneq_f32(c20, b0, a0, 2);
c30 = vfmaq_laneq_f32(c30, b0, a0, 3);
}
}
if (!init_c)
{
c00 = vaddq_f32(c00, vld1q_f32(c));
c01 = vaddq_f32(c01, vld1q_f32(c + 4));
c02 = vaddq_f32(c02, vld1q_f32(c + 8));
c03 = vaddq_f32(c03, vld1q_f32(c + 12));
c04 = vaddq_f32(c04, vld1q_f32(c + 16));
c05 = vaddq_f32(c05, vld1q_f32(c + 20));
c06 = vaddq_f32(c06, vld1q_f32(c + 24));
c10 = vaddq_f32(c10, vld1q_f32(c + ldc));
c11 = vaddq_f32(c11, vld1q_f32(c + ldc + 4));
c12 = vaddq_f32(c12, vld1q_f32(c + ldc + 8));
c13 = vaddq_f32(c13, vld1q_f32(c + ldc + 12));
c14 = vaddq_f32(c14, vld1q_f32(c + ldc + 16));
c15 = vaddq_f32(c15, vld1q_f32(c + ldc + 20));
c16 = vaddq_f32(c16, vld1q_f32(c + ldc + 24));
c20 = vaddq_f32(c20, vld1q_f32(c + ldc*2));
c21 = vaddq_f32(c21, vld1q_f32(c + ldc*2 + 4));
c22 = vaddq_f32(c22, vld1q_f32(c + ldc*2 + 8));
c23 = vaddq_f32(c23, vld1q_f32(c + ldc*2 + 12));
c24 = vaddq_f32(c24, vld1q_f32(c + ldc*2 + 16));
c25 = vaddq_f32(c25, vld1q_f32(c + ldc*2 + 20));
c26 = vaddq_f32(c26, vld1q_f32(c + ldc*2 + 24));
c30 = vaddq_f32(c30, vld1q_f32(c + ldc*3));
c31 = vaddq_f32(c31, vld1q_f32(c + ldc*3 + 4));
c32 = vaddq_f32(c32, vld1q_f32(c + ldc*3 + 8));
c33 = vaddq_f32(c33, vld1q_f32(c + ldc*3 + 12));
c34 = vaddq_f32(c34, vld1q_f32(c + ldc*3 + 16));
c35 = vaddq_f32(c35, vld1q_f32(c + ldc*3 + 20));
c36 = vaddq_f32(c36, vld1q_f32(c + ldc*3 + 24));
}
vst1q_f32(c, c00); vst1q_f32(c+4, c01);
vst1q_f32(c+8, c02); vst1q_f32(c+12, c03);
vst1q_f32(c+16, c04); vst1q_f32(c+20, c05);
vst1q_f32(c+24, c06);
vst1q_f32(c+ldc, c10); vst1q_f32(c+ldc+4, c11);
vst1q_f32(c+ldc+8, c12); vst1q_f32(c+ldc+12, c13);
vst1q_f32(c+ldc+16, c14); vst1q_f32(c+ldc+20, c15);
vst1q_f32(c+ldc+24, c16);
vst1q_f32(c+ldc*2, c20); vst1q_f32(c+ldc*2+4, c21);
vst1q_f32(c+ldc*2+8, c22); vst1q_f32(c+ldc*2+12, c23);
vst1q_f32(c+ldc*2+16, c24); vst1q_f32(c+ldc*2+20, c25);
vst1q_f32(c+ldc*2+24, c26);
vst1q_f32(c+ldc*3, c30); vst1q_f32(c+ldc*3+4, c31);
vst1q_f32(c+ldc*3+8, c32); vst1q_f32(c+ldc*3+12, c33);
vst1q_f32(c+ldc*3+16, c34); vst1q_f32(c+ldc*3+20, c35);
vst1q_f32(c+ldc*3+24, c36);
}
else
#endif
if (convMR == 4 && convNR == 12) // ARMv7
{
float32x4_t c0 = vdupq_n_f32(0.f), c1 = c0, c2 = c0;
float32x4_t c3 = vdupq_n_f32(0.f), c4 = c3, c5 = c3;
float32x4_t c6 = vdupq_n_f32(0.f), c7 = c6, c8 = c6;
float32x4_t c9 = vdupq_n_f32(0.f), c10 = c9, c11 = c9;
float32x2_t a0 = vdup_n_f32(0.0f), a1 = a0;
float32x4_t b0 = vdupq_n_f32(0.0f), b1 = vdupq_n_f32(0.0f), b2 = vdupq_n_f32(0.0f);
if (width > 8)
{
for (int p = 0; p < np; p++, a += convMR, b += convNR)
{
a0 = vld1_f32(a), a1 = vld1_f32(a+2);
b0 = vld1q_f32(b), b1 = vld1q_f32(b + 4), b2 = vld1q_f32(b + 8);
c0 = vmlaq_lane_f32(c0, b0, a0, 0);
c1 = vmlaq_lane_f32(c1, b1, a0, 0);
c2 = vmlaq_lane_f32(c2, b2, a0, 0);
c3 = vmlaq_lane_f32(c3, b0, a0, 1);
c4 = vmlaq_lane_f32(c4, b1, a0, 1);
c5 = vmlaq_lane_f32(c5, b2, a0, 1);
c6 = vmlaq_lane_f32(c6, b0, a1, 0);
c7 = vmlaq_lane_f32(c7, b1, a1, 0);
c8 = vmlaq_lane_f32(c8, b2, a1, 0);
c9 = vmlaq_lane_f32(c9 , b0, a1, 1);
c10 = vmlaq_lane_f32(c10, b1, a1, 1);
c11 = vmlaq_lane_f32(c11, b2, a1, 1);
}
}
else if (width > 4)
{
for (int p = 0; p < np; p++, a += convMR, b += convNR)
{
a0 = vld1_f32(a), a1 = vld1_f32(a+2);
b0 = vld1q_f32(b), b1 = vld1q_f32(b + 4);
c0 = vmlaq_lane_f32(c0, b0, a0, 0);
c1 = vmlaq_lane_f32(c1, b1, a0, 0);
c3 = vmlaq_lane_f32(c3, b0, a0, 1);
c4 = vmlaq_lane_f32(c4, b1, a0, 1);
c6 = vmlaq_lane_f32(c6, b0, a1, 0);
c7 = vmlaq_lane_f32(c7, b1, a1, 0);
c9 = vmlaq_lane_f32(c9 , b0, a1, 1);
c10 = vmlaq_lane_f32(c10, b1, a1, 1);
}
}
else
{
for (int p = 0; p < np; p++, a += convMR, b += convNR)
{
a0 = vld1_f32(a), a1 = vld1_f32(a+2);
b0 = vld1q_f32(b);
c0 = vmlaq_lane_f32(c0, b0, a0, 0);
c3 = vmlaq_lane_f32(c3, b0, a0, 1);
c6 = vmlaq_lane_f32(c6, b0, a1, 0);
c9 = vmlaq_lane_f32(c9 , b0, a1, 1);
}
}
if (!init_c)
{
c0 = vaddq_f32(c0, vld1q_f32(c));
c1 = vaddq_f32(c1, vld1q_f32(c + 4));
c2 = vaddq_f32(c2, vld1q_f32(c + 8));
c3 = vaddq_f32(c3, vld1q_f32(c + ldc));
c4 = vaddq_f32(c4, vld1q_f32(c + ldc + 4));
c5 = vaddq_f32(c5, vld1q_f32(c + ldc + 8));
c6 = vaddq_f32(c6, vld1q_f32(c + ldc * 2));
c7 = vaddq_f32(c7, vld1q_f32(c + ldc * 2 + 4));
c8 = vaddq_f32(c8, vld1q_f32(c + ldc * 2 + 8));
c9 = vaddq_f32(c9 , vld1q_f32(c + ldc * 3));
c10 = vaddq_f32(c10, vld1q_f32(c + ldc * 3 + 4));
c11 = vaddq_f32(c11, vld1q_f32(c + ldc * 3 + 8));
}
vst1q_f32(c, c0), vst1q_f32(c+4, c1), vst1q_f32(c+8, c2);
vst1q_f32(c + ldc, c3), vst1q_f32(c + ldc + 4, c4), vst1q_f32(c + ldc + 8, c5);
vst1q_f32(c + ldc*2, c6), vst1q_f32(c + ldc*2 + 4, c7), vst1q_f32(c + ldc*2 + 8, c8);
vst1q_f32(c + ldc*3, c9), vst1q_f32(c + ldc*3 + 4, c10), vst1q_f32(c + ldc*3 + 8, c11);
}
else
CV_Error(Error::StsNotImplemented, "Unsupported convMR and/or convNR in opt_NEON::convBlock");
}
void convBlockMR1_F32(int np, const float * a, const float * b, float *c, const float bias, bool init_c,
const float minval, const float maxval, bool ifMinMaxAct, const int width, const int convNR)
{
CV_Assert(convNR == 28);
float32x4_t c0 = vdupq_n_f32(bias), c1 = c0, c2 = c0;
float32x4_t c3 = c0, c4 = c0, c5 = c0, c6 = c0;
if (width > 16)
{
for (int p = 0; p < np; p++, a++, b += convNR)
{
float32x4_t b0 = vld1q_f32(b), b1 = vld1q_f32(b + 4), b2 = vld1q_f32(b + 8);
float32x4_t b3 = vld1q_f32(b + 12), b4 = vld1q_f32(b + 16), b5 = vld1q_f32(b + 20);
float32x4_t b6 = vld1q_f32(b + 24);
c0 = vmlaq_n_f32(c0, b0, a[0]);
c1 = vmlaq_n_f32(c1, b1, a[0]);
c2 = vmlaq_n_f32(c2, b2, a[0]);
c3 = vmlaq_n_f32(c3, b3, a[0]);
c4 = vmlaq_n_f32(c4, b4, a[0]);
c5 = vmlaq_n_f32(c5, b5, a[0]);
c6 = vmlaq_n_f32(c6, b6, a[0]);
}
}
else if (width > 8)
{
for (int p = 0; p < np; p++, a++, b += convNR)
{
float32x4_t b0 = vld1q_f32(b), b1 = vld1q_f32(b + 4), b2 = vld1q_f32(b + 8);
float32x4_t b3 = vld1q_f32(b + 12);
c0 = vmlaq_n_f32(c0, b0, a[0]);
c1 = vmlaq_n_f32(c1, b1, a[0]);
c2 = vmlaq_n_f32(c2, b2, a[0]);
c3 = vmlaq_n_f32(c3, b3, a[0]);
}
}
else if (width > 4)
{
for (int p = 0; p < np; p++, a++, b += convNR)
{
float32x4_t b0 = vld1q_f32(b), b1 = vld1q_f32(b + 4);
c0 = vmlaq_n_f32(c0, b0, a[0]);
c1 = vmlaq_n_f32(c1, b1, a[0]);
}
}
else
{
for (int p = 0; p < np; p++, a++, b += convNR)
{
float32x4_t b0 = vld1q_f32(b);
c0 = vmlaq_n_f32(c0, b0, a[0]);
}
}
if (init_c)
{
c0 = vaddq_f32(c0, vld1q_f32(c));
c1 = vaddq_f32(c1, vld1q_f32(c + 4));
c2 = vaddq_f32(c2, vld1q_f32(c + 8));
c3 = vaddq_f32(c3, vld1q_f32(c + 12));
c4 = vaddq_f32(c4, vld1q_f32(c + 16));
c5 = vaddq_f32(c5, vld1q_f32(c + 20));
c6 = vaddq_f32(c6, vld1q_f32(c + 24));
}
if (ifMinMaxAct)
{
float32x4_t v_minval = vdupq_n_f32(minval), v_maxval = vdupq_n_f32(maxval);
c0 = vminq_f32(vmaxq_f32(c0, v_minval), v_maxval);
c1 = vminq_f32(vmaxq_f32(c1, v_minval), v_maxval);
c2 = vminq_f32(vmaxq_f32(c2, v_minval), v_maxval);
c3 = vminq_f32(vmaxq_f32(c3, v_minval), v_maxval);
c4 = vminq_f32(vmaxq_f32(c4, v_minval), v_maxval);
c5 = vminq_f32(vmaxq_f32(c5, v_minval), v_maxval);
c6 = vminq_f32(vmaxq_f32(c6, v_minval), v_maxval);
}
vst1q_f32(c, c0);
vst1q_f32(c + 4, c1);
vst1q_f32(c + 8, c2);
vst1q_f32(c + 12, c3);
vst1q_f32(c + 16, c4);
vst1q_f32(c + 20, c5);
vst1q_f32(c + 24, c6);
}
#endif
#if defined(CV_NEON_AARCH64) && CV_NEON_AARCH64 && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
void convBlock_F16(int np, const char * _a, const char * _b, char * _c, int ldc, bool init_c, int width,
const int convMR_fp16, const int convNR_fp16)
{
const __fp16* a = (const __fp16*)_a;
const __fp16* b = (const __fp16*)_b;
__fp16* c = (__fp16*)_c;
CV_Assert(convMR_fp16 == 8 && convNR_fp16 == 24);
float16x8_t c00 = vdupq_n_f16(0), c01 = c00, c02 = c00;
float16x8_t c10 = c00, c11 = c00, c12 = c00;
float16x8_t c20 = c00, c21 = c00, c22 = c00;
float16x8_t c30 = c00, c31 = c00, c32 = c00;
float16x8_t c40 = c00, c41 = c00, c42 = c00;
float16x8_t c50 = c00, c51 = c00, c52 = c00;
float16x8_t c60 = c00, c61 = c00, c62 = c00;
float16x8_t c70 = c00, c71 = c00, c72 = c00;
float16x8_t b0 = c00, b1 = c00, b2 = c00;
if (width > 16)
{
for (int p = 0; p < np; p++, a += convMR_fp16, b += convNR_fp16)
{
float16x4_t a0 = vld1_f16(a), a1 = vld1_f16(a + 4);
b0 = vld1q_f16(b), b1 = vld1q_f16(b + 8), b2 = vld1q_f16(b + 16);
c00 = vfmaq_lane_f16(c00, b0, a0, 0);
c01 = vfmaq_lane_f16(c01, b1, a0, 0);
c02 = vfmaq_lane_f16(c02, b2, a0, 0);
c10 = vfmaq_lane_f16(c10, b0, a0, 1);
c11 = vfmaq_lane_f16(c11, b1, a0, 1);
c12 = vfmaq_lane_f16(c12, b2, a0, 1);
c20 = vfmaq_lane_f16(c20, b0, a0, 2);
c21 = vfmaq_lane_f16(c21, b1, a0, 2);
c22 = vfmaq_lane_f16(c22, b2, a0, 2);
c30 = vfmaq_lane_f16(c30, b0, a0, 3);
c31 = vfmaq_lane_f16(c31, b1, a0, 3);
c32 = vfmaq_lane_f16(c32, b2, a0, 3);
c40 = vfmaq_lane_f16(c40, b0, a1, 0);
c41 = vfmaq_lane_f16(c41, b1, a1, 0);
c42 = vfmaq_lane_f16(c42, b2, a1, 0);
c50 = vfmaq_lane_f16(c50, b0, a1, 1);
c51 = vfmaq_lane_f16(c51, b1, a1, 1);
c52 = vfmaq_lane_f16(c52, b2, a1, 1);
c60 = vfmaq_lane_f16(c60, b0, a1, 2);
c61 = vfmaq_lane_f16(c61, b1, a1, 2);
c62 = vfmaq_lane_f16(c62, b2, a1, 2);
c70 = vfmaq_lane_f16(c70, b0, a1, 3);
c71 = vfmaq_lane_f16(c71, b1, a1, 3);
c72 = vfmaq_lane_f16(c72, b2, a1, 3);
}
}
else if (width > 8)
{
for( int p = 0; p < np; p++, a += convMR_fp16, b += convNR_fp16)
{
float16x4_t a0 = vld1_f16(a), a1 = vld1_f16(a + 4);
float16x8_t b0 = vld1q_f16(b), b1 = vld1q_f16(b + 8);
c00 = vfmaq_lane_f16(c00, b0, a0, 0);
c01 = vfmaq_lane_f16(c01, b1, a0, 0);
c10 = vfmaq_lane_f16(c10, b0, a0, 1);
c11 = vfmaq_lane_f16(c11, b1, a0, 1);
c20 = vfmaq_lane_f16(c20, b0, a0, 2);
c21 = vfmaq_lane_f16(c21, b1, a0, 2);
c30 = vfmaq_lane_f16(c30, b0, a0, 3);
c31 = vfmaq_lane_f16(c31, b1, a0, 3);
c40 = vfmaq_lane_f16(c40, b0, a1, 0);
c41 = vfmaq_lane_f16(c41, b1, a1, 0);
c50 = vfmaq_lane_f16(c50, b0, a1, 1);
c51 = vfmaq_lane_f16(c51, b1, a1, 1);
c60 = vfmaq_lane_f16(c60, b0, a1, 2);
c61 = vfmaq_lane_f16(c61, b1, a1, 2);
c70 = vfmaq_lane_f16(c70, b0, a1, 3);
c71 = vfmaq_lane_f16(c71, b1, a1, 3);
}
}
else
{
for( int p = 0; p < np; p++, a += convMR_fp16, b += convNR_fp16)
{
float16x4_t a0 = vld1_f16(a), a1 = vld1_f16(a + 4);
float16x8_t b0 = vld1q_f16(b);
c00 = vfmaq_lane_f16(c00, b0, a0, 0);
c10 = vfmaq_lane_f16(c10, b0, a0, 1);
c20 = vfmaq_lane_f16(c20, b0, a0, 2);
c30 = vfmaq_lane_f16(c30, b0, a0, 3);
c40 = vfmaq_lane_f16(c40, b0, a1, 0);
c50 = vfmaq_lane_f16(c50, b0, a1, 1);
c60 = vfmaq_lane_f16(c60, b0, a1, 2);
c70 = vfmaq_lane_f16(c70, b0, a1, 3);
}
}
if (!init_c)
{
#undef _FX_UPDATE_CBUF_ROW
#define _FX_UPDATE_CBUF_ROW(row) \
c##row##0 = c##row##0 + vld1q_f16(c + row*ldc); \
c##row##1 = c##row##1 + vld1q_f16(c + row*ldc + 8); \
c##row##2 = c##row##2 + vld1q_f16(c + row*ldc + 16)
_FX_UPDATE_CBUF_ROW(0);
_FX_UPDATE_CBUF_ROW(1);
_FX_UPDATE_CBUF_ROW(2);
_FX_UPDATE_CBUF_ROW(3);
_FX_UPDATE_CBUF_ROW(4);
_FX_UPDATE_CBUF_ROW(5);
_FX_UPDATE_CBUF_ROW(6);
_FX_UPDATE_CBUF_ROW(7);
}
#undef _FX_STORE_CBUF_ROW
#define _FX_STORE_CBUF_ROW(row) \
vst1q_f16(c + row*ldc, c##row##0); \
vst1q_f16(c + row*ldc + 8, c##row##1); \
vst1q_f16(c + row*ldc + 16, c##row##2)
_FX_STORE_CBUF_ROW(0);
_FX_STORE_CBUF_ROW(1);
_FX_STORE_CBUF_ROW(2);
_FX_STORE_CBUF_ROW(3);
_FX_STORE_CBUF_ROW(4);
_FX_STORE_CBUF_ROW(5);
_FX_STORE_CBUF_ROW(6);
_FX_STORE_CBUF_ROW(7);
}
void convBlockMR1_F16(int np, const char* _a, const char* _b, float *c, const float _bias, bool init_c,
const float minval, const float maxval, bool ifMinMaxAct, const int width, const int convNR_FP16)
{
CV_Assert(convNR_FP16 == 24); // CONV_NR_FP16 = 24
const __fp16* a = (const __fp16*)_a;
const __fp16* b = (const __fp16*)_b;
const __fp16 bias = (__fp16)_bias;
float16x8_t c0 = vdupq_n_f16(bias), c1 = c0, c2 = c0;
if (width > 16)
{
for (int p = 0; p < np; p++, a++, b += convNR_FP16)
{
float16x8_t a0 = vdupq_n_f16(a[0]);
float16x8_t b0 = vld1q_f16(b), b1 = vld1q_f16(b + 8), b2 = vld1q_f16(b + 16);
c0 = vfmaq_f16(c0, a0, b0);
c1 = vfmaq_f16(c1, a0, b1);
c2 = vfmaq_f16(c2, a0, b2);
}
}
else if (width > 8)
{
for (int p = 0; p < np; p++, a++, b += convNR_FP16)
{
float16x8_t a0= vdupq_n_f16(a[0]);
float16x8_t b0 = vld1q_f16(b), b1 = vld1q_f16(b + 8);
c0 = vfmaq_f16(c0, a0, b0);
c1 = vfmaq_f16(c1, a0, b1);
}
}
else
{
for (int p = 0; p < np; p++, a++, b += convNR_FP16)
{
float16x8_t a0= vdupq_n_f16(a[0]);
float16x8_t b0 = vld1q_f16(b);
c0 = vfmaq_f16(c0, a0, b0);
}
}
// convert FP 16 to FP 32.
float32x4_t c00 = vcvt_f32_f16(vget_low_f16(c0));
float32x4_t c01 = vcvt_f32_f16(vget_high_f16(c0));
float32x4_t c10 = vcvt_f32_f16(vget_low_f16(c1));
float32x4_t c11 = vcvt_f32_f16(vget_high_f16(c1));
float32x4_t c20 = vcvt_f32_f16(vget_low_f16(c2));
float32x4_t c21 = vcvt_f32_f16(vget_high_f16(c2));
if (init_c)
{
c00 += vld1q_f32(c);
c01 += vld1q_f32(c + 4);
c10 += vld1q_f32(c + 8);
c11 += vld1q_f32(c + 12);
c20 += vld1q_f32(c + 16);
c21 += vld1q_f32(c + 20);
}
if (ifMinMaxAct)
{
float32x4_t v_minval = vdupq_n_f32(minval), v_maxval = vdupq_n_f32(maxval);
c00 = vminq_f32(vmaxq_f32(c00, v_minval), v_maxval);
c01 = vminq_f32(vmaxq_f32(c01, v_minval), v_maxval);
c10 = vminq_f32(vmaxq_f32(c10, v_minval), v_maxval);
c11 = vminq_f32(vmaxq_f32(c11, v_minval), v_maxval);
c20 = vminq_f32(vmaxq_f32(c20, v_minval), v_maxval);
c21 = vminq_f32(vmaxq_f32(c21, v_minval), v_maxval);
}
vst1q_f32(c, c00);
vst1q_f32(c + 4, c01);
vst1q_f32(c + 8, c10);
vst1q_f32(c + 12, c11);
vst1q_f32(c + 16, c20);
vst1q_f32(c + 20, c21);
}
#endif
#endif // CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY
CV_CPU_OPTIMIZATION_NAMESPACE_END
}} // namespace cv::dnn