Download modules/dnn/src/layers/cpu_kernels/conv_block.simd.hpp from tgollogly/test: direct link, hf CLI and curl.
- Browser
- Download file 27 kB
-
https://huggingface.co/tgollogly/test/resolve/main/modules/dnn/src/layers/cpu_kernels/conv_block.simd.hpp
- Command line
-
hf download hf://tgollogly/test/modules/dnn/src/layers/cpu_kernels/conv_block.simd.hpp
-
curl -L -o conv_block.simd.hpp https://huggingface.co/tgollogly/test/resolve/main/modules/dnn/src/layers/cpu_kernels/conv_block.simd.hpp
27 kB
| // This file is part of OpenCV project. | |
| // It is subject to the license terms in the LICENSE file found in the top-level directory | |
| // of this distribution and at http://opencv.org/license.html. | |
| namespace cv { | |
| namespace dnn { | |
| CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN | |
| void convBlock_F32(int np, const float* a, const float* b, float* c, int ldc, bool init_c, int width, const int convMR, const int convNR); | |
| // FP 16 branch. | |
| void convBlock_F16(int np, const char * _a, const char * _b, char * _c, int ldc, bool init_c, int width, | |
| const int convMR_fp16, const int convNR_fp16); | |
| void convBlockMR1_F16(int np, const char* _a, const char* _b, float *c, const float _bias, bool init_c, | |
| const float minval, const float maxval, bool ifMinMaxAct, const int width, const int convNR_FP16); | |
| void convBlock_F32(int np, const float* a, const float* b, float* c, int ldc, bool init_c, int width, const int convMR, const int convNR) | |
| { | |
| CV_Assert(convMR == 4 && convNR == 24); | |
| __m256 c00 = _mm256_set1_ps(0.f), c01 = c00, c02 = c00; | |
| __m256 c10 = c00, c11 = c00, c12 = c00; | |
| __m256 c20 = c00, c21 = c00, c22 = c00; | |
| __m256 c30 = c00, c31 = c00, c32 = c00; | |
| __m256 a0 = _mm256_setzero_ps(), a1 = _mm256_setzero_ps(); | |
| __m256 b0 = _mm256_setzero_ps(), b1 = _mm256_setzero_ps(), b2 = _mm256_setzero_ps(); | |
| if (width > 16) | |
| { | |
| for (int p = 0; p < np; p++, a += convMR, b += convNR) | |
| { | |
| a0 = _mm256_set1_ps(a[0]), a1 = _mm256_set1_ps(a[1]); | |
| b0 = _mm256_load_ps(b), b1 = _mm256_load_ps(b + 8), b2 = _mm256_load_ps(b + 16); | |
| c00 = _mm256_fmadd_ps(b0, a0, c00); | |
| c01 = _mm256_fmadd_ps(b1, a0, c01); | |
| c02 = _mm256_fmadd_ps(b2, a0, c02); | |
| c10 = _mm256_fmadd_ps(b0, a1, c10); | |
| c11 = _mm256_fmadd_ps(b1, a1, c11); | |
| c12 = _mm256_fmadd_ps(b2, a1, c12); | |
| a0 = _mm256_set1_ps(a[2]), a1 = _mm256_set1_ps(a[3]); | |
| c20 = _mm256_fmadd_ps(b0, a0, c20); | |
| c21 = _mm256_fmadd_ps(b1, a0, c21); | |
| c22 = _mm256_fmadd_ps(b2, a0, c22); | |
| c30 = _mm256_fmadd_ps(b0, a1, c30); | |
| c31 = _mm256_fmadd_ps(b1, a1, c31); | |
| c32 = _mm256_fmadd_ps(b2, a1, c32); | |
| } | |
| } | |
| else if (width > 8) | |
| { | |
| for (int p = 0; p < np; p++, a += convMR, b += convNR) | |
| { | |
| a0 = _mm256_set1_ps(a[0]), a1 = _mm256_set1_ps(a[1]); | |
| b0 = _mm256_load_ps(b), b1 = _mm256_load_ps(b + 8); | |
| c00 = _mm256_fmadd_ps(b0, a0, c00); | |
| c01 = _mm256_fmadd_ps(b1, a0, c01); | |
| c10 = _mm256_fmadd_ps(b0, a1, c10); | |
| c11 = _mm256_fmadd_ps(b1, a1, c11); | |
| a0 = _mm256_set1_ps(a[2]), a1 = _mm256_set1_ps(a[3]); | |
| c20 = _mm256_fmadd_ps(b0, a0, c20); | |
| c21 = _mm256_fmadd_ps(b1, a0, c21); | |
| c30 = _mm256_fmadd_ps(b0, a1, c30); | |
| c31 = _mm256_fmadd_ps(b1, a1, c31); | |
| } | |
| } | |
| else | |
| { | |
| for (int p = 0; p < np; p++, a += convMR, b += convNR) | |
| { | |
| a0 = _mm256_set1_ps(a[0]), a1 = _mm256_set1_ps(a[1]); | |
| b0 = _mm256_load_ps(b); | |
| c00 = _mm256_fmadd_ps(b0, a0, c00); | |
| c10 = _mm256_fmadd_ps(b0, a1, c10); | |
| a0 = _mm256_set1_ps(a[2]), a1 = _mm256_set1_ps(a[3]); | |
| c20 = _mm256_fmadd_ps(b0, a0, c20); | |
| c30 = _mm256_fmadd_ps(b0, a1, c30); | |
| } | |
| } | |
| if (!init_c) | |
| { | |
| c00 = _mm256_add_ps(c00, _mm256_load_ps(c)); | |
| c01 = _mm256_add_ps(c01, _mm256_load_ps(c + 8)); | |
| c02 = _mm256_add_ps(c02, _mm256_load_ps(c + 16)); | |
| c10 = _mm256_add_ps(c10, _mm256_load_ps(c + ldc)); | |
| c11 = _mm256_add_ps(c11, _mm256_load_ps(c + ldc + 8)); | |
| c12 = _mm256_add_ps(c12, _mm256_load_ps(c + ldc + 16)); | |
| c20 = _mm256_add_ps(c20, _mm256_load_ps(c + ldc*2)); | |
| c21 = _mm256_add_ps(c21, _mm256_load_ps(c + ldc*2 + 8)); | |
| c22 = _mm256_add_ps(c22, _mm256_load_ps(c + ldc*2 + 16)); | |
| c30 = _mm256_add_ps(c30, _mm256_load_ps(c + ldc*3)); | |
| c31 = _mm256_add_ps(c31, _mm256_load_ps(c + ldc*3 + 8)); | |
| c32 = _mm256_add_ps(c32, _mm256_load_ps(c + ldc*3 + 16)); | |
| } | |
| _mm256_storeu_ps(c, c00), _mm256_storeu_ps(c+8, c01), _mm256_storeu_ps(c+16, c02); | |
| _mm256_storeu_ps(c + ldc, c10), _mm256_storeu_ps(c + ldc + 8, c11), _mm256_storeu_ps(c + ldc + 16, c12); | |
| _mm256_storeu_ps(c + ldc*2, c20), _mm256_storeu_ps(c + ldc*2 + 8, c21), _mm256_storeu_ps(c + ldc*2 + 16, c22); | |
| _mm256_storeu_ps(c + ldc*3, c30), _mm256_storeu_ps(c + ldc*3 + 8, c31), _mm256_storeu_ps(c + ldc*3 + 16, c32); | |
| _mm256_zeroupper(); | |
| } | |
| void convBlock_F32(int np, const float* a, const float* b, float* c, int ldc, bool init_c, int width, const int convMR, const int convNR) | |
| { | |
| if (convMR == 4 && convNR == 28) // AARCH64 | |
| { | |
| float32x4_t c00 = vdupq_n_f32(0.f), c01 = c00, c02 = c00, c03 = c00, c04 = c00, c05 = c00, c06 = c00; | |
| float32x4_t c10 = vdupq_n_f32(0.f), c11 = c10, c12 = c10, c13 = c10, c14 = c10, c15 = c10, c16 = c10; | |
| float32x4_t c20 = vdupq_n_f32(0.f), c21 = c20, c22 = c20, c23 = c20, c24 = c20, c25 = c20, c26 = c20; | |
| float32x4_t c30 = vdupq_n_f32(0.f), c31 = c30, c32 = c30, c33 = c30, c34 = c30, c35 = c30, c36 = c30; | |
| if (width > 16) | |
| { | |
| for( int p = 0; p < np; p++, a += convMR, b += convNR ) | |
| { | |
| float32x4_t a0 = vld1q_f32(a), b0, b1, b2; | |
| b0 = vld1q_f32(b); b1 = vld1q_f32(b + 4); b2 = vld1q_f32(b + 8); | |
| c00 = vfmaq_laneq_f32(c00, b0, a0, 0); | |
| c01 = vfmaq_laneq_f32(c01, b1, a0, 0); | |
| c02 = vfmaq_laneq_f32(c02, b2, a0, 0); | |
| c10 = vfmaq_laneq_f32(c10, b0, a0, 1); | |
| c11 = vfmaq_laneq_f32(c11, b1, a0, 1); | |
| c12 = vfmaq_laneq_f32(c12, b2, a0, 1); | |
| c20 = vfmaq_laneq_f32(c20, b0, a0, 2); | |
| c21 = vfmaq_laneq_f32(c21, b1, a0, 2); | |
| c22 = vfmaq_laneq_f32(c22, b2, a0, 2); | |
| c30 = vfmaq_laneq_f32(c30, b0, a0, 3); | |
| c31 = vfmaq_laneq_f32(c31, b1, a0, 3); | |
| c32 = vfmaq_laneq_f32(c32, b2, a0, 3); | |
| b0 = vld1q_f32(b + 12); b1 = vld1q_f32(b + 16); b2 = vld1q_f32(b + 20); | |
| c03 = vfmaq_laneq_f32(c03, b0, a0, 0); | |
| c04 = vfmaq_laneq_f32(c04, b1, a0, 0); | |
| c05 = vfmaq_laneq_f32(c05, b2, a0, 0); | |
| c13 = vfmaq_laneq_f32(c13, b0, a0, 1); | |
| c14 = vfmaq_laneq_f32(c14, b1, a0, 1); | |
| c15 = vfmaq_laneq_f32(c15, b2, a0, 1); | |
| c23 = vfmaq_laneq_f32(c23, b0, a0, 2); | |
| c24 = vfmaq_laneq_f32(c24, b1, a0, 2); | |
| c25 = vfmaq_laneq_f32(c25, b2, a0, 2); | |
| c33 = vfmaq_laneq_f32(c33, b0, a0, 3); | |
| c34 = vfmaq_laneq_f32(c34, b1, a0, 3); | |
| c35 = vfmaq_laneq_f32(c35, b2, a0, 3); | |
| b0 = vld1q_f32(b + 24); | |
| c06 = vfmaq_laneq_f32(c06, b0, a0, 0); | |
| c16 = vfmaq_laneq_f32(c16, b0, a0, 1); | |
| c26 = vfmaq_laneq_f32(c26, b0, a0, 2); | |
| c36 = vfmaq_laneq_f32(c36, b0, a0, 3); | |
| } | |
| } | |
| else if (width > 8) | |
| { | |
| for( int p = 0; p < np; p++, a += convMR, b += convNR ) | |
| { | |
| float32x4_t a0 = vld1q_f32(a), b0, b1, b2; | |
| b0 = vld1q_f32(b); b1 = vld1q_f32(b + 4); b2 = vld1q_f32(b + 8); | |
| c00 = vfmaq_laneq_f32(c00, b0, a0, 0); | |
| c01 = vfmaq_laneq_f32(c01, b1, a0, 0); | |
| c02 = vfmaq_laneq_f32(c02, b2, a0, 0); | |
| c10 = vfmaq_laneq_f32(c10, b0, a0, 1); | |
| c11 = vfmaq_laneq_f32(c11, b1, a0, 1); | |
| c12 = vfmaq_laneq_f32(c12, b2, a0, 1); | |
| c20 = vfmaq_laneq_f32(c20, b0, a0, 2); | |
| c21 = vfmaq_laneq_f32(c21, b1, a0, 2); | |
| c22 = vfmaq_laneq_f32(c22, b2, a0, 2); | |
| c30 = vfmaq_laneq_f32(c30, b0, a0, 3); | |
| c31 = vfmaq_laneq_f32(c31, b1, a0, 3); | |
| c32 = vfmaq_laneq_f32(c32, b2, a0, 3); | |
| b0 = vld1q_f32(b + 12); | |
| c03 = vfmaq_laneq_f32(c03, b0, a0, 0); | |
| c13 = vfmaq_laneq_f32(c13, b0, a0, 1); | |
| c23 = vfmaq_laneq_f32(c23, b0, a0, 2); | |
| c33 = vfmaq_laneq_f32(c33, b0, a0, 3); | |
| } | |
| } | |
| else if (width > 4) | |
| { | |
| for( int p = 0; p < np; p++, a += convMR, b += convNR ) | |
| { | |
| float32x4_t a0 = vld1q_f32(a), b0, b1; | |
| b0 = vld1q_f32(b); b1 = vld1q_f32(b + 4); | |
| c00 = vfmaq_laneq_f32(c00, b0, a0, 0); | |
| c01 = vfmaq_laneq_f32(c01, b1, a0, 0); | |
| c10 = vfmaq_laneq_f32(c10, b0, a0, 1); | |
| c11 = vfmaq_laneq_f32(c11, b1, a0, 1); | |
| c20 = vfmaq_laneq_f32(c20, b0, a0, 2); | |
| c21 = vfmaq_laneq_f32(c21, b1, a0, 2); | |
| c30 = vfmaq_laneq_f32(c30, b0, a0, 3); | |
| c31 = vfmaq_laneq_f32(c31, b1, a0, 3); | |
| } | |
| } | |
| else | |
| { | |
| for( int p = 0; p < np; p++, a += convMR, b += convNR ) | |
| { | |
| float32x4_t a0 = vld1q_f32(a), b0; | |
| b0 = vld1q_f32(b); | |
| c00 = vfmaq_laneq_f32(c00, b0, a0, 0); | |
| c10 = vfmaq_laneq_f32(c10, b0, a0, 1); | |
| c20 = vfmaq_laneq_f32(c20, b0, a0, 2); | |
| c30 = vfmaq_laneq_f32(c30, b0, a0, 3); | |
| } | |
| } | |
| if (!init_c) | |
| { | |
| c00 = vaddq_f32(c00, vld1q_f32(c)); | |
| c01 = vaddq_f32(c01, vld1q_f32(c + 4)); | |
| c02 = vaddq_f32(c02, vld1q_f32(c + 8)); | |
| c03 = vaddq_f32(c03, vld1q_f32(c + 12)); | |
| c04 = vaddq_f32(c04, vld1q_f32(c + 16)); | |
| c05 = vaddq_f32(c05, vld1q_f32(c + 20)); | |
| c06 = vaddq_f32(c06, vld1q_f32(c + 24)); | |
| c10 = vaddq_f32(c10, vld1q_f32(c + ldc)); | |
| c11 = vaddq_f32(c11, vld1q_f32(c + ldc + 4)); | |
| c12 = vaddq_f32(c12, vld1q_f32(c + ldc + 8)); | |
| c13 = vaddq_f32(c13, vld1q_f32(c + ldc + 12)); | |
| c14 = vaddq_f32(c14, vld1q_f32(c + ldc + 16)); | |
| c15 = vaddq_f32(c15, vld1q_f32(c + ldc + 20)); | |
| c16 = vaddq_f32(c16, vld1q_f32(c + ldc + 24)); | |
| c20 = vaddq_f32(c20, vld1q_f32(c + ldc*2)); | |
| c21 = vaddq_f32(c21, vld1q_f32(c + ldc*2 + 4)); | |
| c22 = vaddq_f32(c22, vld1q_f32(c + ldc*2 + 8)); | |
| c23 = vaddq_f32(c23, vld1q_f32(c + ldc*2 + 12)); | |
| c24 = vaddq_f32(c24, vld1q_f32(c + ldc*2 + 16)); | |
| c25 = vaddq_f32(c25, vld1q_f32(c + ldc*2 + 20)); | |
| c26 = vaddq_f32(c26, vld1q_f32(c + ldc*2 + 24)); | |
| c30 = vaddq_f32(c30, vld1q_f32(c + ldc*3)); | |
| c31 = vaddq_f32(c31, vld1q_f32(c + ldc*3 + 4)); | |
| c32 = vaddq_f32(c32, vld1q_f32(c + ldc*3 + 8)); | |
| c33 = vaddq_f32(c33, vld1q_f32(c + ldc*3 + 12)); | |
| c34 = vaddq_f32(c34, vld1q_f32(c + ldc*3 + 16)); | |
| c35 = vaddq_f32(c35, vld1q_f32(c + ldc*3 + 20)); | |
| c36 = vaddq_f32(c36, vld1q_f32(c + ldc*3 + 24)); | |
| } | |
| vst1q_f32(c, c00); vst1q_f32(c+4, c01); | |
| vst1q_f32(c+8, c02); vst1q_f32(c+12, c03); | |
| vst1q_f32(c+16, c04); vst1q_f32(c+20, c05); | |
| vst1q_f32(c+24, c06); | |
| vst1q_f32(c+ldc, c10); vst1q_f32(c+ldc+4, c11); | |
| vst1q_f32(c+ldc+8, c12); vst1q_f32(c+ldc+12, c13); | |
| vst1q_f32(c+ldc+16, c14); vst1q_f32(c+ldc+20, c15); | |
| vst1q_f32(c+ldc+24, c16); | |
| vst1q_f32(c+ldc*2, c20); vst1q_f32(c+ldc*2+4, c21); | |
| vst1q_f32(c+ldc*2+8, c22); vst1q_f32(c+ldc*2+12, c23); | |
| vst1q_f32(c+ldc*2+16, c24); vst1q_f32(c+ldc*2+20, c25); | |
| vst1q_f32(c+ldc*2+24, c26); | |
| vst1q_f32(c+ldc*3, c30); vst1q_f32(c+ldc*3+4, c31); | |
| vst1q_f32(c+ldc*3+8, c32); vst1q_f32(c+ldc*3+12, c33); | |
| vst1q_f32(c+ldc*3+16, c34); vst1q_f32(c+ldc*3+20, c35); | |
| vst1q_f32(c+ldc*3+24, c36); | |
| } | |
| else | |
| if (convMR == 4 && convNR == 12) // ARMv7 | |
| { | |
| float32x4_t c0 = vdupq_n_f32(0.f), c1 = c0, c2 = c0; | |
| float32x4_t c3 = vdupq_n_f32(0.f), c4 = c3, c5 = c3; | |
| float32x4_t c6 = vdupq_n_f32(0.f), c7 = c6, c8 = c6; | |
| float32x4_t c9 = vdupq_n_f32(0.f), c10 = c9, c11 = c9; | |
| float32x2_t a0 = vdup_n_f32(0.0f), a1 = a0; | |
| float32x4_t b0 = vdupq_n_f32(0.0f), b1 = vdupq_n_f32(0.0f), b2 = vdupq_n_f32(0.0f); | |
| if (width > 8) | |
| { | |
| for (int p = 0; p < np; p++, a += convMR, b += convNR) | |
| { | |
| a0 = vld1_f32(a), a1 = vld1_f32(a+2); | |
| b0 = vld1q_f32(b), b1 = vld1q_f32(b + 4), b2 = vld1q_f32(b + 8); | |
| c0 = vmlaq_lane_f32(c0, b0, a0, 0); | |
| c1 = vmlaq_lane_f32(c1, b1, a0, 0); | |
| c2 = vmlaq_lane_f32(c2, b2, a0, 0); | |
| c3 = vmlaq_lane_f32(c3, b0, a0, 1); | |
| c4 = vmlaq_lane_f32(c4, b1, a0, 1); | |
| c5 = vmlaq_lane_f32(c5, b2, a0, 1); | |
| c6 = vmlaq_lane_f32(c6, b0, a1, 0); | |
| c7 = vmlaq_lane_f32(c7, b1, a1, 0); | |
| c8 = vmlaq_lane_f32(c8, b2, a1, 0); | |
| c9 = vmlaq_lane_f32(c9 , b0, a1, 1); | |
| c10 = vmlaq_lane_f32(c10, b1, a1, 1); | |
| c11 = vmlaq_lane_f32(c11, b2, a1, 1); | |
| } | |
| } | |
| else if (width > 4) | |
| { | |
| for (int p = 0; p < np; p++, a += convMR, b += convNR) | |
| { | |
| a0 = vld1_f32(a), a1 = vld1_f32(a+2); | |
| b0 = vld1q_f32(b), b1 = vld1q_f32(b + 4); | |
| c0 = vmlaq_lane_f32(c0, b0, a0, 0); | |
| c1 = vmlaq_lane_f32(c1, b1, a0, 0); | |
| c3 = vmlaq_lane_f32(c3, b0, a0, 1); | |
| c4 = vmlaq_lane_f32(c4, b1, a0, 1); | |
| c6 = vmlaq_lane_f32(c6, b0, a1, 0); | |
| c7 = vmlaq_lane_f32(c7, b1, a1, 0); | |
| c9 = vmlaq_lane_f32(c9 , b0, a1, 1); | |
| c10 = vmlaq_lane_f32(c10, b1, a1, 1); | |
| } | |
| } | |
| else | |
| { | |
| for (int p = 0; p < np; p++, a += convMR, b += convNR) | |
| { | |
| a0 = vld1_f32(a), a1 = vld1_f32(a+2); | |
| b0 = vld1q_f32(b); | |
| c0 = vmlaq_lane_f32(c0, b0, a0, 0); | |
| c3 = vmlaq_lane_f32(c3, b0, a0, 1); | |
| c6 = vmlaq_lane_f32(c6, b0, a1, 0); | |
| c9 = vmlaq_lane_f32(c9 , b0, a1, 1); | |
| } | |
| } | |
| if (!init_c) | |
| { | |
| c0 = vaddq_f32(c0, vld1q_f32(c)); | |
| c1 = vaddq_f32(c1, vld1q_f32(c + 4)); | |
| c2 = vaddq_f32(c2, vld1q_f32(c + 8)); | |
| c3 = vaddq_f32(c3, vld1q_f32(c + ldc)); | |
| c4 = vaddq_f32(c4, vld1q_f32(c + ldc + 4)); | |
| c5 = vaddq_f32(c5, vld1q_f32(c + ldc + 8)); | |
| c6 = vaddq_f32(c6, vld1q_f32(c + ldc * 2)); | |
| c7 = vaddq_f32(c7, vld1q_f32(c + ldc * 2 + 4)); | |
| c8 = vaddq_f32(c8, vld1q_f32(c + ldc * 2 + 8)); | |
| c9 = vaddq_f32(c9 , vld1q_f32(c + ldc * 3)); | |
| c10 = vaddq_f32(c10, vld1q_f32(c + ldc * 3 + 4)); | |
| c11 = vaddq_f32(c11, vld1q_f32(c + ldc * 3 + 8)); | |
| } | |
| vst1q_f32(c, c0), vst1q_f32(c+4, c1), vst1q_f32(c+8, c2); | |
| vst1q_f32(c + ldc, c3), vst1q_f32(c + ldc + 4, c4), vst1q_f32(c + ldc + 8, c5); | |
| vst1q_f32(c + ldc*2, c6), vst1q_f32(c + ldc*2 + 4, c7), vst1q_f32(c + ldc*2 + 8, c8); | |
| vst1q_f32(c + ldc*3, c9), vst1q_f32(c + ldc*3 + 4, c10), vst1q_f32(c + ldc*3 + 8, c11); | |
| } | |
| else | |
| CV_Error(Error::StsNotImplemented, "Unsupported convMR and/or convNR in opt_NEON::convBlock"); | |
| } | |
| void convBlockMR1_F32(int np, const float * a, const float * b, float *c, const float bias, bool init_c, | |
| const float minval, const float maxval, bool ifMinMaxAct, const int width, const int convNR) | |
| { | |
| CV_Assert(convNR == 28); | |
| float32x4_t c0 = vdupq_n_f32(bias), c1 = c0, c2 = c0; | |
| float32x4_t c3 = c0, c4 = c0, c5 = c0, c6 = c0; | |
| if (width > 16) | |
| { | |
| for (int p = 0; p < np; p++, a++, b += convNR) | |
| { | |
| float32x4_t b0 = vld1q_f32(b), b1 = vld1q_f32(b + 4), b2 = vld1q_f32(b + 8); | |
| float32x4_t b3 = vld1q_f32(b + 12), b4 = vld1q_f32(b + 16), b5 = vld1q_f32(b + 20); | |
| float32x4_t b6 = vld1q_f32(b + 24); | |
| c0 = vmlaq_n_f32(c0, b0, a[0]); | |
| c1 = vmlaq_n_f32(c1, b1, a[0]); | |
| c2 = vmlaq_n_f32(c2, b2, a[0]); | |
| c3 = vmlaq_n_f32(c3, b3, a[0]); | |
| c4 = vmlaq_n_f32(c4, b4, a[0]); | |
| c5 = vmlaq_n_f32(c5, b5, a[0]); | |
| c6 = vmlaq_n_f32(c6, b6, a[0]); | |
| } | |
| } | |
| else if (width > 8) | |
| { | |
| for (int p = 0; p < np; p++, a++, b += convNR) | |
| { | |
| float32x4_t b0 = vld1q_f32(b), b1 = vld1q_f32(b + 4), b2 = vld1q_f32(b + 8); | |
| float32x4_t b3 = vld1q_f32(b + 12); | |
| c0 = vmlaq_n_f32(c0, b0, a[0]); | |
| c1 = vmlaq_n_f32(c1, b1, a[0]); | |
| c2 = vmlaq_n_f32(c2, b2, a[0]); | |
| c3 = vmlaq_n_f32(c3, b3, a[0]); | |
| } | |
| } | |
| else if (width > 4) | |
| { | |
| for (int p = 0; p < np; p++, a++, b += convNR) | |
| { | |
| float32x4_t b0 = vld1q_f32(b), b1 = vld1q_f32(b + 4); | |
| c0 = vmlaq_n_f32(c0, b0, a[0]); | |
| c1 = vmlaq_n_f32(c1, b1, a[0]); | |
| } | |
| } | |
| else | |
| { | |
| for (int p = 0; p < np; p++, a++, b += convNR) | |
| { | |
| float32x4_t b0 = vld1q_f32(b); | |
| c0 = vmlaq_n_f32(c0, b0, a[0]); | |
| } | |
| } | |
| if (init_c) | |
| { | |
| c0 = vaddq_f32(c0, vld1q_f32(c)); | |
| c1 = vaddq_f32(c1, vld1q_f32(c + 4)); | |
| c2 = vaddq_f32(c2, vld1q_f32(c + 8)); | |
| c3 = vaddq_f32(c3, vld1q_f32(c + 12)); | |
| c4 = vaddq_f32(c4, vld1q_f32(c + 16)); | |
| c5 = vaddq_f32(c5, vld1q_f32(c + 20)); | |
| c6 = vaddq_f32(c6, vld1q_f32(c + 24)); | |
| } | |
| if (ifMinMaxAct) | |
| { | |
| float32x4_t v_minval = vdupq_n_f32(minval), v_maxval = vdupq_n_f32(maxval); | |
| c0 = vminq_f32(vmaxq_f32(c0, v_minval), v_maxval); | |
| c1 = vminq_f32(vmaxq_f32(c1, v_minval), v_maxval); | |
| c2 = vminq_f32(vmaxq_f32(c2, v_minval), v_maxval); | |
| c3 = vminq_f32(vmaxq_f32(c3, v_minval), v_maxval); | |
| c4 = vminq_f32(vmaxq_f32(c4, v_minval), v_maxval); | |
| c5 = vminq_f32(vmaxq_f32(c5, v_minval), v_maxval); | |
| c6 = vminq_f32(vmaxq_f32(c6, v_minval), v_maxval); | |
| } | |
| vst1q_f32(c, c0); | |
| vst1q_f32(c + 4, c1); | |
| vst1q_f32(c + 8, c2); | |
| vst1q_f32(c + 12, c3); | |
| vst1q_f32(c + 16, c4); | |
| vst1q_f32(c + 20, c5); | |
| vst1q_f32(c + 24, c6); | |
| } | |
| void convBlock_F16(int np, const char * _a, const char * _b, char * _c, int ldc, bool init_c, int width, | |
| const int convMR_fp16, const int convNR_fp16) | |
| { | |
| const __fp16* a = (const __fp16*)_a; | |
| const __fp16* b = (const __fp16*)_b; | |
| __fp16* c = (__fp16*)_c; | |
| CV_Assert(convMR_fp16 == 8 && convNR_fp16 == 24); | |
| float16x8_t c00 = vdupq_n_f16(0), c01 = c00, c02 = c00; | |
| float16x8_t c10 = c00, c11 = c00, c12 = c00; | |
| float16x8_t c20 = c00, c21 = c00, c22 = c00; | |
| float16x8_t c30 = c00, c31 = c00, c32 = c00; | |
| float16x8_t c40 = c00, c41 = c00, c42 = c00; | |
| float16x8_t c50 = c00, c51 = c00, c52 = c00; | |
| float16x8_t c60 = c00, c61 = c00, c62 = c00; | |
| float16x8_t c70 = c00, c71 = c00, c72 = c00; | |
| float16x8_t b0 = c00, b1 = c00, b2 = c00; | |
| if (width > 16) | |
| { | |
| for (int p = 0; p < np; p++, a += convMR_fp16, b += convNR_fp16) | |
| { | |
| float16x4_t a0 = vld1_f16(a), a1 = vld1_f16(a + 4); | |
| b0 = vld1q_f16(b), b1 = vld1q_f16(b + 8), b2 = vld1q_f16(b + 16); | |
| c00 = vfmaq_lane_f16(c00, b0, a0, 0); | |
| c01 = vfmaq_lane_f16(c01, b1, a0, 0); | |
| c02 = vfmaq_lane_f16(c02, b2, a0, 0); | |
| c10 = vfmaq_lane_f16(c10, b0, a0, 1); | |
| c11 = vfmaq_lane_f16(c11, b1, a0, 1); | |
| c12 = vfmaq_lane_f16(c12, b2, a0, 1); | |
| c20 = vfmaq_lane_f16(c20, b0, a0, 2); | |
| c21 = vfmaq_lane_f16(c21, b1, a0, 2); | |
| c22 = vfmaq_lane_f16(c22, b2, a0, 2); | |
| c30 = vfmaq_lane_f16(c30, b0, a0, 3); | |
| c31 = vfmaq_lane_f16(c31, b1, a0, 3); | |
| c32 = vfmaq_lane_f16(c32, b2, a0, 3); | |
| c40 = vfmaq_lane_f16(c40, b0, a1, 0); | |
| c41 = vfmaq_lane_f16(c41, b1, a1, 0); | |
| c42 = vfmaq_lane_f16(c42, b2, a1, 0); | |
| c50 = vfmaq_lane_f16(c50, b0, a1, 1); | |
| c51 = vfmaq_lane_f16(c51, b1, a1, 1); | |
| c52 = vfmaq_lane_f16(c52, b2, a1, 1); | |
| c60 = vfmaq_lane_f16(c60, b0, a1, 2); | |
| c61 = vfmaq_lane_f16(c61, b1, a1, 2); | |
| c62 = vfmaq_lane_f16(c62, b2, a1, 2); | |
| c70 = vfmaq_lane_f16(c70, b0, a1, 3); | |
| c71 = vfmaq_lane_f16(c71, b1, a1, 3); | |
| c72 = vfmaq_lane_f16(c72, b2, a1, 3); | |
| } | |
| } | |
| else if (width > 8) | |
| { | |
| for( int p = 0; p < np; p++, a += convMR_fp16, b += convNR_fp16) | |
| { | |
| float16x4_t a0 = vld1_f16(a), a1 = vld1_f16(a + 4); | |
| float16x8_t b0 = vld1q_f16(b), b1 = vld1q_f16(b + 8); | |
| c00 = vfmaq_lane_f16(c00, b0, a0, 0); | |
| c01 = vfmaq_lane_f16(c01, b1, a0, 0); | |
| c10 = vfmaq_lane_f16(c10, b0, a0, 1); | |
| c11 = vfmaq_lane_f16(c11, b1, a0, 1); | |
| c20 = vfmaq_lane_f16(c20, b0, a0, 2); | |
| c21 = vfmaq_lane_f16(c21, b1, a0, 2); | |
| c30 = vfmaq_lane_f16(c30, b0, a0, 3); | |
| c31 = vfmaq_lane_f16(c31, b1, a0, 3); | |
| c40 = vfmaq_lane_f16(c40, b0, a1, 0); | |
| c41 = vfmaq_lane_f16(c41, b1, a1, 0); | |
| c50 = vfmaq_lane_f16(c50, b0, a1, 1); | |
| c51 = vfmaq_lane_f16(c51, b1, a1, 1); | |
| c60 = vfmaq_lane_f16(c60, b0, a1, 2); | |
| c61 = vfmaq_lane_f16(c61, b1, a1, 2); | |
| c70 = vfmaq_lane_f16(c70, b0, a1, 3); | |
| c71 = vfmaq_lane_f16(c71, b1, a1, 3); | |
| } | |
| } | |
| else | |
| { | |
| for( int p = 0; p < np; p++, a += convMR_fp16, b += convNR_fp16) | |
| { | |
| float16x4_t a0 = vld1_f16(a), a1 = vld1_f16(a + 4); | |
| float16x8_t b0 = vld1q_f16(b); | |
| c00 = vfmaq_lane_f16(c00, b0, a0, 0); | |
| c10 = vfmaq_lane_f16(c10, b0, a0, 1); | |
| c20 = vfmaq_lane_f16(c20, b0, a0, 2); | |
| c30 = vfmaq_lane_f16(c30, b0, a0, 3); | |
| c40 = vfmaq_lane_f16(c40, b0, a1, 0); | |
| c50 = vfmaq_lane_f16(c50, b0, a1, 1); | |
| c60 = vfmaq_lane_f16(c60, b0, a1, 2); | |
| c70 = vfmaq_lane_f16(c70, b0, a1, 3); | |
| } | |
| } | |
| if (!init_c) | |
| { | |
| _FX_UPDATE_CBUF_ROW(0); | |
| _FX_UPDATE_CBUF_ROW(1); | |
| _FX_UPDATE_CBUF_ROW(2); | |
| _FX_UPDATE_CBUF_ROW(3); | |
| _FX_UPDATE_CBUF_ROW(4); | |
| _FX_UPDATE_CBUF_ROW(5); | |
| _FX_UPDATE_CBUF_ROW(6); | |
| _FX_UPDATE_CBUF_ROW(7); | |
| } | |
| _FX_STORE_CBUF_ROW(0); | |
| _FX_STORE_CBUF_ROW(1); | |
| _FX_STORE_CBUF_ROW(2); | |
| _FX_STORE_CBUF_ROW(3); | |
| _FX_STORE_CBUF_ROW(4); | |
| _FX_STORE_CBUF_ROW(5); | |
| _FX_STORE_CBUF_ROW(6); | |
| _FX_STORE_CBUF_ROW(7); | |
| } | |
| void convBlockMR1_F16(int np, const char* _a, const char* _b, float *c, const float _bias, bool init_c, | |
| const float minval, const float maxval, bool ifMinMaxAct, const int width, const int convNR_FP16) | |
| { | |
| CV_Assert(convNR_FP16 == 24); // CONV_NR_FP16 = 24 | |
| const __fp16* a = (const __fp16*)_a; | |
| const __fp16* b = (const __fp16*)_b; | |
| const __fp16 bias = (__fp16)_bias; | |
| float16x8_t c0 = vdupq_n_f16(bias), c1 = c0, c2 = c0; | |
| if (width > 16) | |
| { | |
| for (int p = 0; p < np; p++, a++, b += convNR_FP16) | |
| { | |
| float16x8_t a0 = vdupq_n_f16(a[0]); | |
| float16x8_t b0 = vld1q_f16(b), b1 = vld1q_f16(b + 8), b2 = vld1q_f16(b + 16); | |
| c0 = vfmaq_f16(c0, a0, b0); | |
| c1 = vfmaq_f16(c1, a0, b1); | |
| c2 = vfmaq_f16(c2, a0, b2); | |
| } | |
| } | |
| else if (width > 8) | |
| { | |
| for (int p = 0; p < np; p++, a++, b += convNR_FP16) | |
| { | |
| float16x8_t a0= vdupq_n_f16(a[0]); | |
| float16x8_t b0 = vld1q_f16(b), b1 = vld1q_f16(b + 8); | |
| c0 = vfmaq_f16(c0, a0, b0); | |
| c1 = vfmaq_f16(c1, a0, b1); | |
| } | |
| } | |
| else | |
| { | |
| for (int p = 0; p < np; p++, a++, b += convNR_FP16) | |
| { | |
| float16x8_t a0= vdupq_n_f16(a[0]); | |
| float16x8_t b0 = vld1q_f16(b); | |
| c0 = vfmaq_f16(c0, a0, b0); | |
| } | |
| } | |
| // convert FP 16 to FP 32. | |
| float32x4_t c00 = vcvt_f32_f16(vget_low_f16(c0)); | |
| float32x4_t c01 = vcvt_f32_f16(vget_high_f16(c0)); | |
| float32x4_t c10 = vcvt_f32_f16(vget_low_f16(c1)); | |
| float32x4_t c11 = vcvt_f32_f16(vget_high_f16(c1)); | |
| float32x4_t c20 = vcvt_f32_f16(vget_low_f16(c2)); | |
| float32x4_t c21 = vcvt_f32_f16(vget_high_f16(c2)); | |
| if (init_c) | |
| { | |
| c00 += vld1q_f32(c); | |
| c01 += vld1q_f32(c + 4); | |
| c10 += vld1q_f32(c + 8); | |
| c11 += vld1q_f32(c + 12); | |
| c20 += vld1q_f32(c + 16); | |
| c21 += vld1q_f32(c + 20); | |
| } | |
| if (ifMinMaxAct) | |
| { | |
| float32x4_t v_minval = vdupq_n_f32(minval), v_maxval = vdupq_n_f32(maxval); | |
| c00 = vminq_f32(vmaxq_f32(c00, v_minval), v_maxval); | |
| c01 = vminq_f32(vmaxq_f32(c01, v_minval), v_maxval); | |
| c10 = vminq_f32(vmaxq_f32(c10, v_minval), v_maxval); | |
| c11 = vminq_f32(vmaxq_f32(c11, v_minval), v_maxval); | |
| c20 = vminq_f32(vmaxq_f32(c20, v_minval), v_maxval); | |
| c21 = vminq_f32(vmaxq_f32(c21, v_minval), v_maxval); | |
| } | |
| vst1q_f32(c, c00); | |
| vst1q_f32(c + 4, c01); | |
| vst1q_f32(c + 8, c10); | |
| vst1q_f32(c + 12, c11); | |
| vst1q_f32(c + 16, c20); | |
| vst1q_f32(c + 20, c21); | |
| } | |
| CV_CPU_OPTIMIZATION_NAMESPACE_END | |
| }} // namespace cv::dnn | |