src/layer/x86/convolution_3x3_pack8.h

// Tencent is pleased to support the open source community by making ncnn available.
//
// Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved.
//
// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
// in compliance with the License. You may obtain a copy of the License at
//
// https://opensource.org/licenses/BSD-3-Clause
//
// Unless required by applicable law or agreed to in writing, software distributed
// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
// specific language governing permissions and limitations under the License.

static void conv3x3s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel, const Mat& _bias, const Option& opt)
{
    int inch = bottom_blob.c;

    int outw = top_blob.w;
    int outh = top_blob.h;
    int outch = top_blob.c;

    const float* bias = _bias;

    #pragma omp parallel for num_threads(opt.num_threads)
    for (int p = 0; p < outch; p++)
    {
        Mat out = top_blob.channel(p);

        __m256 _bias0 = bias ? _mm256_loadu_ps(bias + p * 8) : _mm256_setzero_ps();
        out.fill(_bias0);

        for (int q = 0; q < inch; q++)
        {
            float* outptr = out;

            const Mat img0 = bottom_blob.channel(q);

            const float* r0 = img0.row(0);
            const float* r1 = img0.row(1);
            const float* r2 = img0.row(2);

            const float* kptr = kernel.channel(p).row(q);

            int i = 0;
            for (; i < outh; i++)
            {
                int j = 0;
                for (; j + 1 < outw; j += 2)
                {
                    __m256 _sum00 = _mm256_loadu_ps(outptr);
                    __m256 _sum01 = _mm256_setzero_ps();
                    __m256 _sum10 = _mm256_loadu_ps(outptr + 8);
                    __m256 _sum11 = _mm256_setzero_ps();

                    __m256 _r000 = _mm256_broadcast_ss(r0 + 0);
                    __m256 _r001 = _mm256_broadcast_ss(r0 + 1);
                    __m256 _r002 = _mm256_broadcast_ss(r0 + 2);
                    __m256 _r003 = _mm256_broadcast_ss(r0 + 3);
                    __m256 _r004 = _mm256_broadcast_ss(r0 + 4);
                    __m256 _r005 = _mm256_broadcast_ss(r0 + 5);
                    __m256 _r006 = _mm256_broadcast_ss(r0 + 6);
                    __m256 _r007 = _mm256_broadcast_ss(r0 + 7);

                    __m256 _k00 = _mm256_loadu_ps(kptr);
                    __m256 _k01 = _mm256_loadu_ps(kptr + 8);
                    __m256 _k02 = _mm256_loadu_ps(kptr + 16);
                    __m256 _k03 = _mm256_loadu_ps(kptr + 24);
                    __m256 _k04 = _mm256_loadu_ps(kptr + 32);
                    __m256 _k05 = _mm256_loadu_ps(kptr + 40);
                    __m256 _k06 = _mm256_loadu_ps(kptr + 48);
                    __m256 _k07 = _mm256_loadu_ps(kptr + 56);

                    kptr += 64;

                    _sum00 = _mm256_comp_fmadd_ps(_r000, _k00, _sum00);
                    _sum01 = _mm256_comp_fmadd_ps(_r001, _k01, _sum01);
                    _sum00 = _mm256_comp_fmadd_ps(_r002, _k02, _sum00);
                    _sum01 = _mm256_comp_fmadd_ps(_r003, _k03, _sum01);
                    _sum00 = _mm256_comp_fmadd_ps(_r004, _k04, _sum00);
                    _sum01 = _mm256_comp_fmadd_ps(_r005, _k05, _sum01);
                    _sum00 = _mm256_comp_fmadd_ps(_r006, _k06, _sum00);
                    _sum01 = _mm256_comp_fmadd_ps(_r007, _k07, _sum01);

                    __m256 _r010 = _mm256_broadcast_ss(r0 + 8);
                    __m256 _r011 = _mm256_broadcast_ss(r0 + 9);
                    __m256 _r012 = _mm256_broadcast_ss(r0 + 10);
                    __m256 _r013 = _mm256_broadcast_ss(r0 + 11);
                    __m256 _r014 = _mm256_broadcast_ss(r0 + 12);
                    __m256 _r015 = _mm256_broadcast_ss(r0 + 13);
                    __m256 _r016 = _mm256_broadcast_ss(r0 + 14);
                    __m256 _r017 = _mm256_broadcast_ss(r0 + 15);

                    _sum10 = _mm256_comp_fmadd_ps(_r010, _k00, _sum10);
                    _sum11 = _mm256_comp_fmadd_ps(_r011, _k01, _sum11);
                    _sum10 = _mm256_comp_fmadd_ps(_r012, _k02, _sum10);
                    _sum11 = _mm256_comp_fmadd_ps(_r013, _k03, _sum11);
                    _sum10 = _mm256_comp_fmadd_ps(_r014, _k04, _sum10);
                    _sum11 = _mm256_comp_fmadd_ps(_r015, _k05, _sum11);
                    _sum10 = _mm256_comp_fmadd_ps(_r016, _k06, _sum10);
                    _sum11 = _mm256_comp_fmadd_ps(_r017, _k07, _sum11);

                    __m256 _k10 = _mm256_loadu_ps(kptr);
                    __m256 _k11 = _mm256_loadu_ps(kptr + 8);
                    __m256 _k12 = _mm256_loadu_ps(kptr + 16);
                    __m256 _k13 = _mm256_loadu_ps(kptr + 24);
                    __m256 _k14 = _mm256_loadu_ps(kptr + 32);
                    __m256 _k15 = _mm256_loadu_ps(kptr + 40);
                    __m256 _k16 = _mm256_loadu_ps(kptr + 48);
                    __m256 _k17 = _mm256_loadu_ps(kptr + 56);

                    kptr += 64;

                    _sum00 = _mm256_comp_fmadd_ps(_r010, _k10, _sum00);
                    _sum01 = _mm256_comp_fmadd_ps(_r011, _k11, _sum01);
                    _sum00 = _mm256_comp_fmadd_ps(_r012, _k12, _sum00);
                    _sum01 = _mm256_comp_fmadd_ps(_r013, _k13, _sum01);
                    _sum00 = _mm256_comp_fmadd_ps(_r014, _k14, _sum00);
                    _sum01 = _mm256_comp_fmadd_ps(_r015, _k15, _sum01);
                    _sum00 = _mm256_comp_fmadd_ps(_r016, _k16, _sum00);
                    _sum01 = _mm256_comp_fmadd_ps(_r017, _k17, _sum01);

                    __m256 _r020 = _mm256_broadcast_ss(r0 + 16);
                    __m256 _r021 = _mm256_broadcast_ss(r0 + 17);
                    __m256 _r022 = _mm256_broadcast_ss(r0 + 18);
                    __m256 _r023 = _mm256_broadcast_ss(r0 + 19);
                    __m256 _r024 = _mm256_broadcast_ss(r0 + 20);
                    __m256 _r025 = _mm256_broadcast_ss(r0 + 21);
                    __m256 _r026 = _mm256_broadcast_ss(r0 + 22);
                    __m256 _r027 = _mm256_broadcast_ss(r0 + 23);

                    _sum10 = _mm256_comp_fmadd_ps(_r020, _k10, _sum10);
                    _sum11 = _mm256_comp_fmadd_ps(_r021, _k11, _sum11);
                    _sum10 = _mm256_comp_fmadd_ps(_r022, _k12, _sum10);
                    _sum11 = _mm256_comp_fmadd_ps(_r023, _k13, _sum11);
                    _sum10 = _mm256_comp_fmadd_ps(_r024, _k14, _sum10);
                    _sum11 = _mm256_comp_fmadd_ps(_r025, _k15, _sum11);
                    _sum10 = _mm256_comp_fmadd_ps(_r026, _k16, _sum10);
                    _sum11 = _mm256_comp_fmadd_ps(_r027, _k17, _sum11);

                    __m256 _k20 = _mm256_loadu_ps(kptr);
                    __m256 _k21 = _mm256_loadu_ps(kptr + 8);
                    __m256 _k22 = _mm256_loadu_ps(kptr + 16);
                    __m256 _k23 = _mm256_loadu_ps(kptr + 24);
                    __m256 _k24 = _mm256_loadu_ps(kptr + 32);
                    __m256 _k25 = _mm256_loadu_ps(kptr + 40);
                    __m256 _k26 = _mm256_loadu_ps(kptr + 48);
                    __m256 _k27 = _mm256_loadu_ps(kptr + 56);

                    kptr += 64;

                    _sum00 = _mm256_comp_fmadd_ps(_r020, _k20, _sum00);
                    _sum01 = _mm256_comp_fmadd_ps(_r021, _k21, _sum01);
                    _sum00 = _mm256_comp_fmadd_ps(_r022, _k22, _sum00);
                    _sum01 = _mm256_comp_fmadd_ps(_r023, _k23, _sum01);
                    _sum00 = _mm256_comp_fmadd_ps(_r024, _k24, _sum00);
                    _sum01 = _mm256_comp_fmadd_ps(_r025, _k25, _sum01);
                    _sum00 = _mm256_comp_fmadd_ps(_r026, _k26, _sum00);
                    _sum01 = _mm256_comp_fmadd_ps(_r027, _k27, _sum01);

                    __m256 _r030 = _mm256_broadcast_ss(r0 + 24);
                    __m256 _r031 = _mm256_broadcast_ss(r0 + 25);
                    __m256 _r032 = _mm256_broadcast_ss(r0 + 26);
                    __m256 _r033 = _mm256_broadcast_ss(r0 + 27);
                    __m256 _r034 = _mm256_broadcast_ss(r0 + 28);
                    __m256 _r035 = _mm256_broadcast_ss(r0 + 29);
                    __m256 _r036 = _mm256_broadcast_ss(r0 + 30);
                    __m256 _r037 = _mm256_broadcast_ss(r0 + 31);

                    _sum10 = _mm256_comp_fmadd_ps(_r030, _k20, _sum10);
                    _sum11 = _mm256_comp_fmadd_ps(_r031, _k21, _sum11);
                    _sum10 = _mm256_comp_fmadd_ps(_r032, _k22, _sum10);
                    _sum11 = _mm256_comp_fmadd_ps(_r033, _k23, _sum11);
                    _sum10 = _mm256_comp_fmadd_ps(_r034, _k24, _sum10);
                    _sum11 = _mm256_comp_fmadd_ps(_r035, _k25, _sum11);
                    _sum10 = _mm256_comp_fmadd_ps(_r036, _k26, _sum10);
                    _sum11 = _mm256_comp_fmadd_ps(_r037, _k27, _sum11);

                    __m256 _r100 = _mm256_broadcast_ss(r1 + 0);
                    __m256 _r101 = _mm256_broadcast_ss(r1 + 1);
                    __m256 _r102 = _mm256_broadcast_ss(r1 + 2);
                    __m256 _r103 = _mm256_broadcast_ss(r1 + 3);
                    __m256 _r104 = _mm256_broadcast_ss(r1 + 4);
                    __m256 _r105 = _mm256_broadcast_ss(r1 + 5);
                    __m256 _r106 = _mm256_broadcast_ss(r1 + 6);
                    __m256 _r107 = _mm256_broadcast_ss(r1 + 7);

                    __m256 _k30 = _mm256_loadu_ps(kptr);
                    __m256 _k31 = _mm256_loadu_ps(kptr + 8);
                    __m256 _k32 = _mm256_loadu_ps(kptr + 16);
                    __m256 _k33 = _mm256_loadu_ps(kptr + 24);
                    __m256 _k34 = _mm256_loadu_ps(kptr + 32);
                    __m256 _k35 = _mm256_loadu_ps(kptr + 40);
                    __m256 _k36 = _mm256_loadu_ps(kptr + 48);
                    __m256 _k37 = _mm256_loadu_ps(kptr + 56);

                    kptr += 64;

                    _sum00 = _mm256_comp_fmadd_ps(_r100, _k30, _sum00);
                    _sum01 = _mm256_comp_fmadd_ps(_r101, _k31, _sum01);
                    _sum00 = _mm256_comp_fmadd_ps(_r102, _k32, _sum00);
                    _sum01 = _mm256_comp_fmadd_ps(_r103, _k33, _sum01);
                    _sum00 = _mm256_comp_fmadd_ps(_r104, _k34, _sum00);
                    _sum01 = _mm256_comp_fmadd_ps(_r105, _k35, _sum01);
                    _sum00 = _mm256_comp_fmadd_ps(_r106, _k36, _sum00);
                    _sum01 = _mm256_comp_fmadd_ps(_r107, _k37, _sum01);

                    __m256 _r110 = _mm256_broadcast_ss(r1 + 8);
                    __m256 _r111 = _mm256_broadcast_ss(r1 + 9);
                    __m256 _r112 = _mm256_broadcast_ss(r1 + 10);
                    __m256 _r113 = _mm256_broadcast_ss(r1 + 11);
                    __m256 _r114 = _mm256_broadcast_ss(r1 + 12);
                    __m256 _r115 = _mm256_broadcast_ss(r1 + 13);
                    __m256 _r116 = _mm256_broadcast_ss(r1 + 14);
                    __m256 _r117 = _mm256_broadcast_ss(r1 + 15);

                    _sum10 = _mm256_comp_fmadd_ps(_r110, _k30, _sum10);
                    _sum11 = _mm256_comp_fmadd_ps(_r111, _k31, _sum11);
                    _sum10 = _mm256_comp_fmadd_ps(_r112, _k32, _sum10);
                    _sum11 = _mm256_comp_fmadd_ps(_r113, _k33, _sum11);
                    _sum10 = _mm256_comp_fmadd_ps(_r114, _k34, _sum10);
                    _sum11 = _mm256_comp_fmadd_ps(_r115, _k35, _sum11);
                    _sum10 = _mm256_comp_fmadd_ps(_r116, _k36, _sum10);
                    _sum11 = _mm256_comp_fmadd_ps(_r117, _k37, _sum11);

                    __m256 _k40 = _mm256_loadu_ps(kptr);
                    __m256 _k41 = _mm256_loadu_ps(kptr + 8);
                    __m256 _k42 = _mm256_loadu_ps(kptr + 16);
                    __m256 _k43 = _mm256_loadu_ps(kptr + 24);
                    __m256 _k44 = _mm256_loadu_ps(kptr + 32);
                    __m256 _k45 = _mm256_loadu_ps(kptr + 40);
                    __m256 _k46 = _mm256_loadu_ps(kptr + 48);
                    __m256 _k47 = _mm256_loadu_ps(kptr + 56);

                    kptr += 64;

                    _sum00 = _mm256_comp_fmadd_ps(_r110, _k40, _sum00);
                    _sum01 = _mm256_comp_fmadd_ps(_r111, _k41, _sum01);
                    _sum00 = _mm256_comp_fmadd_ps(_r112, _k42, _sum00);
                    _sum01 = _mm256_comp_fmadd_ps(_r113, _k43, _sum01);
                    _sum00 = _mm256_comp_fmadd_ps(_r114, _k44, _sum00);
                    _sum01 = _mm256_comp_fmadd_ps(_r115, _k45, _sum01);
                    _sum00 = _mm256_comp_fmadd_ps(_r116, _k46, _sum00);
                    _sum01 = _mm256_comp_fmadd_ps(_r117, _k47, _sum01);

                    __m256 _r120 = _mm256_broadcast_ss(r1 + 16);
                    __m256 _r121 = _mm256_broadcast_ss(r1 + 17);
                    __m256 _r122 = _mm256_broadcast_ss(r1 + 18);
                    __m256 _r123 = _mm256_broadcast_ss(r1 + 19);
                    __m256 _r124 = _mm256_broadcast_ss(r1 + 20);
                    __m256 _r125 = _mm256_broadcast_ss(r1 + 21);
                    __m256 _r126 = _mm256_broadcast_ss(r1 + 22);
                    __m256 _r127 = _mm256_broadcast_ss(r1 + 23);

                    _sum10 = _mm256_comp_fmadd_ps(_r120, _k40, _sum10);
                    _sum11 = _mm256_comp_fmadd_ps(_r121, _k41, _sum11);
                    _sum10 = _mm256_comp_fmadd_ps(_r122, _k42, _sum10);
                    _sum11 = _mm256_comp_fmadd_ps(_r123, _k43, _sum11);
                    _sum10 = _mm256_comp_fmadd_ps(_r124, _k44, _sum10);
                    _sum11 = _mm256_comp_fmadd_ps(_r125, _k45, _sum11);
                    _sum10 = _mm256_comp_fmadd_ps(_r126, _k46, _sum10);
                    _sum11 = _mm256_comp_fmadd_ps(_r127, _k47, _sum11);

                    __m256 _k50 = _mm256_loadu_ps(kptr);
                    __m256 _k51 = _mm256_loadu_ps(kptr + 8);
                    __m256 _k52 = _mm256_loadu_ps(kptr + 16);
                    __m256 _k53 = _mm256_loadu_ps(kptr + 24);
                    __m256 _k54 = _mm256_loadu_ps(kptr + 32);
                    __m256 _k55 = _mm256_loadu_ps(kptr + 40);
                    __m256 _k56 = _mm256_loadu_ps(kptr + 48);
                    __m256 _k57 = _mm256_loadu_ps(kptr + 56);

                    kptr += 64;

                    _sum00 = _mm256_comp_fmadd_ps(_r120, _k50, _sum00);
                    _sum01 = _mm256_comp_fmadd_ps(_r121, _k51, _sum01);
                    _sum00 = _mm256_comp_fmadd_ps(_r122, _k52, _sum00);
                    _sum01 = _mm256_comp_fmadd_ps(_r123, _k53, _sum01);
                    _sum00 = _mm256_comp_fmadd_ps(_r124, _k54, _sum00);
                    _sum01 = _mm256_comp_fmadd_ps(_r125, _k55, _sum01);
                    _sum00 = _mm256_comp_fmadd_ps(_r126, _k56, _sum00);
                    _sum01 = _mm256_comp_fmadd_ps(_r127, _k57, _sum01);

                    __m256 _r130 = _mm256_broadcast_ss(r1 + 24);
                    __m256 _r131 = _mm256_broadcast_ss(r1 + 25);
                    __m256 _r132 = _mm256_broadcast_ss(r1 + 26);
                    __m256 _r133 = _mm256_broadcast_ss(r1 + 27);
                    __m256 _r134 = _mm256_broadcast_ss(r1 + 28);
                    __m256 _r135 = _mm256_broadcast_ss(r1 + 29);
                    __m256 _r136 = _mm256_broadcast_ss(r1 + 30);
                    __m256 _r137 = _mm256_broadcast_ss(r1 + 31);

                    _sum10 = _mm256_comp_fmadd_ps(_r130, _k50, _sum10);
                    _sum11 = _mm256_comp_fmadd_ps(_r131, _k51, _sum11);
                    _sum10 = _mm256_comp_fmadd_ps(_r132, _k52, _sum10);
                    _sum11 = _mm256_comp_fmadd_ps(_r133, _k53, _sum11);
                    _sum10 = _mm256_comp_fmadd_ps(_r134, _k54, _sum10);
                    _sum11 = _mm256_comp_fmadd_ps(_r135, _k55, _sum11);
                    _sum10 = _mm256_comp_fmadd_ps(_r136, _k56, _sum10);
                    _sum11 = _mm256_comp_fmadd_ps(_r137, _k57, _sum11);

                    __m256 _r200 = _mm256_broadcast_ss(r2 + 0);
                    __m256 _r201 = _mm256_broadcast_ss(r2 + 1);
                    __m256 _r202 = _mm256_broadcast_ss(r2 + 2);
                    __m256 _r203 = _mm256_broadcast_ss(r2 + 3);
                    __m256 _r204 = _mm256_broadcast_ss(r2 + 4);
                    __m256 _r205 = _mm256_broadcast_ss(r2 + 5);
                    __m256 _r206 = _mm256_broadcast_ss(r2 + 6);
                    __m256 _r207 = _mm256_broadcast_ss(r2 + 7);

                    __m256 _k60 = _mm256_loadu_ps(kptr);
                    __m256 _k61 = _mm256_loadu_ps(kptr + 8);
                    __m256 _k62 = _mm256_loadu_ps(kptr + 16);
                    __m256 _k63 = _mm256_loadu_ps(kptr + 24);
                    __m256 _k64 = _mm256_loadu_ps(kptr + 32);
                    __m256 _k65 = _mm256_loadu_ps(kptr + 40);
                    __m256 _k66 = _mm256_loadu_ps(kptr + 48);
                    __m256 _k67 = _mm256_loadu_ps(kptr + 56);

                    kptr += 64;

                    _sum00 = _mm256_comp_fmadd_ps(_r200, _k60, _sum00);
                    _sum01 = _mm256_comp_fmadd_ps(_r201, _k61, _sum01);
                    _sum00 = _mm256_comp_fmadd_ps(_r202, _k62, _sum00);
                    _sum01 = _mm256_comp_fmadd_ps(_r203, _k63, _sum01);
                    _sum00 = _mm256_comp_fmadd_ps(_r204, _k64, _sum00);
                    _sum01 = _mm256_comp_fmadd_ps(_r205, _k65, _sum01);
                    _sum00 = _mm256_comp_fmadd_ps(_r206, _k66, _sum00);
                    _sum01 = _mm256_comp_fmadd_ps(_r207, _k67, _sum01);

                    __m256 _r210 = _mm256_broadcast_ss(r2 + 8);
                    __m256 _r211 = _mm256_broadcast_ss(r2 + 9);
                    __m256 _r212 = _mm256_broadcast_ss(r2 + 10);
                    __m256 _r213 = _mm256_broadcast_ss(r2 + 11);
                    __m256 _r214 = _mm256_broadcast_ss(r2 + 12);
                    __m256 _r215 = _mm256_broadcast_ss(r2 + 13);
                    __m256 _r216 = _mm256_broadcast_ss(r2 + 14);
                    __m256 _r217 = _mm256_broadcast_ss(r2 + 15);

                    _sum10 = _mm256_comp_fmadd_ps(_r210, _k60, _sum10);
                    _sum11 = _mm256_comp_fmadd_ps(_r211, _k61, _sum11);
                    _sum10 = _mm256_comp_fmadd_ps(_r212, _k62, _sum10);
                    _sum11 = _mm256_comp_fmadd_ps(_r213, _k63, _sum11);
                    _sum10 = _mm256_comp_fmadd_ps(_r214, _k64, _sum10);
                    _sum11 = _mm256_comp_fmadd_ps(_r215, _k65, _sum11);
                    _sum10 = _mm256_comp_fmadd_ps(_r216, _k66, _sum10);
                    _sum11 = _mm256_comp_fmadd_ps(_r217, _k67, _sum11);

                    __m256 _k70 = _mm256_loadu_ps(kptr);
                    __m256 _k71 = _mm256_loadu_ps(kptr + 8);
                    __m256 _k72 = _mm256_loadu_ps(kptr + 16);
                    __m256 _k73 = _mm256_loadu_ps(kptr + 24);
                    __m256 _k74 = _mm256_loadu_ps(kptr + 32);
                    __m256 _k75 = _mm256_loadu_ps(kptr + 40);
                    __m256 _k76 = _mm256_loadu_ps(kptr + 48);
                    __m256 _k77 = _mm256_loadu_ps(kptr + 56);

                    kptr += 64;

                    _sum00 = _mm256_comp_fmadd_ps(_r210, _k70, _sum00);
                    _sum01 = _mm256_comp_fmadd_ps(_r211, _k71, _sum01);
                    _sum00 = _mm256_comp_fmadd_ps(_r212, _k72, _sum00);
                    _sum01 = _mm256_comp_fmadd_ps(_r213, _k73, _sum01);
                    _sum00 = _mm256_comp_fmadd_ps(_r214, _k74, _sum00);
                    _sum01 = _mm256_comp_fmadd_ps(_r215, _k75, _sum01);
                    _sum00 = _mm256_comp_fmadd_ps(_r216, _k76, _sum00);
                    _sum01 = _mm256_comp_fmadd_ps(_r217, _k77, _sum01);

                    __m256 _r220 = _mm256_broadcast_ss(r2 + 16);
                    __m256 _r221 = _mm256_broadcast_ss(r2 + 17);
                    __m256 _r222 = _mm256_broadcast_ss(r2 + 18);
                    __m256 _r223 = _mm256_broadcast_ss(r2 + 19);
                    __m256 _r224 = _mm256_broadcast_ss(r2 + 20);
                    __m256 _r225 = _mm256_broadcast_ss(r2 + 21);
                    __m256 _r226 = _mm256_broadcast_ss(r2 + 22);
                    __m256 _r227 = _mm256_broadcast_ss(r2 + 23);

                    _sum10 = _mm256_comp_fmadd_ps(_r220, _k70, _sum10);
                    _sum11 = _mm256_comp_fmadd_ps(_r221, _k71, _sum11);
                    _sum10 = _mm256_comp_fmadd_ps(_r222, _k72, _sum10);
                    _sum11 = _mm256_comp_fmadd_ps(_r223, _k73, _sum11);
                    _sum10 = _mm256_comp_fmadd_ps(_r224, _k74, _sum10);
                    _sum11 = _mm256_comp_fmadd_ps(_r225, _k75, _sum11);
                    _sum10 = _mm256_comp_fmadd_ps(_r226, _k76, _sum10);
                    _sum11 = _mm256_comp_fmadd_ps(_r227, _k77, _sum11);

                    __m256 _k80 = _mm256_loadu_ps(kptr);
                    __m256 _k81 = _mm256_loadu_ps(kptr + 8);
                    __m256 _k82 = _mm256_loadu_ps(kptr + 16);
                    __m256 _k83 = _mm256_loadu_ps(kptr + 24);
                    __m256 _k84 = _mm256_loadu_ps(kptr + 32);
                    __m256 _k85 = _mm256_loadu_ps(kptr + 40);
                    __m256 _k86 = _mm256_loadu_ps(kptr + 48);
                    __m256 _k87 = _mm256_loadu_ps(kptr + 56);

                    _sum00 = _mm256_comp_fmadd_ps(_r220, _k80, _sum00);
                    _sum01 = _mm256_comp_fmadd_ps(_r221, _k81, _sum01);
                    _sum00 = _mm256_comp_fmadd_ps(_r222, _k82, _sum00);
                    _sum01 = _mm256_comp_fmadd_ps(_r223, _k83, _sum01);
                    _sum00 = _mm256_comp_fmadd_ps(_r224, _k84, _sum00);
                    _sum01 = _mm256_comp_fmadd_ps(_r225, _k85, _sum01);
                    _sum00 = _mm256_comp_fmadd_ps(_r226, _k86, _sum00);
                    _sum01 = _mm256_comp_fmadd_ps(_r227, _k87, _sum01);

                    __m256 _r230 = _mm256_broadcast_ss(r2 + 24);
                    __m256 _r231 = _mm256_broadcast_ss(r2 + 25);
                    __m256 _r232 = _mm256_broadcast_ss(r2 + 26);
                    __m256 _r233 = _mm256_broadcast_ss(r2 + 27);
                    __m256 _r234 = _mm256_broadcast_ss(r2 + 28);
                    __m256 _r235 = _mm256_broadcast_ss(r2 + 29);
                    __m256 _r236 = _mm256_broadcast_ss(r2 + 30);
                    __m256 _r237 = _mm256_broadcast_ss(r2 + 31);

                    _sum10 = _mm256_comp_fmadd_ps(_r230, _k80, _sum10);
                    _sum11 = _mm256_comp_fmadd_ps(_r231, _k81, _sum11);
                    _sum10 = _mm256_comp_fmadd_ps(_r232, _k82, _sum10);
                    _sum11 = _mm256_comp_fmadd_ps(_r233, _k83, _sum11);
                    _sum10 = _mm256_comp_fmadd_ps(_r234, _k84, _sum10);
                    _sum11 = _mm256_comp_fmadd_ps(_r235, _k85, _sum11);
                    _sum10 = _mm256_comp_fmadd_ps(_r236, _k86, _sum10);
                    _sum11 = _mm256_comp_fmadd_ps(_r237, _k87, _sum11);

                    kptr -= 64 * 8;

                    _sum00 = _mm256_add_ps(_sum00, _sum01);
                    _sum10 = _mm256_add_ps(_sum10, _sum11);

                    _mm256_storeu_ps(outptr, _sum00);
                    _mm256_storeu_ps(outptr + 8, _sum10);

                    r0 += 16;
                    r1 += 16;
                    r2 += 16;
                    outptr += 16;
                }
                for (; j < outw; j++)
                {
                    __m256 _sum0 = _mm256_loadu_ps(outptr);
                    __m256 _sum1 = _mm256_setzero_ps();

                    __m256 _r000 = _mm256_broadcast_ss(r0 + 0);
                    __m256 _r001 = _mm256_broadcast_ss(r0 + 1);
                    __m256 _r002 = _mm256_broadcast_ss(r0 + 2);
                    __m256 _r003 = _mm256_broadcast_ss(r0 + 3);
                    __m256 _r004 = _mm256_broadcast_ss(r0 + 4);
                    __m256 _r005 = _mm256_broadcast_ss(r0 + 5);
                    __m256 _r006 = _mm256_broadcast_ss(r0 + 6);
                    __m256 _r007 = _mm256_broadcast_ss(r0 + 7);

                    __m256 _k00 = _mm256_loadu_ps(kptr);
                    __m256 _k01 = _mm256_loadu_ps(kptr + 8);
                    __m256 _k02 = _mm256_loadu_ps(kptr + 16);
                    __m256 _k03 = _mm256_loadu_ps(kptr + 24);
                    __m256 _k04 = _mm256_loadu_ps(kptr + 32);
                    __m256 _k05 = _mm256_loadu_ps(kptr + 40);
                    __m256 _k06 = _mm256_loadu_ps(kptr + 48);
                    __m256 _k07 = _mm256_loadu_ps(kptr + 56);

                    kptr += 64;

                    _sum0 = _mm256_comp_fmadd_ps(_r000, _k00, _sum0);
                    _sum1 = _mm256_comp_fmadd_ps(_r001, _k01, _sum1);
                    _sum0 = _mm256_comp_fmadd_ps(_r002, _k02, _sum0);
                    _sum1 = _mm256_comp_fmadd_ps(_r003, _k03, _sum1);
                    _sum0 = _mm256_comp_fmadd_ps(_r004, _k04, _sum0);
                    _sum1 = _mm256_comp_fmadd_ps(_r005, _k05, _sum1);
                    _sum0 = _mm256_comp_fmadd_ps(_r006, _k06, _sum0);
                    _sum1 = _mm256_comp_fmadd_ps(_r007, _k07, _sum1);

                    __m256 _r010 = _mm256_broadcast_ss(r0 + 8);
                    __m256 _r011 = _mm256_broadcast_ss(r0 + 9);
                    __m256 _r012 = _mm256_broadcast_ss(r0 + 10);
                    __m256 _r013 = _mm256_broadcast_ss(r0 + 11);
                    __m256 _r014 = _mm256_broadcast_ss(r0 + 12);
                    __m256 _r015 = _mm256_broadcast_ss(r0 + 13);
                    __m256 _r016 = _mm256_broadcast_ss(r0 + 14);
                    __m256 _r017 = _mm256_broadcast_ss(r0 + 15);

                    __m256 _k10 = _mm256_loadu_ps(kptr);
                    __m256 _k11 = _mm256_loadu_ps(kptr + 8);
                    __m256 _k12 = _mm256_loadu_ps(kptr + 16);
                    __m256 _k13 = _mm256_loadu_ps(kptr + 24);
                    __m256 _k14 = _mm256_loadu_ps(kptr + 32);
                    __m256 _k15 = _mm256_loadu_ps(kptr + 40);
                    __m256 _k16 = _mm256_loadu_ps(kptr + 48);
                    __m256 _k17 = _mm256_loadu_ps(kptr + 56);

                    kptr += 64;

                    _sum0 = _mm256_comp_fmadd_ps(_r010, _k10, _sum0);
                    _sum1 = _mm256_comp_fmadd_ps(_r011, _k11, _sum1);
                    _sum0 = _mm256_comp_fmadd_ps(_r012, _k12, _sum0);
                    _sum1 = _mm256_comp_fmadd_ps(_r013, _k13, _sum1);
                    _sum0 = _mm256_comp_fmadd_ps(_r014, _k14, _sum0);
                    _sum1 = _mm256_comp_fmadd_ps(_r015, _k15, _sum1);
                    _sum0 = _mm256_comp_fmadd_ps(_r016, _k16, _sum0);
                    _sum1 = _mm256_comp_fmadd_ps(_r017, _k17, _sum1);

                    __m256 _r020 = _mm256_broadcast_ss(r0 + 16);
                    __m256 _r021 = _mm256_broadcast_ss(r0 + 17);
                    __m256 _r022 = _mm256_broadcast_ss(r0 + 18);
                    __m256 _r023 = _mm256_broadcast_ss(r0 + 19);
                    __m256 _r024 = _mm256_broadcast_ss(r0 + 20);
                    __m256 _r025 = _mm256_broadcast_ss(r0 + 21);
                    __m256 _r026 = _mm256_broadcast_ss(r0 + 22);
                    __m256 _r027 = _mm256_broadcast_ss(r0 + 23);

                    __m256 _k20 = _mm256_loadu_ps(kptr);
                    __m256 _k21 = _mm256_loadu_ps(kptr + 8);
                    __m256 _k22 = _mm256_loadu_ps(kptr + 16);
                    __m256 _k23 = _mm256_loadu_ps(kptr + 24);
                    __m256 _k24 = _mm256_loadu_ps(kptr + 32);
                    __m256 _k25 = _mm256_loadu_ps(kptr + 40);
                    __m256 _k26 = _mm256_loadu_ps(kptr + 48);
                    __m256 _k27 = _mm256_loadu_ps(kptr + 56);

                    kptr += 64;

                    _sum0 = _mm256_comp_fmadd_ps(_r020, _k20, _sum0);
                    _sum1 = _mm256_comp_fmadd_ps(_r021, _k21, _sum1);
                    _sum0 = _mm256_comp_fmadd_ps(_r022, _k22, _sum0);
                    _sum1 = _mm256_comp_fmadd_ps(_r023, _k23, _sum1);
                    _sum0 = _mm256_comp_fmadd_ps(_r024, _k24, _sum0);
                    _sum1 = _mm256_comp_fmadd_ps(_r025, _k25, _sum1);
                    _sum0 = _mm256_comp_fmadd_ps(_r026, _k26, _sum0);
                    _sum1 = _mm256_comp_fmadd_ps(_r027, _k27, _sum1);

                    __m256 _r100 = _mm256_broadcast_ss(r1 + 0);
                    __m256 _r101 = _mm256_broadcast_ss(r1 + 1);
                    __m256 _r102 = _mm256_broadcast_ss(r1 + 2);
                    __m256 _r103 = _mm256_broadcast_ss(r1 + 3);
                    __m256 _r104 = _mm256_broadcast_ss(r1 + 4);
                    __m256 _r105 = _mm256_broadcast_ss(r1 + 5);
                    __m256 _r106 = _mm256_broadcast_ss(r1 + 6);
                    __m256 _r107 = _mm256_broadcast_ss(r1 + 7);

                    __m256 _k30 = _mm256_loadu_ps(kptr);
                    __m256 _k31 = _mm256_loadu_ps(kptr + 8);
                    __m256 _k32 = _mm256_loadu_ps(kptr + 16);
                    __m256 _k33 = _mm256_loadu_ps(kptr + 24);
                    __m256 _k34 = _mm256_loadu_ps(kptr + 32);
                    __m256 _k35 = _mm256_loadu_ps(kptr + 40);
                    __m256 _k36 = _mm256_loadu_ps(kptr + 48);
                    __m256 _k37 = _mm256_loadu_ps(kptr + 56);

                    kptr += 64;

                    _sum0 = _mm256_comp_fmadd_ps(_r100, _k30, _sum0);
                    _sum1 = _mm256_comp_fmadd_ps(_r101, _k31, _sum1);
                    _sum0 = _mm256_comp_fmadd_ps(_r102, _k32, _sum0);
                    _sum1 = _mm256_comp_fmadd_ps(_r103, _k33, _sum1);
                    _sum0 = _mm256_comp_fmadd_ps(_r104, _k34, _sum0);
                    _sum1 = _mm256_comp_fmadd_ps(_r105, _k35, _sum1);
                    _sum0 = _mm256_comp_fmadd_ps(_r106, _k36, _sum0);
                    _sum1 = _mm256_comp_fmadd_ps(_r107, _k37, _sum1);

                    __m256 _r110 = _mm256_broadcast_ss(r1 + 8);
                    __m256 _r111 = _mm256_broadcast_ss(r1 + 9);
                    __m256 _r112 = _mm256_broadcast_ss(r1 + 10);
                    __m256 _r113 = _mm256_broadcast_ss(r1 + 11);
                    __m256 _r114 = _mm256_broadcast_ss(r1 + 12);
                    __m256 _r115 = _mm256_broadcast_ss(r1 + 13);
                    __m256 _r116 = _mm256_broadcast_ss(r1 + 14);
                    __m256 _r117 = _mm256_broadcast_ss(r1 + 15);

                    __m256 _k40 = _mm256_loadu_ps(kptr);
                    __m256 _k41 = _mm256_loadu_ps(kptr + 8);
                    __m256 _k42 = _mm256_loadu_ps(kptr + 16);
                    __m256 _k43 = _mm256_loadu_ps(kptr + 24);
                    __m256 _k44 = _mm256_loadu_ps(kptr + 32);
                    __m256 _k45 = _mm256_loadu_ps(kptr + 40);
                    __m256 _k46 = _mm256_loadu_ps(kptr + 48);
                    __m256 _k47 = _mm256_loadu_ps(kptr + 56);

                    kptr += 64;

                    _sum0 = _mm256_comp_fmadd_ps(_r110, _k40, _sum0);
                    _sum1 = _mm256_comp_fmadd_ps(_r111, _k41, _sum1);
                    _sum0 = _mm256_comp_fmadd_ps(_r112, _k42, _sum0);
                    _sum1 = _mm256_comp_fmadd_ps(_r113, _k43, _sum1);
                    _sum0 = _mm256_comp_fmadd_ps(_r114, _k44, _sum0);
                    _sum1 = _mm256_comp_fmadd_ps(_r115, _k45, _sum1);
                    _sum0 = _mm256_comp_fmadd_ps(_r116, _k46, _sum0);
                    _sum1 = _mm256_comp_fmadd_ps(_r117, _k47, _sum1);

                    __m256 _r120 = _mm256_broadcast_ss(r1 + 16);
                    __m256 _r121 = _mm256_broadcast_ss(r1 + 17);
                    __m256 _r122 = _mm256_broadcast_ss(r1 + 18);
                    __m256 _r123 = _mm256_broadcast_ss(r1 + 19);
                    __m256 _r124 = _mm256_broadcast_ss(r1 + 20);
                    __m256 _r125 = _mm256_broadcast_ss(r1 + 21);
                    __m256 _r126 = _mm256_broadcast_ss(r1 + 22);
                    __m256 _r127 = _mm256_broadcast_ss(r1 + 23);

                    __m256 _k50 = _mm256_loadu_ps(kptr);
                    __m256 _k51 = _mm256_loadu_ps(kptr + 8);
                    __m256 _k52 = _mm256_loadu_ps(kptr + 16);
                    __m256 _k53 = _mm256_loadu_ps(kptr + 24);
                    __m256 _k54 = _mm256_loadu_ps(kptr + 32);
                    __m256 _k55 = _mm256_loadu_ps(kptr + 40);
                    __m256 _k56 = _mm256_loadu_ps(kptr + 48);
                    __m256 _k57 = _mm256_loadu_ps(kptr + 56);

                    kptr += 64;

                    _sum0 = _mm256_comp_fmadd_ps(_r120, _k50, _sum0);
                    _sum1 = _mm256_comp_fmadd_ps(_r121, _k51, _sum1);
                    _sum0 = _mm256_comp_fmadd_ps(_r122, _k52, _sum0);
                    _sum1 = _mm256_comp_fmadd_ps(_r123, _k53, _sum1);
                    _sum0 = _mm256_comp_fmadd_ps(_r124, _k54, _sum0);
                    _sum1 = _mm256_comp_fmadd_ps(_r125, _k55, _sum1);
                    _sum0 = _mm256_comp_fmadd_ps(_r126, _k56, _sum0);
                    _sum1 = _mm256_comp_fmadd_ps(_r127, _k57, _sum1);

                    __m256 _r200 = _mm256_broadcast_ss(r2 + 0);
                    __m256 _r201 = _mm256_broadcast_ss(r2 + 1);
                    __m256 _r202 = _mm256_broadcast_ss(r2 + 2);
                    __m256 _r203 = _mm256_broadcast_ss(r2 + 3);
                    __m256 _r204 = _mm256_broadcast_ss(r2 + 4);
                    __m256 _r205 = _mm256_broadcast_ss(r2 + 5);
                    __m256 _r206 = _mm256_broadcast_ss(r2 + 6);
                    __m256 _r207 = _mm256_broadcast_ss(r2 + 7);

                    __m256 _k60 = _mm256_loadu_ps(kptr);
                    __m256 _k61 = _mm256_loadu_ps(kptr + 8);
                    __m256 _k62 = _mm256_loadu_ps(kptr + 16);
                    __m256 _k63 = _mm256_loadu_ps(kptr + 24);
                    __m256 _k64 = _mm256_loadu_ps(kptr + 32);
                    __m256 _k65 = _mm256_loadu_ps(kptr + 40);
                    __m256 _k66 = _mm256_loadu_ps(kptr + 48);
                    __m256 _k67 = _mm256_loadu_ps(kptr + 56);

                    kptr += 64;

                    _sum0 = _mm256_comp_fmadd_ps(_r200, _k60, _sum0);
                    _sum1 = _mm256_comp_fmadd_ps(_r201, _k61, _sum1);
                    _sum0 = _mm256_comp_fmadd_ps(_r202, _k62, _sum0);
                    _sum1 = _mm256_comp_fmadd_ps(_r203, _k63, _sum1);
                    _sum0 = _mm256_comp_fmadd_ps(_r204, _k64, _sum0);
                    _sum1 = _mm256_comp_fmadd_ps(_r205, _k65, _sum1);
                    _sum0 = _mm256_comp_fmadd_ps(_r206, _k66, _sum0);
                    _sum1 = _mm256_comp_fmadd_ps(_r207, _k67, _sum1);

                    __m256 _r210 = _mm256_broadcast_ss(r2 + 8);
                    __m256 _r211 = _mm256_broadcast_ss(r2 + 9);
                    __m256 _r212 = _mm256_broadcast_ss(r2 + 10);
                    __m256 _r213 = _mm256_broadcast_ss(r2 + 11);
                    __m256 _r214 = _mm256_broadcast_ss(r2 + 12);
                    __m256 _r215 = _mm256_broadcast_ss(r2 + 13);
                    __m256 _r216 = _mm256_broadcast_ss(r2 + 14);
                    __m256 _r217 = _mm256_broadcast_ss(r2 + 15);

                    __m256 _k70 = _mm256_loadu_ps(kptr);
                    __m256 _k71 = _mm256_loadu_ps(kptr + 8);
                    __m256 _k72 = _mm256_loadu_ps(kptr + 16);
                    __m256 _k73 = _mm256_loadu_ps(kptr + 24);
                    __m256 _k74 = _mm256_loadu_ps(kptr + 32);
                    __m256 _k75 = _mm256_loadu_ps(kptr + 40);
                    __m256 _k76 = _mm256_loadu_ps(kptr + 48);
                    __m256 _k77 = _mm256_loadu_ps(kptr + 56);

                    kptr += 64;

                    _sum0 = _mm256_comp_fmadd_ps(_r210, _k70, _sum0);
                    _sum1 = _mm256_comp_fmadd_ps(_r211, _k71, _sum1);
                    _sum0 = _mm256_comp_fmadd_ps(_r212, _k72, _sum0);
                    _sum1 = _mm256_comp_fmadd_ps(_r213, _k73, _sum1);
                    _sum0 = _mm256_comp_fmadd_ps(_r214, _k74, _sum0);
                    _sum1 = _mm256_comp_fmadd_ps(_r215, _k75, _sum1);
                    _sum0 = _mm256_comp_fmadd_ps(_r216, _k76, _sum0);
                    _sum1 = _mm256_comp_fmadd_ps(_r217, _k77, _sum1);

                    __m256 _r220 = _mm256_broadcast_ss(r2 + 16);
                    __m256 _r221 = _mm256_broadcast_ss(r2 + 17);
                    __m256 _r222 = _mm256_broadcast_ss(r2 + 18);
                    __m256 _r223 = _mm256_broadcast_ss(r2 + 19);
                    __m256 _r224 = _mm256_broadcast_ss(r2 + 20);
                    __m256 _r225 = _mm256_broadcast_ss(r2 + 21);
                    __m256 _r226 = _mm256_broadcast_ss(r2 + 22);
                    __m256 _r227 = _mm256_broadcast_ss(r2 + 23);

                    __m256 _k80 = _mm256_loadu_ps(kptr);
                    __m256 _k81 = _mm256_loadu_ps(kptr + 8);
                    __m256 _k82 = _mm256_loadu_ps(kptr + 16);
                    __m256 _k83 = _mm256_loadu_ps(kptr + 24);
                    __m256 _k84 = _mm256_loadu_ps(kptr + 32);
                    __m256 _k85 = _mm256_loadu_ps(kptr + 40);
                    __m256 _k86 = _mm256_loadu_ps(kptr + 48);
                    __m256 _k87 = _mm256_loadu_ps(kptr + 56);

                    _sum0 = _mm256_comp_fmadd_ps(_r220, _k80, _sum0);
                    _sum1 = _mm256_comp_fmadd_ps(_r221, _k81, _sum1);
                    _sum0 = _mm256_comp_fmadd_ps(_r222, _k82, _sum0);
                    _sum1 = _mm256_comp_fmadd_ps(_r223, _k83, _sum1);
                    _sum0 = _mm256_comp_fmadd_ps(_r224, _k84, _sum0);
                    _sum1 = _mm256_comp_fmadd_ps(_r225, _k85, _sum1);
                    _sum0 = _mm256_comp_fmadd_ps(_r226, _k86, _sum0);
                    _sum1 = _mm256_comp_fmadd_ps(_r227, _k87, _sum1);

                    kptr -= 64 * 8;

                    _sum0 = _mm256_add_ps(_sum0, _sum1);

                    _mm256_storeu_ps(outptr, _sum0);

                    r0 += 8;
                    r1 += 8;
                    r2 += 8;
                    outptr += 8;
                }

                r0 += 16;
                r1 += 16;
                r2 += 16;
            }
        }
    }
}