src/layer/riscv/convolution_winograd_transform.h

// Tencent is pleased to support the open source community by making ncnn available.
//
// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved.
//
// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
// in compliance with the License. You may obtain a copy of the License at
//
// https://opensource.org/licenses/BSD-3-Clause
//
// Unless required by applicable law or agreed to in writing, software distributed
// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
// specific language governing permissions and limitations under the License.

static void conv3x3s1_winograd43_transform_input_rvv(const Mat& bottom_blob, Mat& bottom_blob_tm, const Option& opt)
{
    const int w = bottom_blob.w;
    const int h = bottom_blob.h;
    const int inch = bottom_blob.c;

    const int w_tiles = (w - 2) / 4;
    const int h_tiles = (h - 2) / 4;
    const int tiles = w_tiles * h_tiles;

    const float sq2 = 1.41421356237;
    const float sq2_d2 = 1.41421356237 / 2;

    // const float itm[6][6] = {
    //     {1.0f,  0.0f,  -2.5f,  0.0f,  1.0f, 0.0f},
    //     {0.0f, -sq2,   -2.0f,  sq2/2, 1.0f, 0.0f},
    //     {0.0f,  sq2,   -2.0f, -sq2/2, 1.0f, 0.0f},
    //     {0.0f, -sq2/2, -0.5f,  sq2,   1.0f, 0.0f},
    //     {0.0f,  sq2/2, -0.5f, -sq2,   1.0f, 0.0f},
    //     {0.0f,  1.0f,   0.0f,  -2.5f, 0.0f, 1.0f}
    // };

    // 0 =  r00 - 2.5f * r02 + r04
    // 1 = -(sq2 * r01 - sq2_d2 * r03) + (r04 - 2 * r02)
    // 2 =  (sq2 * r01 - sq2_d2 * r03) + (r04 - 2 * r02)
    // 3 = -(sq2_d2 * r01 - sq2 * r03) + (r04 - 0.5f * r02)
    // 4 =  (sq2_d2 * r01 - sq2 * r03) + (r04 - 0.5f * r02)
    // 5 =  r01 - 2.5f * r03 + r05

    #pragma omp parallel for num_threads(opt.num_threads)
    for (int q = 0; q < inch; q++)
    {
        const Mat img0 = bottom_blob.channel(q);
        Mat img0_tm = bottom_blob_tm.channel(q);

        float tmp[6][6];

        // tile
        for (int i = 0; i < h_tiles; i++)
        {
            for (int j = 0; j < w_tiles; j++)
            {
                const float* r0 = img0.row(i * 4) + (j * 4);

                for (int m = 0; m < 6; m++)
                {
                    float r00 = r0[0];
                    float r01 = r0[1];
                    float r02 = r0[2];
                    float r03 = r0[3];
                    float r04 = r0[4];
                    float r05 = r0[5];

                    float tmp01a = sq2 * r01 - sq2_d2 * r03;
                    float tmp01b = r04 - 2 * r02;
                    float tmp23a = sq2_d2 * r01 - sq2 * r03;
                    float tmp23b = r04 - 0.5f * r02;

                    tmp[0][m] = r00 - 2.5f * r02 + r04;
                    tmp[1][m] = tmp01b - tmp01a;
                    tmp[2][m] = tmp01b + tmp01a;
                    tmp[3][m] = tmp23b - tmp23a;
                    tmp[4][m] = tmp23b + tmp23a;
                    tmp[5][m] = r01 - 2.5f * r03 + r05;

                    r0 += w;
                }

                float* r0_tm_0 = (float*)img0_tm + (i * w_tiles + j);
                float* r0_tm_1 = r0_tm_0 + tiles;
                float* r0_tm_2 = r0_tm_0 + tiles * 2;
                float* r0_tm_3 = r0_tm_0 + tiles * 3;
                float* r0_tm_4 = r0_tm_0 + tiles * 4;
                float* r0_tm_5 = r0_tm_0 + tiles * 5;

                for (int m = 0; m < 6; m++)
                {
                    float r00 = tmp[m][0];
                    float r01 = tmp[m][1];
                    float r02 = tmp[m][2];
                    float r03 = tmp[m][3];
                    float r04 = tmp[m][4];
                    float r05 = tmp[m][5];

                    float tmp01a = sq2 * r01 - sq2_d2 * r03;
                    float tmp01b = r04 - 2 * r02;
                    float tmp23a = sq2_d2 * r01 - sq2 * r03;
                    float tmp23b = r04 - 0.5f * r02;

                    r0_tm_0[0] = r00 - 2.5f * r02 + r04;
                    r0_tm_1[0] = tmp01b - tmp01a;
                    r0_tm_2[0] = tmp01b + tmp01a;
                    r0_tm_3[0] = tmp23b - tmp23a;
                    r0_tm_4[0] = tmp23b + tmp23a;
                    r0_tm_5[0] = r01 - 2.5f * r03 + r05;

                    r0_tm_0 += tiles * 6;
                    r0_tm_1 += tiles * 6;
                    r0_tm_2 += tiles * 6;
                    r0_tm_3 += tiles * 6;
                    r0_tm_4 += tiles * 6;
                    r0_tm_5 += tiles * 6;
                }
            }
        }
    }
}

static void conv3x3s1_winograd43_transform_output_rvv(const Mat& top_blob_tm, Mat& top_blob, const Mat& bias, const Option& opt)
{
    const int outw = top_blob.w;
    const int outh = top_blob.h;
    const int outch = top_blob.c;

    const int w_tiles = outw / 4;
    const int h_tiles = outh / 4;
    const int tiles = w_tiles * h_tiles;

    const float* biasptr = bias;

    const float sq2 = 1.41421356237;
    const float sq2_m2 = 1.41421356237 * 2;
    const float sq2_d2 = 1.41421356237 / 2;
    const float sq2_d4 = 1.41421356237 / 4;

    // const float otm[4][6] = {
    //     {1.0f, 1.0f,   1.0f,  1.0f,  1.0f,   0.0f},
    //     {0.0f, sq2/2, -sq2/2, sq2,   -sq2,   0.0f},
    //     {0.0f, 0.5f,   0.5f,  2.0f,  2.0f,   0.0f},
    //     {0.0f, sq2/4, -sq2/4, sq2*2, -sq2*2, 1.0f}
    // };

    // 0 = r00 + (r01 + r02) + (r03 + r04)
    // 1 =       (r01 - r02) * sq2_d2 + (r03 - r04) * sq2
    // 2 =       (r01 + r02) * 0.5f + (r03 + r04) * 2
    // 3 = r05 + (r01 - r02) * sq2_d4 + (r03 - r04) * sq2_m2

    #pragma omp parallel for num_threads(opt.num_threads)
    for (int p = 0; p < outch; p++)
    {
        const Mat out0_tm = top_blob_tm.channel(p);
        Mat out0 = top_blob.channel(p);

        float bias0 = biasptr ? biasptr[p] : 0.f;

        float tmp[4][6];

        // tile
        for (int i = 0; i < h_tiles; i++)
        {
            for (int j = 0; j < w_tiles; j++)
            {
                const float* output0_tm_0 = (const float*)out0_tm + (i * w_tiles + j);
                const float* output0_tm_1 = output0_tm_0 + tiles;
                const float* output0_tm_2 = output0_tm_0 + tiles * 2;
                const float* output0_tm_3 = output0_tm_0 + tiles * 3;
                const float* output0_tm_4 = output0_tm_0 + tiles * 4;
                const float* output0_tm_5 = output0_tm_0 + tiles * 5;

                float* output0 = out0.row(i * 4) + (j * 4);

                for (int m = 0; m < 6; m++)
                {
                    float r00 = output0_tm_0[0];
                    float r01 = output0_tm_1[0];
                    float r02 = output0_tm_2[0];
                    float r03 = output0_tm_3[0];
                    float r04 = output0_tm_4[0];
                    float r05 = output0_tm_5[0];

                    float tmp02a = r01 + r02;
                    float tmp02b = r03 + r04;
                    float tmp13a = r01 - r02;
                    float tmp13b = r03 - r04;

                    tmp[0][m] = r00 + tmp02a + tmp02b;
                    tmp[1][m] = tmp13a * sq2_d2 + tmp13b * sq2;
                    tmp[2][m] = tmp02a * 0.5f + tmp02b * 2;
                    tmp[3][m] = r05 + tmp13a * sq2_d4 + tmp13b * sq2_m2;

                    output0_tm_0 += tiles * 6;
                    output0_tm_1 += tiles * 6;
                    output0_tm_2 += tiles * 6;
                    output0_tm_3 += tiles * 6;
                    output0_tm_4 += tiles * 6;
                    output0_tm_5 += tiles * 6;
                }

                for (int m = 0; m < 4; m++)
                {
                    float r00 = tmp[m][0];
                    float r01 = tmp[m][1];
                    float r02 = tmp[m][2];
                    float r03 = tmp[m][3];
                    float r04 = tmp[m][4];
                    float r05 = tmp[m][5];

                    float tmp02a = r01 + r02;
                    float tmp02b = r03 + r04;
                    float tmp13a = r01 - r02;
                    float tmp13b = r03 - r04;

                    float out00 = bias0 + r00 + tmp02a + tmp02b;
                    float out01 = bias0 + tmp13a * sq2_d2 + tmp13b * sq2;
                    float out02 = bias0 + tmp02a * 0.5f + tmp02b * 2;
                    float out03 = bias0 + r05 + tmp13a * sq2_d4 + tmp13b * sq2_m2;

                    output0[0] = out00;
                    output0[1] = out01;
                    output0[2] = out02;
                    output0[3] = out03;

                    output0 += outw;
                }
            }
        }
    }
}

static void conv3x3s1_winograd23_transform_input_rvv(const Mat& bottom_blob, Mat& bottom_blob_tm, const Option& opt)
{
    const int w = bottom_blob.w;
    const int h = bottom_blob.h;
    const int inch = bottom_blob.c;

    const int w_tiles = (w - 2) / 2;
    const int h_tiles = (h - 2) / 2;
    const int tiles = w_tiles * h_tiles;

    // const float itm[4][4] = {
    //     {1.0f,  0.0f, -1.0f,  0.0f},
    //     {0.0f,  1.0f,  1.00f, 0.0f},
    //     {0.0f, -1.0f,  1.00f, 0.0f},
    //     {0.0f, -1.0f,  0.00f, 1.0f}
    // };

    // 0 = r00 - r02
    // 1 = r01 + r02
    // 2 = r02 - r01
    // 3 = r03 - r01

    #pragma omp parallel for num_threads(opt.num_threads)
    for (int q = 0; q < inch; q++)
    {
        const Mat img0 = bottom_blob.channel(q);
        Mat img0_tm = bottom_blob_tm.channel(q);

        float tmp[4][4];

        // tile
        for (int i = 0; i < h_tiles; i++)
        {
            for (int j = 0; j < w_tiles; j++)
            {
                const float* r0 = img0.row(i * 2) + (j * 2);

                for (int m = 0; m < 4; m++)
                {
                    float r00 = r0[0];
                    float r01 = r0[1];
                    float r02 = r0[2];
                    float r03 = r0[3];

                    float tmp0m = r00 - r02;
                    float tmp1m = r01 + r02;
                    float tmp2m = r02 - r01;
                    float tmp3m = r03 - r01;

                    tmp[0][m] = tmp0m;
                    tmp[1][m] = tmp1m;
                    tmp[2][m] = tmp2m;
                    tmp[3][m] = tmp3m;

                    r0 += w;
                }

                float* r0_tm_0 = (float*)img0_tm + (i * w_tiles + j);
                float* r0_tm_1 = r0_tm_0 + tiles;
                float* r0_tm_2 = r0_tm_0 + tiles * 2;
                float* r0_tm_3 = r0_tm_0 + tiles * 3;

                for (int m = 0; m < 4; m++)
                {
                    float tmp00 = tmp[m][0];
                    float tmp01 = tmp[m][1];
                    float tmp02 = tmp[m][2];
                    float tmp03 = tmp[m][3];

                    float r0tm0 = tmp00 - tmp02;
                    float r0tm1 = tmp01 + tmp02;
                    float r0tm2 = tmp02 - tmp01;
                    float r0tm3 = tmp03 - tmp01;

                    r0_tm_0[0] = r0tm0;
                    r0_tm_1[0] = r0tm1;
                    r0_tm_2[0] = r0tm2;
                    r0_tm_3[0] = r0tm3;

                    r0_tm_0 += tiles * 4;
                    r0_tm_1 += tiles * 4;
                    r0_tm_2 += tiles * 4;
                    r0_tm_3 += tiles * 4;
                }
            }
        }
    }
}

static void conv3x3s1_winograd23_transform_output_rvv(const Mat& top_blob_tm, Mat& top_blob, const Mat& bias, const Option& opt)
{
    const int outw = top_blob.w;
    const int outh = top_blob.h;
    const int outch = top_blob.c;

    const int w_tiles = outw / 2;
    const int h_tiles = outh / 2;
    const int tiles = w_tiles * h_tiles;

    const float* biasptr = bias;

    // const float otm[2][4] = {
    //     {1.0f,  1.0f,  1.0f,  0.0f},
    //     {0.0f,  1.0f, -1.0f,  1.0f}
    // };

    // 0 = r00 + r01 + r02
    // 1 = r01 - r02 + r03

    #pragma omp parallel for num_threads(opt.num_threads)
    for (int p = 0; p < outch; p++)
    {
        const Mat out0_tm = top_blob_tm.channel(p);
        Mat out0 = top_blob.channel(p);

        float bias0 = biasptr ? biasptr[p] : 0.f;

        float tmp[2][4];

        // tile
        for (int i = 0; i < h_tiles; i++)
        {
            for (int j = 0; j < w_tiles; j++)
            {
                const float* output0_tm_0 = (const float*)out0_tm + (i * w_tiles + j);
                const float* output0_tm_1 = output0_tm_0 + tiles;
                const float* output0_tm_2 = output0_tm_0 + tiles * 2;
                const float* output0_tm_3 = output0_tm_0 + tiles * 3;

                float* output0 = out0.row(i * 2) + (j * 2);

                for (int m = 0; m < 4; m++)
                {
                    float out0tm0 = output0_tm_0[0];
                    float out0tm1 = output0_tm_1[0];
                    float out0tm2 = output0_tm_2[0];
                    float out0tm3 = output0_tm_3[0];

                    float tmp0m = out0tm0 + out0tm1 + out0tm2;
                    float tmp1m = out0tm1 - out0tm2 + out0tm3;

                    tmp[0][m] = tmp0m;
                    tmp[1][m] = tmp1m;

                    output0_tm_0 += tiles * 4;
                    output0_tm_1 += tiles * 4;
                    output0_tm_2 += tiles * 4;
                    output0_tm_3 += tiles * 4;
                }

                for (int m = 0; m < 2; m++)
                {
                    float tmp00 = tmp[m][0];
                    float tmp01 = tmp[m][1];
                    float tmp02 = tmp[m][2];
                    float tmp03 = tmp[m][3];

                    float out00 = bias0 + tmp00 + tmp01 + tmp02;
                    float out01 = bias0 + tmp01 - tmp02 + tmp03;

                    output0[0] = out00;
                    output0[1] = out01;

                    output0 += outw;
                }
            }
        }
    }
}