File size: 7,254 Bytes
25ade36 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 | #include "edge-impulse-sdk/classifier/ei_classifier_config.h"
#if EI_CLASSIFIER_TFLITE_LOAD_CMSIS_NN_SOURCES
/*
* Copyright (C) 2010-2022 Arm Limited or its affiliates. All rights reserved.
*
* SPDX-License-Identifier: Apache-2.0
*
* Licensed under the Apache License, Version 2.0 (the License); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an AS IS BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/* ----------------------------------------------------------------------
* Project: CMSIS NN Library
* Title: arm_softmax_s8.c
* Description: S8 softmax function
*
* $Date: 9 March 2022
* $Revision: V.2.1.0
*
* Target Processor: Cortex-M cores
*
* -------------------------------------------------------------------- */
#include "edge-impulse-sdk/CMSIS/NN/Include/arm_nnfunctions.h"
#include "edge-impulse-sdk/CMSIS/NN/Include/arm_nnsupportfunctions.h"
#define ACCUM_BITS 12
#ifdef ARM_MATH_MVEI
static int32x4_t arm_exp_on_negative_values_mve_32x4(int32x4_t val)
{
#define SHIFT_START (24)
int32_t shift = SHIFT_START;
int32x4_t mask;
const int32x4_t val_mod_minus_quarter =
vandq_s32(val, vdupq_n_s32((1 << SHIFT_START) - 1)) - vdupq_n_s32(1 << SHIFT_START);
const int32x4_t remainder = vsubq_s32(val_mod_minus_quarter, val);
const int32x4_t x = vaddq_n_s32(val_mod_minus_quarter << 5, 1 << 28);
const int32x4_t x2 = MUL_SAT_MVE(x, x);
const int32x4_t op_1 = DIV_POW2_MVE(MUL_SAT_MVE(x2, x2), 2) + MUL_SAT_MVE(x2, x);
const int32x4_t op_2 = x + DIV_POW2_MVE(MUL_SAT_MVE(op_1, vdupq_n_s32(715827883)) + x2, 1);
int32x4_t result = vdupq_n_s32(1895147668) + MUL_SAT_MVE(vdupq_n_s32(1895147668), op_2);
#define SELECT_IF_NON_ZERO(x) \
{ \
mve_pred16_t p = vcmpneq_n_s32(remainder & vdupq_n_s32(1 << shift++), 0); \
mask = vmvnq_m_s32(vdupq_n_s32(0), vdupq_n_s32(0), p); \
result = SELECT_USING_MASK(mask, MUL_SAT_MVE(result, vdupq_n_s32(x)), result); \
}
SELECT_IF_NON_ZERO(1672461947)
SELECT_IF_NON_ZERO(1302514674)
SELECT_IF_NON_ZERO(790015084)
SELECT_IF_NON_ZERO(290630308)
SELECT_IF_NON_ZERO(39332535)
SELECT_IF_NON_ZERO(720401)
SELECT_IF_NON_ZERO(242)
#undef SELECT_IF_NON_ZERO
mve_pred16_t p = vcmpeqq_n_s32(val, 0);
mask = vmvnq_m_s32(vdupq_n_s32(0), vdupq_n_s32(0), p);
result = SELECT_USING_MASK(mask, vdupq_n_s32(NN_Q31_MAX), result);
return result;
}
#endif
/**
* @ingroup groupNN
*/
/**
* @addtogroup Softmax
* @{
*/
void arm_softmax_s8(const int8_t *input,
const int32_t num_rows,
const int32_t row_size,
const int32_t mult,
const int32_t shift,
const int32_t diff_min,
int8_t *output)
{
#ifdef ARM_MATH_MVEI
#define ACT_MIN ((int8_t)NN_Q7_MIN)
#define ACT_MAX ((int8_t)NN_Q7_MAX)
const int32_t mask = (1 << shift);
for (int i_num_rows = 0; i_num_rows < num_rows; ++i_num_rows)
{
int8_t max = ACT_MIN;
int32_t vec_count = (row_size + 15) / 16;
uint32_t r_count = (uint32_t)row_size;
for (int i = 0; i < vec_count; i++)
{
mve_pred16_t p = vctp8q(r_count);
const int8x16_t ip = vldrbq_z_s8(&input[i * 16], p);
max = vmaxvq_p_s8(max, ip, p);
r_count -= 16;
}
vec_count = row_size / 4;
int32_t idx = 0;
int32_t sum = 0;
while (vec_count)
{
int32x4_t ip = vldrbq_s32(&input[idx * 4]);
ip = vsubq_n_s32(ip, max);
mve_pred16_t p = vcmpgeq_n_s32(ip, diff_min);
if (p != 0)
{
ip = vmulq_n_s32(ip, mask);
int32x4_t res = MUL_SAT_MVE(ip, vdupq_n_s32(mult));
res = arm_exp_on_negative_values_mve_32x4(res);
res = DIV_POW2_MVE(res, ACCUM_BITS);
res = vpselq_s32(res, vdupq_n_s32(0), p);
sum += vaddvq_s32(res);
}
vec_count--;
idx++;
}
const int32_t tail_idx = row_size & ~3;
for (int i = 0; i < (row_size & 3); i++)
{
const int32_t diff = input[tail_idx + i] - max;
if (diff >= diff_min)
{
sum += DIV_POW2(EXP_ON_NEG(MUL_SAT(diff * mask, mult)), ACCUM_BITS);
}
}
const int32_t headroom = __CLZ((uint32_t)sum);
const int32_t bits_over_unit = ACCUM_BITS - headroom + 23;
const int32_t shifted_scale = ONE_OVER1((sum > 0 ? sum << headroom : 0) - (1 << 31));
vec_count = row_size / 4;
idx = 0;
while (vec_count)
{
int32x4_t ip = vldrbq_s32(&input[idx]);
ip = vsubq_n_s32(ip, max);
mve_pred16_t p = vcmpgeq_n_s32(ip, diff_min);
int32x4_t tmp_res;
if (p != 0)
{
ip = vmulq_n_s32(ip, mask);
tmp_res = MUL_SAT_MVE(ip, vdupq_n_s32(mult));
tmp_res = arm_exp_on_negative_values_mve_32x4(tmp_res);
tmp_res = MUL_SAT_MVE(vdupq_n_s32(shifted_scale), tmp_res);
tmp_res = DIV_POW2_MVE(tmp_res, bits_over_unit);
tmp_res += vdupq_n_s32(ACT_MIN);
tmp_res = vmaxq_s32(tmp_res, vdupq_n_s32(ACT_MIN));
tmp_res = vminq_s32(tmp_res, vdupq_n_s32(ACT_MAX));
tmp_res = vpselq_s32(tmp_res, vdupq_n_s32(ACT_MIN), p);
}
else
{
tmp_res = vdupq_n_s32(ACT_MIN);
}
vstrbq_s32(&output[idx], tmp_res);
vec_count--;
idx += 4;
}
for (int i = 0; i < (row_size & 3); i++)
{
int32_t diff = input[tail_idx + i] - max;
if (diff >= diff_min)
{
const int32_t res =
DIV_POW2(MUL_SAT(shifted_scale, EXP_ON_NEG(MUL_SAT(diff * mask, mult))), bits_over_unit) +
NN_Q7_MIN;
output[tail_idx + i] = (int8_t)CLAMP(res, (int32_t)ACT_MAX, (int32_t)ACT_MIN);
}
else
{
output[tail_idx + i] = ACT_MIN;
}
}
input += row_size;
output += row_size;
}
#else
arm_nn_softmax_common_s8(input, num_rows, row_size, mult, shift, diff_min, false, (void *)output);
#endif
}
/**
* @} end of Softmax group
*/
#endif // EI_CLASSIFIER_TFLITE_LOAD_CMSIS_NN_SOURCES
|