Aksal/nuttx-apps
1
0
Fork 1
mirror of https://github.com/apache/nuttx-apps.git synced 2026-08-01 20:29:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

ml: follow nxstyle

Browse source 
Signed-off-by: jihandong <jihandong@xiaomi.com>
This commit is contained in:
jihandong 2024-10-17 16:16:58 +08:00 • committed by Xiang Xiao
parent 7d87768f78
commit 25937282ed
4 changed files with 653 additions and 562 deletions
Download patch file Download diff file Expand all files Collapse all files

324
mlearning/tflite-micro/operators/neon/arm_convolve_s8.c
View file

@ -1,5 +1,8 @@
/*
* SPDX-FileCopyrightText: Copyright 2010-2023 Arm Limited and/or its affiliates <open-source-office@arm.com>
/****************************************************************************
* apps/mlearning/tflite-micro/operators/neon/arm_convolve_s8.c
*
* SPDX-FileCopyrightText: Copyright 2010-2023 Arm Limited and/or
* its affiliates <open-source-office@arm.com>
*
* SPDX-License-Identifier: Apache-2.0
*
@ -14,191 +17,210 @@
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
****************************************************************************/
/****************************************************************************
* Included Files
****************************************************************************/
#include <arm_neon.h>
#include "arm_nnfunctions.h"
#include "arm_nnsupportfunctions.h"
/**
* @ingroup Public
/****************************************************************************
* Public Functions
****************************************************************************/
/* Basic s8 convolution function.
*
* Refer header file for details. Optimal use case for the DSP/MVE
* implementation is when input and output channels are multiples of 4 or
* atleast greater than 4.
*/
/**
* @addtogroup NNConv
* @{
*/
/*
* Basic s8 convolution function.
*
* Refer header file for details. Optimal use case for the DSP/MVE implementation is when input and output channels
* are multiples of 4 or atleast greater than 4.
*
*/
arm_cmsis_nn_status arm_convolve_s8(const cmsis_nn_context *ctx,
const cmsis_nn_conv_params *conv_params,
const cmsis_nn_per_channel_quant_params *quant_params,
const cmsis_nn_dims *input_dims,
const int8_t *input_data,
const cmsis_nn_dims *filter_dims,
const int8_t *filter_data,
const cmsis_nn_dims *bias_dims,
const int32_t *bias_data,
const cmsis_nn_dims *output_dims,
int8_t *output_data)
arm_cmsis_nn_status
arm_convolve_s8(const cmsis_nn_context *ctx,
const cmsis_nn_conv_params *conv_params,
const cmsis_nn_per_channel_quant_params *quant_params,
const cmsis_nn_dims *input_dims,
const int8_t *input_data,
const cmsis_nn_dims *filter_dims,
const int8_t *filter_data,
const cmsis_nn_dims *bias_dims,
const int32_t *bias_data,
const cmsis_nn_dims *output_dims,
int8_t *output_data)
{
(void)bias_dims;
(void)bias_dims;
if (ctx->buf == NULL)
if (ctx->buf == NULL)
{
return ARM_CMSIS_NN_ARG_ERROR;
return ARM_CMSIS_NN_ARG_ERROR;
}
int16_t *buffer_a = (int16_t *)ctx->buf;
const int32_t input_batches = input_dims->n;
const uint16_t input_x = input_dims->w;
const uint16_t input_y = input_dims->h;
const uint16_t input_ch = input_dims->c;
const uint16_t kernel_x = filter_dims->w;
const uint16_t kernel_y = filter_dims->h;
const uint16_t output_x = output_dims->w;
const uint16_t output_y = output_dims->h;
const uint16_t output_ch = output_dims->c;
int16_t *buffer_a = (int16_t *)ctx->buf;
const uint16_t pad_x = conv_params->padding.w;
const uint16_t pad_y = conv_params->padding.h;
const uint16_t stride_x = conv_params->stride.w;
const uint16_t stride_y = conv_params->stride.h;
const int32_t dilation_x = conv_params->dilation.w;
const int32_t dilation_y = conv_params->dilation.h;
const int32_t out_offset = conv_params->output_offset;
const int32_t out_activation_min = conv_params->activation.min;
const int32_t out_activation_max = conv_params->activation.max;
const int32_t rhs_cols = kernel_x * kernel_y * input_ch;
const int32_t input_offset = conv_params->input_offset;
const int32_t input_batches = input_dims->n;
const uint16_t input_x = input_dims->w;
const uint16_t input_y = input_dims->h;
const uint16_t input_ch = input_dims->c;
const uint16_t kernel_x = filter_dims->w;
const uint16_t kernel_y = filter_dims->h;
const uint16_t output_x = output_dims->w;
const uint16_t output_y = output_dims->h;
const uint16_t output_ch = output_dims->c;
int32_t *output_mult = quant_params->multiplier;
int32_t *output_shift = quant_params->shift;
const uint16_t pad_x = conv_params->padding.w;
const uint16_t pad_y = conv_params->padding.h;
const uint16_t stride_x = conv_params->stride.w;
const uint16_t stride_y = conv_params->stride.h;
const int32_t dilation_x = conv_params->dilation.w;
const int32_t dilation_y = conv_params->dilation.h;
const int32_t out_offset = conv_params->output_offset;
const int32_t out_activation_min = conv_params->activation.min;
const int32_t out_activation_max = conv_params->activation.max;
const int32_t rhs_cols = kernel_x * kernel_y * input_ch;
const int32_t input_offset = conv_params->input_offset;
int i_batch;
for (i_batch = 0; i_batch < input_batches; i_batch++)
int32_t *output_mult = quant_params->multiplier;
int32_t *output_shift = quant_params->shift;
int i_batch;
for (i_batch = 0; i_batch < input_batches; i_batch++)
{
const int32_t remainder = rhs_cols % 4;
const int32_t aligned_rhs_cols = remainder != 0 ? rhs_cols + 4 - remainder : rhs_cols;
/**
* Use Im2col to speed up conv2d calculations.
* Use as a ping-pong buffer for unordered elements.
*/
int8_t *im2col_buf = (int8_t *)buffer_a + aligned_rhs_cols * 2;
int16_t *im2col_buf_start_s16 = buffer_a;
int8_t *out = output_data;
int32_t lhs_rows = 0;
/* This part implements the im2col function */
for (int i_out_x = 0; i_out_x < output_x; i_out_x++)
const int32_t remainder = rhs_cols % 4;
const int32_t aligned_rhs_cols = remainder != 0 ?
rhs_cols + 4 - remainder : rhs_cols;
/**
* Use Im2col to speed up conv2d calculations.
* Use as a ping-pong buffer for unordered elements.
*/
int8_t *im2col_buf = (int8_t *)buffer_a + aligned_rhs_cols * 2;
int16_t *im2col_buf_start_s16 = buffer_a;
int8_t *out = output_data;
int32_t lhs_rows = 0;
/* This part implements the im2col function */
for (int i_out_x = 0; i_out_x < output_x; i_out_x++)
{
const int32_t base_idx_x = stride_x * i_out_x - pad_x;
for (int i_out_y = 0; i_out_y < output_y; i_out_y++)
const int32_t base_idx_x = stride_x * i_out_x - pad_x;
for (int i_out_y = 0; i_out_y < output_y; i_out_y++)
{
const int32_t base_idx_y = stride_y * i_out_y - pad_y;
for (int32_t i_ker_x = 0; i_ker_x < kernel_x; i_ker_x++)
const int32_t base_idx_y = stride_y * i_out_y - pad_y;
for (int32_t i_ker_x = 0; i_ker_x < kernel_x; i_ker_x++)
{
int32_t k_x = base_idx_x + dilation_x * i_ker_x;
int32_t k_y = base_idx_y - dilation_y;
for (int32_t i_ker_y = 0; i_ker_y < kernel_y; i_ker_y++)
int32_t k_x = base_idx_x + dilation_x * i_ker_x;
int32_t k_y = base_idx_y - dilation_y;
for (int32_t i_ker_y = 0; i_ker_y < kernel_y; i_ker_y++)
{
k_y += dilation_y;
arm_memcpy_s8(im2col_buf,
input_data + (k_y * input_x + k_x) * input_ch,
input_ch);
im2col_buf += input_ch;
k_y += dilation_y;
arm_memcpy_s8(im2col_buf,
input_data + (k_y * input_x + k_x) * input_ch,
input_ch);
im2col_buf += input_ch;
}
}
lhs_rows++;
/* Extend the input data from int8 to int16, and add offset. */
arm_q7_to_q15_with_offset(im2col_buf - rhs_cols,
im2col_buf_start_s16,
rhs_cols,
(int16_t)input_offset);
im2col_buf_start_s16 += aligned_rhs_cols;
if (lhs_rows & 2)
lhs_rows++;
/* Extend the input data from int8 to int16, and add offset. */
arm_q7_to_q15_with_offset(im2col_buf - rhs_cols,
im2col_buf_start_s16,
rhs_cols,
(int16_t)input_offset);
im2col_buf_start_s16 += aligned_rhs_cols;
if (lhs_rows & 2)
{
out = arm_nn_mat_mult_kernel_s8_s16(filter_data,
buffer_a,
output_ch,
output_shift,
output_mult,
out_offset,
out_activation_min,
out_activation_max,
rhs_cols,
aligned_rhs_cols,
bias_data,
out);
/* counter reset */
im2col_buf_start_s16 = buffer_a;
im2col_buf = (int8_t *)buffer_a + (aligned_rhs_cols << 1);
lhs_rows = 0;
out = arm_nn_mat_mult_kernel_s8_s16(filter_data,
buffer_a,
output_ch,
output_shift,
output_mult,
out_offset,
out_activation_min,
out_activation_max,
rhs_cols,
aligned_rhs_cols,
bias_data,
out);
/* counter reset */
im2col_buf_start_s16 = buffer_a;
im2col_buf = (int8_t *)buffer_a + (aligned_rhs_cols << 1);
lhs_rows = 0;
}
}
}
if (lhs_rows != 0)
if (lhs_rows != 0)
{
const int8_t *ker_a = filter_data;
int i;
for (i = 0; i < output_ch; i++)
const int8_t *ker_a = filter_data;
int i;
for (i = 0; i < output_ch; i++)
{
/* Load the accumulator with bias first */
uint16_t col_count = rhs_cols / 8;
int32_t sum = 0;
const int16_t *ip_as_col = buffer_a;
int32x4_t res_s32 = vdupq_n_s32(0);
if (bias_data)
/* Load the accumulator with bias first */
uint16_t col_count = rhs_cols / 8;
int32_t sum = 0;
const int16_t *ip_as_col = buffer_a;
int32x4_t res_s32 = vdupq_n_s32(0);
if (bias_data)
{
sum = bias_data[i];
sum = bias_data[i];
}
while (col_count)
while (col_count)
{
int8x8_t filter_s8 = vld1_s8(ker_a);
int16x8_t input_s16 = vld1q_s16(ip_as_col);
int16x8_t filter_s16 = vmovl_s8(filter_s8);
ker_a += 8;
ip_as_col += 8;
res_s32 = vmlal_s16(res_s32,
vget_low_s16(input_s16),
vget_low_s16(filter_s16));
res_s32 = vmlal_s16(res_s32,
vget_high_s16(input_s16),
vget_high_s16(filter_s16));
col_count --;
int8x8_t filter_s8 = vld1_s8(ker_a);
int16x8_t input_s16 = vld1q_s16(ip_as_col);
int16x8_t filter_s16 = vmovl_s8(filter_s8);
ker_a += 8;
ip_as_col += 8;
res_s32 = vmlal_s16(res_s32,
vget_low_s16(input_s16),
vget_low_s16(filter_s16));
res_s32 = vmlal_s16(res_s32,
vget_high_s16(input_s16),
vget_high_s16(filter_s16));
col_count--;
}
sum += vgetq_lane_s32(res_s32, 0);
sum += vgetq_lane_s32(res_s32, 1);
sum += vgetq_lane_s32(res_s32, 2);
sum += vgetq_lane_s32(res_s32, 3);
col_count = rhs_cols % 8;
while (col_count)
sum += vgetq_lane_s32(res_s32, 0);
sum += vgetq_lane_s32(res_s32, 1);
sum += vgetq_lane_s32(res_s32, 2);
sum += vgetq_lane_s32(res_s32, 3);
col_count = rhs_cols % 8;
while (col_count)
{
int8_t ker_a1 = *ker_a++;
int16_t ip_b1 = *ip_as_col++;
sum += ker_a1 * ip_b1;
col_count--;
int8_t ker_a1 = *ker_a++;
int16_t ip_b1 = *ip_as_col++;
sum += ker_a1 * ip_b1;
col_count--;
}
sum = arm_nn_requantize(sum, output_mult[i], output_shift[i]);
sum += out_offset;
sum = MAX(sum, out_activation_min);
sum = MIN(sum, out_activation_max);
*out++ = (int8_t)sum;
sum = arm_nn_requantize(sum,
output_mult[i], output_shift[i]);
sum += out_offset;
sum = MAX(sum, out_activation_min);
sum = MIN(sum, out_activation_max);
*out++ = (int8_t)sum;
}
}
/* Advance to the next batch */
input_data += (input_x * input_y * input_ch);
output_data += (output_x * output_y * output_ch);
/* Advance to the next batch */
input_data += (input_x * input_y * input_ch);
output_data += (output_x * output_y * output_ch);
}
/* Return to application */
return ARM_CMSIS_NN_SUCCESS;
/* Return to application */
return ARM_CMSIS_NN_SUCCESS;
}
/**
* @} end of NNConv group
*/

234
mlearning/tflite-micro/operators/neon/arm_elementwise_add_s8.c
View file

@ -1,5 +1,8 @@
/*
* SPDX-FileCopyrightText: Copyright 2010-2023 Arm Limited and/or its affiliates <open-source-office@arm.com>
/****************************************************************************
* apps/mlearning/tflite-micro/operators/neon/arm_elementwise_add_s8.c
*
* SPDX-FileCopyrightText: Copyright 2010-2023 Arm Limited and/or
* its affiliates <open-source-office@arm.com>
*
* SPDX-License-Identifier: Apache-2.0
*
@ -14,136 +17,157 @@
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
****************************************************************************/
/****************************************************************************
* Included Files
****************************************************************************/
#include <arm_neon.h>
#include "arm_nnfunctions.h"
#include "arm_nnsupportfunctions.h"
/****************************************************************************
* Public Functions
****************************************************************************/
/* Note: __SHIFT is expected to be <=0 */
__STATIC_FORCEINLINE int32x4_t arm_requantize_neon(const int32x4_t val, const int32_t multiplier, const int32_t shift)
__STATIC_FORCEINLINE int32x4_t
arm_requantize_neon(const int32x4_t val,
const int32_t multiplier,
const int32_t shift)
{
int32x4_t dividend = vqrdmulhq_n_s32(vshlq_s32(val, vdupq_n_s32(LEFT_SHIFT(shift))), multiplier);
int32_t exponent = RIGHT_SHIFT(shift);
int32x4_t shift__ = vdupq_n_s32(-exponent);
int32x4_t fixup__ = vshrq_n_s32(vandq_s32(dividend, shift__), 31);
int32x4_t fixed_up_dividend = vqaddq_s32(dividend, fixup__);
return vrshlq_s32(fixed_up_dividend, shift__);
int32x4_t dividend = vqrdmulhq_n_s32(
vshlq_s32(val, vdupq_n_s32(LEFT_SHIFT(shift))), multiplier);
int32_t exponent = RIGHT_SHIFT(shift);
int32x4_t shift__ = vdupq_n_s32(-exponent);
int32x4_t fixup__ = vshrq_n_s32(vandq_s32(dividend, shift__), 31);
int32x4_t fixed_up_dividend = vqaddq_s32(dividend, fixup__);
return vrshlq_s32(fixed_up_dividend, shift__);
}
/**
* @ingroup Public
*/
/**
* @addtogroup groupElementwise
* @{
*/
/*
* s8 elementwise add
*
* Refer header file for details.
*
*/
arm_cmsis_nn_status arm_elementwise_add_s8(const int8_t *input_1_vect,
const int8_t *input_2_vect,
const int32_t input_1_offset,
const int32_t input_1_mult,
const int32_t input_1_shift,
const int32_t input_2_offset,
const int32_t input_2_mult,
const int32_t input_2_shift,
const int32_t left_shift,
int8_t *output,
const int32_t out_offset,
const int32_t out_mult,
const int32_t out_shift,
const int32_t out_activation_min,
const int32_t out_activation_max,
const int32_t block_size)
arm_cmsis_nn_status
arm_elementwise_add_s8(const int8_t *input_1_vect,
const int8_t *input_2_vect,
const int32_t input_1_offset,
const int32_t input_1_mult,
const int32_t input_1_shift,
const int32_t input_2_offset,
const int32_t input_2_mult,
const int32_t input_2_shift,
const int32_t left_shift,
int8_t *output,
const int32_t out_offset,
const int32_t out_mult,
const int32_t out_shift,
const int32_t out_activation_min,
const int32_t out_activation_max,
const int32_t block_size)
{
int32_t loop_count = block_size / 8;
const int8_t *input_1 = input_1_vect;
const int8_t *input_2 = input_2_vect;
int8_t *output_ = output;
int32_t loop_count = block_size / 8;
const int8_t *input_1 = input_1_vect;
const int8_t *input_2 = input_2_vect;
int8_t *output_ = output;
while (loop_count)
while (loop_count)
{
int8x8_t res;
int8x8_t input_1_s8;
int8x8_t input_2_s8;
int16x8_t i1_val_16;
int16x8_t input_1_s16;
int16x8_t input_2_s16;
int32x4_t input_1_s16_low;
int32x4_t input_1_s16_high;
int32x4_t input_2_s16_low;
int32x4_t input_2_s16_high;
int8x8_t res;
int8x8_t input_1_s8;
int8x8_t input_2_s8;
int16x8_t i1_val_16;
int16x8_t input_1_s16;
int16x8_t input_2_s16;
int32x4_t input_1_s16_low;
int32x4_t input_1_s16_high;
int32x4_t input_2_s16_low;
int32x4_t input_2_s16_high;
input_1_s8 = vld1_s8(input_1);
input_2_s8 = vld1_s8(input_2);
input_1_s16 = vmovl_s8(input_1_s8);
input_2_s16 = vmovl_s8(input_2_s8);
input_1 += 8;
input_2 += 8;
input_1_s8 = vld1_s8(input_1);
input_2_s8 = vld1_s8(input_2);
input_1_s16 = vmovl_s8(input_1_s8);
input_2_s16 = vmovl_s8(input_2_s8);
input_1 += 8;
input_2 += 8;
input_1_s16_low = vaddw_s16(vdupq_n_s32(input_1_offset), vget_low_s16(input_1_s16));
input_1_s16_high = vaddw_s16(vdupq_n_s32(input_1_offset), vget_high_s16(input_1_s16));
input_2_s16_low = vaddw_s16(vdupq_n_s32(input_2_offset), vget_low_s16(input_2_s16));
input_2_s16_high = vaddw_s16(vdupq_n_s32(input_2_offset), vget_high_s16(input_2_s16));
input_1_s16_low = vaddw_s16(
vdupq_n_s32(input_1_offset), vget_low_s16(input_1_s16));
input_1_s16_high = vaddw_s16(
vdupq_n_s32(input_1_offset), vget_high_s16(input_1_s16));
input_2_s16_low = vaddw_s16(
vdupq_n_s32(input_2_offset), vget_low_s16(input_2_s16));
input_2_s16_high = vaddw_s16(
vdupq_n_s32(input_2_offset), vget_high_s16(input_2_s16));
input_1_s16_low = vshlq_s32(input_1_s16_low, vdupq_n_s32(left_shift));
input_2_s16_low = vshlq_s32(input_2_s16_low, vdupq_n_s32(left_shift));
input_1_s16_high = vshlq_s32(input_1_s16_high, vdupq_n_s32(left_shift));
input_2_s16_high = vshlq_s32(input_2_s16_high, vdupq_n_s32(left_shift));
input_1_s16_low = vshlq_s32(
input_1_s16_low, vdupq_n_s32(left_shift));
input_2_s16_low = vshlq_s32(
input_2_s16_low, vdupq_n_s32(left_shift));
input_1_s16_high = vshlq_s32(
input_1_s16_high, vdupq_n_s32(left_shift));
input_2_s16_high = vshlq_s32(
input_2_s16_high, vdupq_n_s32(left_shift));
input_1_s16_low = arm_requantize_neon(input_1_s16_low, input_1_mult, input_1_shift);
input_1_s16_high = arm_requantize_neon(input_1_s16_high, input_1_mult, input_1_shift);
input_2_s16_low = arm_requantize_neon(input_2_s16_low, input_2_mult, input_2_shift);
input_2_s16_high = arm_requantize_neon(input_2_s16_high, input_2_mult, input_2_shift);
input_1_s16_low = arm_requantize_neon(
input_1_s16_low, input_1_mult, input_1_shift);
input_1_s16_high = arm_requantize_neon(
input_1_s16_high, input_1_mult, input_1_shift);
input_2_s16_low = arm_requantize_neon(
input_2_s16_low, input_2_mult, input_2_shift);
input_2_s16_high = arm_requantize_neon(
input_2_s16_high, input_2_mult, input_2_shift);
input_1_s16_low = vaddq_s32(input_1_s16_low, input_2_s16_low);
input_1_s16_high = vaddq_s32(input_1_s16_high, input_2_s16_high);
input_1_s16_low = vaddq_s32(
input_1_s16_low, input_2_s16_low);
input_1_s16_high = vaddq_s32(
input_1_s16_high, input_2_s16_high);
input_1_s16_low = arm_requantize_neon(input_1_s16_low, out_mult, out_shift);
input_1_s16_high = arm_requantize_neon(input_1_s16_high, out_mult, out_shift);
input_1_s16_low = arm_requantize_neon(
input_1_s16_low, out_mult, out_shift);
input_1_s16_high = arm_requantize_neon(
input_1_s16_high, out_mult, out_shift);
input_1_s16_low = vaddq_s32(input_1_s16_low, vdupq_n_s32(out_offset));
input_1_s16_high = vaddq_s32(input_1_s16_high, vdupq_n_s32(out_offset));
input_1_s16_low = vaddq_s32(
input_1_s16_low, vdupq_n_s32(out_offset));
input_1_s16_high = vaddq_s32(
input_1_s16_high, vdupq_n_s32(out_offset));
input_1_s16_low = vmaxq_s32(input_1_s16_low, vdupq_n_s32(out_activation_min));
input_1_s16_high = vmaxq_s32(input_1_s16_high, vdupq_n_s32(out_activation_min));
input_1_s16_low = vminq_s32(input_1_s16_low, vdupq_n_s32(out_activation_max));
input_1_s16_high = vminq_s32(input_1_s16_high, vdupq_n_s32(out_activation_max));
input_1_s16_low = vmaxq_s32(
input_1_s16_low, vdupq_n_s32(out_activation_min));
input_1_s16_high = vmaxq_s32(
input_1_s16_high, vdupq_n_s32(out_activation_min));
input_1_s16_low = vminq_s32(
input_1_s16_low, vdupq_n_s32(out_activation_max));
input_1_s16_high = vminq_s32(
input_1_s16_high, vdupq_n_s32(out_activation_max));
i1_val_16 = vcombine_s16(vmovn_s32(input_1_s16_low), vmovn_s32(input_1_s16_high));
res = vmovn_s16(i1_val_16);
i1_val_16 = vcombine_s16(
vmovn_s32(input_1_s16_low), vmovn_s32(input_1_s16_high));
res = vmovn_s16(i1_val_16);
vst1_s8(output_, res);
output_ += 8;
loop_count--;
vst1_s8(output_, res);
output_ += 8;
loop_count--;
}
loop_count = block_size % 8;
while (loop_count)
loop_count = block_size % 8;
while (loop_count)
{
int32_t a1 = (*input_1++ + input_1_offset) << left_shift;
int32_t a2 = (*input_2++ + input_2_offset) << left_shift;
a1 = arm_nn_requantize(a1, input_1_mult, input_1_shift);
a2 = arm_nn_requantize(a2, input_2_mult, input_2_shift);
int32_t a1 = (*input_1++ + input_1_offset) << left_shift;
int32_t a2 = (*input_2++ + input_2_offset) << left_shift;
a1 = arm_nn_requantize(a1, input_1_mult, input_1_shift);
a2 = arm_nn_requantize(a2, input_2_mult, input_2_shift);
int32_t sum = a1 + a2;
sum = arm_nn_requantize(sum, out_mult, out_shift);
sum += out_offset;
int32_t sum = a1 + a2;
sum = arm_nn_requantize(sum, out_mult, out_shift);
sum += out_offset;
sum = MAX(sum, out_activation_min);
sum = MIN(sum, out_activation_max);
*output_ = (int8_t) sum;
loop_count--;
output_++;
sum = MAX(sum, out_activation_min);
sum = MIN(sum, out_activation_max);
*output_ = (int8_t) sum;
loop_count--;
output_++;
}
return (ARM_CMSIS_NN_SUCCESS);
return (ARM_CMSIS_NN_SUCCESS);
}

587
mlearning/tflite-micro/operators/neon/arm_nn_mat_mult_kernel_s8_s16.c
View file

@ -1,5 +1,8 @@
/*
* SPDX-FileCopyrightText: Copyright 2010-2023 Arm Limited and/or its affiliates <open-source-office@arm.com>
/****************************************************************************
* apps/mlearning/tflite-micro/operators/neon/arm_nn_mat_mult_kernel_s8_s16.c
*
* SPDX-FileCopyrightText: Copyright 2010-2023 Arm Limited and/or
* its affiliates <open-source-office@arm.com>
*
* SPDX-License-Identifier: Apache-2.0
*
@ -14,9 +17,9 @@
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
****************************************************************************/
/* ----------------------------------------------------------------------
/****************************************************************************
* Project: CMSIS NN Library
* Title: arm_nn_mat_mult_kernel_s8_s16.c
* Description: Matrix-multiplication function for convolution
@ -25,18 +28,24 @@
* $Revision: V.2.0.0
*
* Target : Arm(R) M-Profile Architecture
* -------------------------------------------------------------------- */
****************************************************************************/
/****************************************************************************
* Included Files
****************************************************************************/
#include <arm_neon.h>
#include "arm_nnfunctions.h"
#include "arm_nnsupportfunctions.h"
/*
* Matrix-multiplication function for convolution with per-channel requantization.
*
* Refer header file for details.
*
/* Matrix-multiplication function for convolution with per-channel
* requantization. Refer header file for details.
*/
/****************************************************************************
* Public Functions
****************************************************************************/
int8_t *arm_nn_mat_mult_kernel_s8_s16(const int8_t *input_a,
const int16_t *input_b,
const uint16_t output_ch,
@ -50,318 +59,352 @@ int8_t *arm_nn_mat_mult_kernel_s8_s16(const int8_t *input_a,
const int32_t *const output_bias,
int8_t *out_0)
{
int8_t *out_1 = out_0 + output_ch;
const int32_t *bias = output_bias;
int8_t *out_1 = out_0 + output_ch;
const int32_t *bias = output_bias;
uint16_t row_count = output_ch / 4;
const int8_t *ip_a0 = input_a;
uint16_t row_count = output_ch / 4;
const int8_t *ip_a0 = input_a;
/* this loop over rows in A */
while (row_count)
/* this loop over rows in A */
while (row_count)
{
int32_t col_count = num_col_a / 8;
const int16_t *ip_b0 = input_b;
const int16_t *ip_b1 = ip_b0 + aligned_num_col_a;
const int8_t *ip_a[4] = {ip_a0,
ip_a0 + num_col_a,
ip_a0 + 2 * num_col_a,
ip_a0 + 3 * num_col_a};
int32_t ch_out[4][2] = {0};
int32x4_t res[8];
for (int i = 0; i < 8; i++)
int32_t col_count = num_col_a / 8;
const int16_t *ip_b0 = input_b;
const int16_t *ip_b1 = ip_b0 + aligned_num_col_a;
const int8_t *ip_a[4] =
{
res[i] = vdupq_n_s32(0);
ip_a0,
ip_a0 + num_col_a,
ip_a0 + 2 * num_col_a,
ip_a0 + 3 * num_col_a
};
int32_t ch_out[4][2] =
{
0
};
int32x4_t res[8];
for (int i = 0; i < 8; i++)
{
res[i] = vdupq_n_s32(0);
}
/* Init accumulator with bias for channel N and N + 1 */
if (bias)
/* Init accumulator with bias for channel N and N + 1 */
if (bias)
{
for (int i = 0; i < 4; i++)
for (int i = 0; i < 4; i++)
{
ch_out[i][0] = *bias;
ch_out[i][1] = *bias++;
ch_out[i][0] = *bias;
ch_out[i][1] = *bias++;
}
}
/**
* Each time eight int8 data of four filters and eight int16 data
* of two inputs are read.First, the filter data is expanded to
* int16, and then cross-multiplied to obtain eight calculation results.
*/
while (col_count)
/* Each time eight int8 data of four filters and eight int16 data
* of two inputs are read.First, the filter data is expanded to
* int16, and then cross-multiplied to obtain eight
* calculation results.
*/
while (col_count)
{
int8x8_t filter_s8[4];
int16x8_t input_s16[2];
int16x8_t filter_s16[4];
int8x8_t filter_s8[4];
int16x8_t input_s16[2];
int16x8_t filter_s16[4];
input_s16[0] = vld1q_s16(ip_b0);
ip_b0 += 8;
input_s16[1] = vld1q_s16(ip_b1);
ip_b1 += 8;
input_s16[0] = vld1q_s16(ip_b0);
ip_b0 += 8;
input_s16[1] = vld1q_s16(ip_b1);
ip_b1 += 8;
for (int i = 0; i < 4; i++)
for (int i = 0; i < 4; i++)
{
filter_s8[i] = vld1_s8(ip_a[i]);
ip_a[i] += 8;
filter_s16[i] = vmovl_s8(filter_s8[i]);
res[i * 2] = vmlal_s16(res[i * 2],
vget_low_s16(filter_s16[i]),
vget_low_s16(input_s16[0]));
res[i * 2 + 1] = vmlal_s16(res[i * 2 + 1],
vget_low_s16(filter_s16[i]),
vget_low_s16(input_s16[1]));
res[i * 2] = vmlal_s16(res[i * 2],
vget_high_s16(filter_s16[i]),
vget_high_s16(input_s16[0]));
res[i * 2 + 1] = vmlal_s16(res[i * 2 + 1],
vget_high_s16(filter_s16[i]),
vget_high_s16(input_s16[1]));
filter_s8[i] = vld1_s8(ip_a[i]);
ip_a[i] += 8;
filter_s16[i] = vmovl_s8(filter_s8[i]);
res[i * 2] = vmlal_s16(res[i * 2],
vget_low_s16(filter_s16[i]),
vget_low_s16(input_s16[0]));
res[i * 2 + 1] = vmlal_s16(res[i * 2 + 1],
vget_low_s16(filter_s16[i]),
vget_low_s16(input_s16[1]));
res[i * 2] = vmlal_s16(res[i * 2],
vget_high_s16(filter_s16[i]),
vget_high_s16(input_s16[0]));
res[i * 2 + 1] = vmlal_s16(res[i * 2 + 1],
vget_high_s16(filter_s16[i]),
vget_high_s16(input_s16[1]));
}
col_count --;
col_count--;
}
for (int i = 0; i < 4; i++)
for (int i = 0; i < 4; i++)
{
for (int j = 0; j < 2; j++)
for (int j = 0; j < 2; j++)
{
ch_out[i][j] += vgetq_lane_s32(res[i * 2 + j], 0);
ch_out[i][j] += vgetq_lane_s32(res[i * 2 + j], 1);
ch_out[i][j] += vgetq_lane_s32(res[i * 2 + j], 2);
ch_out[i][j] += vgetq_lane_s32(res[i * 2 + j], 3);
ch_out[i][j] += vgetq_lane_s32(res[i * 2 + j], 0);
ch_out[i][j] += vgetq_lane_s32(res[i * 2 + j], 1);
ch_out[i][j] += vgetq_lane_s32(res[i * 2 + j], 2);
ch_out[i][j] += vgetq_lane_s32(res[i * 2 + j], 3);
}
}
col_count = num_col_a % 8;
while (col_count)
col_count = num_col_a % 8;
while (col_count) /* while over col_count */
{
int16_t b0 = *ip_b0++;
int16_t b1 = *ip_b1++;
int16_t b0 = *ip_b0++;
int16_t b1 = *ip_b1++;
for (int i = 0; i < 4; i++)
for (int i = 0; i < 4; i++)
{
int8_t input_remaining = *(ip_a[i]++);
ch_out[i][0] += input_remaining * b0;
ch_out[i][1] += input_remaining * b1;
}
col_count--;
} /* while over col_count */
for (int i = 0; i < 4; i++)
{
ch_out[i][0] = arm_nn_requantize(ch_out[i][0], *out_mult, *out_shift);
ch_out[i][1] = arm_nn_requantize(ch_out[i][1], *out_mult, *out_shift);
ch_out[i][0] += out_offset;
ch_out[i][1] += out_offset;
ch_out[i][0] = MAX(ch_out[i][0], activation_min);
ch_out[i][1] = MAX(ch_out[i][1], activation_min);
ch_out[i][0] = MIN(ch_out[i][0], activation_max);
ch_out[i][1] = MIN(ch_out[i][1], activation_max);
*out_0++ = (int8_t)ch_out[i][0];
*out_1++ = (int8_t)ch_out[i][1];
out_mult++;
out_shift++;
}
/* skip row */
ip_a0 = ip_a[3];
row_count--;
}
row_count = output_ch % 4;
if (row_count >= 2)
{
int32_t col_count = num_col_a / 8;
const int8_t *ip_a1 = ip_a0 + num_col_a;
const int16_t *ip_b0 = input_b;
const int16_t *ip_b1 = ip_b0 + aligned_num_col_a;
int32_t ch_out[2][2] = {0};
int32x4_t res[4];
/* Init accumulator with bias for channel N and N + 1 */
if (bias)
{
for (int i = 0; i < 2; i++)
{
ch_out[i][0] = *bias;
ch_out[i][1] = *bias++;
}
}
for (int i = 0; i < 4; i++)
{
res[i] = vdupq_n_s32(0);
}
/**
* Each time eight int8 data of four filters and eight int16 data
* of two inputs are read.First, the filter data is expanded to
* int16, and then cross-multiplied to obtain 8 calculation results.
*/
while (col_count)
{
int8x8_t filter_s8[2];
int16x8_t input_s16[2];
int16x8_t filter_s16[2];
filter_s8[0] = vld1_s8(ip_a0);
ip_a0 += 8;
filter_s8[1] = vld1_s8(ip_a1);
ip_a1 += 8;
input_s16[0] = vld1q_s16(ip_b0);
ip_b0 += 8;
input_s16[1] = vld1q_s16(ip_b1);
ip_b1 += 8;
for (int i = 0; i < 2; i++)
{
filter_s16[i] = vmovl_s8(filter_s8[i]);
res[i * 2] = vmlal_s16(res[i * 2],
vget_low_s16(filter_s16[i]),
vget_low_s16(input_s16[0]));
res[i * 2 + 1] = vmlal_s16(res[i * 2 + 1],
vget_low_s16(filter_s16[i]),
vget_low_s16(input_s16[1]));
res[i * 2] = vmlal_s16(res[i * 2],
vget_high_s16(filter_s16[i]),
vget_high_s16(input_s16[0]));
res[i * 2 + 1] = vmlal_s16(res[i * 2 + 1],
vget_high_s16(filter_s16[i]),
vget_high_s16(input_s16[1]));
int8_t input_remaining = *(ip_a[i]++);
ch_out[i][0] += input_remaining * b0;
ch_out[i][1] += input_remaining * b1;
}
col_count --;
}
for (int i = 0; i < 2; i++)
{
for (int j = 0; j < 2; j++)
{
ch_out[i][j] += vgetq_lane_s32(res[i * 2 + j], 0);
ch_out[i][j] += vgetq_lane_s32(res[i * 2 + j], 1);
ch_out[i][j] += vgetq_lane_s32(res[i * 2 + j], 2);
ch_out[i][j] += vgetq_lane_s32(res[i * 2 + j], 3);
}
col_count--;
}
col_count = num_col_a % 8;
while (col_count)
for (int i = 0; i < 4; i++)
{
int8_t a0 = *ip_a0++; // filter
int8_t a1 = *ip_a1++;
int16_t b0 = *ip_b0++; // input
int16_t b1 = *ip_b1++;
ch_out[0][0] += a0 * b0;
ch_out[1][1] += a1 * b1;
ch_out[1][0] += a1 * b0;
ch_out[0][1] += a0 * b1;
col_count--;
} /* while over col_count */
for (int i = 0; i < 2; i++)
{
ch_out[i][0] = arm_nn_requantize(ch_out[i][0], *out_mult, *out_shift);
ch_out[i][1] = arm_nn_requantize(ch_out[i][1], *out_mult, *out_shift);
ch_out[i][0] += out_offset;
ch_out[i][1] += out_offset;
ch_out[i][0] = MAX(ch_out[i][0], activation_min);
ch_out[i][1] = MAX(ch_out[i][1], activation_min);
ch_out[i][0] = MIN(ch_out[i][0], activation_max);
ch_out[i][1] = MIN(ch_out[i][1], activation_max);
*out_0++ = (int8_t)ch_out[i][0];
*out_1++ = (int8_t)ch_out[i][1];
out_mult++;
out_shift++;
ch_out[i][0] = arm_nn_requantize(
ch_out[i][0], *out_mult, *out_shift);
ch_out[i][1] = arm_nn_requantize(
ch_out[i][1], *out_mult, *out_shift);
ch_out[i][0] += out_offset;
ch_out[i][1] += out_offset;
ch_out[i][0] = MAX(ch_out[i][0], activation_min);
ch_out[i][1] = MAX(ch_out[i][1], activation_min);
ch_out[i][0] = MIN(ch_out[i][0], activation_max);
ch_out[i][1] = MIN(ch_out[i][1], activation_max);
*out_0++ = (int8_t)ch_out[i][0];
*out_1++ = (int8_t)ch_out[i][1];
out_mult++;
out_shift++;
}
/* skip row */
ip_a0 += num_col_a;
row_count -= 2;
/* skip row */
ip_a0 = ip_a[3];
row_count--;
}
/* compute the last odd numbered row if any */
if (output_ch & 0x1)
row_count = output_ch % 4;
if (row_count >= 2)
{
int32_t col_count = num_col_a / 8;
const int16_t *ip_b0 = input_b;
const int16_t *ip_b1 = ip_b0 + aligned_num_col_a;
int32_t ch_out[2] = {0};
int32x4_t res[2];
/* load the bias */
if (bias)
int32_t col_count = num_col_a / 8;
const int8_t *ip_a1 = ip_a0 + num_col_a;
const int16_t *ip_b0 = input_b;
const int16_t *ip_b1 = ip_b0 + aligned_num_col_a;
int32_t ch_out[2][2] =
{
ch_out[0] = *bias;
ch_out[1] = *bias++;
0
};
int32x4_t res[4];
/* Init accumulator with bias for channel N and N + 1 */
if (bias)
{
for (int i = 0; i < 2; i++)
{
ch_out[i][0] = *bias;
ch_out[i][1] = *bias++;
}
}
res[0] = vdupq_n_s32(0);
res[1] = vdupq_n_s32(0);
while(col_count)
for (int i = 0; i < 4; i++)
{
int8x8_t filter_s8 = vld1_s8(ip_a0);
int16x8_t filter_s16 = vmovl_s8(filter_s8);
int16x8_t input_0_s16 = vld1q_s16(ip_b0);
int16x8_t input_1_s16 = vld1q_s16(ip_b1);
ip_a0 += 8;
ip_b0 += 8;
ip_b1 += 8;
res[0] = vmlal_s16(res[0],
vget_low_s16(filter_s16),
vget_low_s16(input_0_s16));
res[1] = vmlal_s16(res[1],
vget_low_s16(filter_s16),
vget_low_s16(input_1_s16));
res[0] = vmlal_s16(res[0],
vget_high_s16(filter_s16),
vget_high_s16(input_0_s16));
res[1] = vmlal_s16(res[1],
vget_high_s16(filter_s16),
vget_high_s16(input_1_s16));
col_count --;
res[i] = vdupq_n_s32(0);
}
ch_out[0] += vgetq_lane_s32(res[0], 0);
ch_out[0] += vgetq_lane_s32(res[0], 1);
ch_out[0] += vgetq_lane_s32(res[0], 2);
ch_out[0] += vgetq_lane_s32(res[0], 3);
/* Each time eight int8 data of four filters and eight int16 data
* of two inputs are read.First, the filter data is expanded to
* int16, and then cross-multiplied to obtain 8 calculation results.
*/
ch_out[1] += vgetq_lane_s32(res[1], 0);
ch_out[1] += vgetq_lane_s32(res[1], 1);
ch_out[1] += vgetq_lane_s32(res[1], 2);
ch_out[1] += vgetq_lane_s32(res[1], 3);
col_count = num_col_a % 8;
while (col_count)
while (col_count)
{
int8_t a0 = *ip_a0++;
int16_t b0 = *ip_b0++;
int16_t b1 = *ip_b1++;
int8x8_t filter_s8[2];
int16x8_t input_s16[2];
int16x8_t filter_s16[2];
ch_out[0] += a0 * b0;
ch_out[1] += a0 * b1;
col_count--;
filter_s8[0] = vld1_s8(ip_a0);
ip_a0 += 8;
filter_s8[1] = vld1_s8(ip_a1);
ip_a1 += 8;
input_s16[0] = vld1q_s16(ip_b0);
ip_b0 += 8;
input_s16[1] = vld1q_s16(ip_b1);
ip_b1 += 8;
for (int i = 0; i < 2; i++)
{
filter_s16[i] = vmovl_s8(filter_s8[i]);
res[i * 2] = vmlal_s16(res[i * 2],
vget_low_s16(filter_s16[i]),
vget_low_s16(input_s16[0]));
res[i * 2 + 1] = vmlal_s16(res[i * 2 + 1],
vget_low_s16(filter_s16[i]),
vget_low_s16(input_s16[1]));
res[i * 2] = vmlal_s16(res[i * 2],
vget_high_s16(filter_s16[i]),
vget_high_s16(input_s16[0]));
res[i * 2 + 1] = vmlal_s16(res[i * 2 + 1],
vget_high_s16(filter_s16[i]),
vget_high_s16(input_s16[1]));
}
col_count--;
}
ch_out[0] = arm_nn_requantize(ch_out[0], *out_mult, *out_shift);
ch_out[0] += out_offset;
ch_out[0] = MAX(ch_out[0], activation_min);
ch_out[0] = MIN(ch_out[0], activation_max);
*out_0++ = (int8_t)ch_out[0];
for (int i = 0; i < 2; i++)
{
for (int j = 0; j < 2; j++)
{
ch_out[i][j] += vgetq_lane_s32(res[i * 2 + j], 0);
ch_out[i][j] += vgetq_lane_s32(res[i * 2 + j], 1);
ch_out[i][j] += vgetq_lane_s32(res[i * 2 + j], 2);
ch_out[i][j] += vgetq_lane_s32(res[i * 2 + j], 3);
}
}
ch_out[1] = arm_nn_requantize(ch_out[1], *out_mult, *out_shift);
ch_out[1] += out_offset;
ch_out[1] = MAX(ch_out[1], activation_min);
ch_out[1] = MIN(ch_out[1], activation_max);
*out_1++ = (int8_t)ch_out[1];
col_count = num_col_a % 8;
while (col_count) /* while over col_count */
{
int8_t a0 = *ip_a0++; /* filter */
int8_t a1 = *ip_a1++;
int16_t b0 = *ip_b0++; /* input */
int16_t b1 = *ip_b1++;
out_mult++;
out_shift++;
ch_out[0][0] += a0 * b0;
ch_out[1][1] += a1 * b1;
ch_out[1][0] += a1 * b0;
ch_out[0][1] += a0 * b1;
col_count--;
}
for (int i = 0; i < 2; i++)
{
ch_out[i][0] = arm_nn_requantize(
ch_out[i][0], *out_mult, *out_shift);
ch_out[i][1] = arm_nn_requantize(
ch_out[i][1], *out_mult, *out_shift);
ch_out[i][0] += out_offset;
ch_out[i][1] += out_offset;
ch_out[i][0] = MAX(ch_out[i][0], activation_min);
ch_out[i][1] = MAX(ch_out[i][1], activation_min);
ch_out[i][0] = MIN(ch_out[i][0], activation_max);
ch_out[i][1] = MIN(ch_out[i][1], activation_max);
*out_0++ = (int8_t)ch_out[i][0];
*out_1++ = (int8_t)ch_out[i][1];
out_mult++;
out_shift++;
}
/* skip row */
ip_a0 += num_col_a;
row_count -= 2;
}
out_0 += output_ch;
/* compute the last odd numbered row if any */
/* return the new output pointer with offset */
return out_0;
if (output_ch & 0x1)
{
int32_t col_count = num_col_a / 8;
const int16_t *ip_b0 = input_b;
const int16_t *ip_b1 = ip_b0 + aligned_num_col_a;
int32_t ch_out[2] =
{
0
};
int32x4_t res[2];
/* load the bias */
if (bias)
{
ch_out[0] = *bias;
ch_out[1] = *bias++;
}
res[0] = vdupq_n_s32(0);
res[1] = vdupq_n_s32(0);
while (col_count)
{
int8x8_t filter_s8 = vld1_s8(ip_a0);
int16x8_t filter_s16 = vmovl_s8(filter_s8);
int16x8_t input_0_s16 = vld1q_s16(ip_b0);
int16x8_t input_1_s16 = vld1q_s16(ip_b1);
ip_a0 += 8;
ip_b0 += 8;
ip_b1 += 8;
res[0] = vmlal_s16(res[0],
vget_low_s16(filter_s16),
vget_low_s16(input_0_s16));
res[1] = vmlal_s16(res[1],
vget_low_s16(filter_s16),
vget_low_s16(input_1_s16));
res[0] = vmlal_s16(res[0],
vget_high_s16(filter_s16),
vget_high_s16(input_0_s16));
res[1] = vmlal_s16(res[1],
vget_high_s16(filter_s16),
vget_high_s16(input_1_s16));
col_count--;
}
ch_out[0] += vgetq_lane_s32(res[0], 0);
ch_out[0] += vgetq_lane_s32(res[0], 1);
ch_out[0] += vgetq_lane_s32(res[0], 2);
ch_out[0] += vgetq_lane_s32(res[0], 3);
ch_out[1] += vgetq_lane_s32(res[1], 0);
ch_out[1] += vgetq_lane_s32(res[1], 1);
ch_out[1] += vgetq_lane_s32(res[1], 2);
ch_out[1] += vgetq_lane_s32(res[1], 3);
col_count = num_col_a % 8;
while (col_count)
{
int8_t a0 = *ip_a0++;
int16_t b0 = *ip_b0++;
int16_t b1 = *ip_b1++;
ch_out[0] += a0 * b0;
ch_out[1] += a0 * b1;
col_count--;
}
ch_out[0] = arm_nn_requantize(
ch_out[0], *out_mult, *out_shift);
ch_out[0] += out_offset;
ch_out[0] = MAX(ch_out[0], activation_min);
ch_out[0] = MIN(ch_out[0], activation_max);
*out_0++ = (int8_t)ch_out[0];
ch_out[1] = arm_nn_requantize(
ch_out[1], *out_mult, *out_shift);
ch_out[1] += out_offset;
ch_out[1] = MAX(ch_out[1], activation_min);
ch_out[1] = MIN(ch_out[1], activation_max);
*out_1++ = (int8_t)ch_out[1];
out_mult++;
out_shift++;
}
out_0 += output_ch;
/* return the new output pointer with offset */
return out_0;
}

70
mlearning/tflite-micro/operators/neon/arm_q7_to_q15_with_offset.c
View file

@ -1,60 +1,62 @@
/*
* SPDX-FileCopyrightText: Copyright 2010-2023 Arm Limited and/or its affiliates <open-source-office@arm.com>
/****************************************************************************
* apps/mlearning/tflite-micro/operators/neon/arm_q7_to_q15_with_offset.c
*
* SPDX-FileCopyrightText: Copyright 2010-2023 Arm Limited and/or
* its affiliates <open-source-office@arm.com>
*
* SPDX-License-Identifier: Apache-2.0
*
* Licensed under the Apache License, Version 2.0 (the License); you may
* not use this file except in_q7x4 compliance with the License.
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in_q7x4 writing, software
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an AS IS BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
****************************************************************************/
/****************************************************************************
* Included Files
****************************************************************************/
#include <arm_neon.h>
#include "arm_nnsupportfunctions.h"
/**
* @ingroup groupSupport
*/
/****************************************************************************
* Public Functions
****************************************************************************/
/**
* @addtogroup supportConversion
* @{
*/
void arm_q7_to_q15_with_offset(const int8_t *src, int16_t *dst, int32_t block_size, int16_t offset)
void arm_q7_to_q15_with_offset(const int8_t *src,
int16_t *dst,
int32_t block_size,
int16_t offset)
{
int32_t block_cnt;
int32_t block_cnt;
block_cnt = block_size / 8;
int16x8_t offset_s16 = vdupq_n_s16(offset);
while (block_cnt)
block_cnt = block_size / 8;
int16x8_t offset_s16 = vdupq_n_s16(offset);
while (block_cnt)
{
int8x8_t src_s8 = vld1_s8(src);
int16x8_t src_s16 = vmovl_s8(src_s8);
src += 8;
src_s16 = vaddq_s16(offset_s16, src_s16);
block_cnt--;
vst1q_s16(dst, src_s16);
dst += 8;
int8x8_t src_s8 = vld1_s8(src);
int16x8_t src_s16 = vmovl_s8(src_s8);
src += 8;
src_s16 = vaddq_s16(offset_s16, src_s16);
block_cnt--;
vst1q_s16(dst, src_s16);
dst += 8;
}
block_cnt = block_size % 8;
while (block_cnt > 0)
block_cnt = block_size % 8;
while (block_cnt > 0)
{
*dst++ = (int16_t)*src++ + offset;
*dst++ = (int16_t)*src++ + offset;
/* Decrement the loop counter */
block_cnt--;
/* Decrement the loop counter */
block_cnt--;
}
}
/**
* @} end of Doxygen group
*/