Namespaces
namespace	depthwise_conv

Data Structures
struct	BinaryOpActivationFloatMax

struct	BinaryOpActivationFloatMinMax

struct	BinaryOpActivationFloatNone

struct	BinaryOpFuncAddFloat

struct	BinaryOpFuncDivFloat

struct	BinaryOpFuncMulFloat

struct	BinaryOpFuncSubFloat

struct	BinaryOpFuncSwapArgs

struct	FloatDepthwiseConvKernel

struct	GemmlowpOutputPipeline

Typedefs
using	BinaryOpImplFloatFuncs = std::pair< void()(int, const BinaryArithmeticOpParam &, const float , const float , float ), void()(int, const BinaryArithmeticOpParam &, const float, const float , float *)>

Functions
template<typename ElementwiseF , typename ScalarBroadcastF , typename T >
void	BinaryBroadcastFiveFold (const BinaryArithmeticOpParam &params, bool switch_inputs, const Shape &, const T unswitched_input1_data, const Shape &, const T unswitched_input2_data, const Shape &, T *output_data, ElementwiseF elementwise_f, ScalarBroadcastF scalar_broadcast_f)

template<typename ElementwiseF , typename ScalarBroadcastF , typename T >
void	BinaryBroadcastFiveFold (const BinaryArithmeticOpParam &unswitched_params, const Shape &, const T unswitched_input1_data, const Shape &, const T unswitched_input2_data, const Shape &, T *output_data, ElementwiseF elementwise_f, ScalarBroadcastF scalar_broadcast_f)

template<typename T >
std::enable_if_t< is_quant8< T >::value, int32_t >	quant8_sum (const BinaryArithmeticOpParam &params, const T input1_data, const T input2_data)

void	AddElementwise (int size, const BinaryArithmeticOpParam &params, const uint8_t input1_data, const uint8_t input2_data, uint8_t *output_data)

void	AddElementwise (int size, const BinaryArithmeticOpParam &params, const int8_t input1_data, const int8_t input2_data, int8_t *output_data)

template<class OPERATOR , class ACTIVATION >
void	BinaryOpElementwise (int size, const BinaryArithmeticOpParam &params, const float input1_data, const float input2_data, float *output_data)

template<class OPERATOR , class ACTIVATION >
void	BinaryOpScalarBroadcast (int size, const BinaryArithmeticOpParam &params, const float broadcast_value, const float input2_data, float output_data)

template<class FUNC >
BinaryOpImplFloatFuncs	getBinaryOpWithActivationImplFloat (const BinaryArithmeticOpParam &params)

template<typename T >
std::enable_if_t< is_quant8< T >::value >	Add (const BinaryArithmeticOpParam &params, const Shape &input1_shape, const T input1_data, const Shape &input2_shape, const T input2_data, const Shape &output_shape, T *output_data)

void	Add (const BinaryArithmeticOpParam &params, const Shape &input1_shape, const float input1_data, const Shape &input2_shape, const float input2_data, const Shape &output_shape, float *output_data)

void	AddScalarBroadcast (int size, const BinaryArithmeticOpParam &params, uint8_t broadcast_value, const uint8_t input2_data, uint8_t output_data)

void	AddScalarBroadcast (int size, const BinaryArithmeticOpParam &params, int8_t input1_data, const int8_t input2_data, int8_t output_data)

template<typename T >
std::enable_if_t< is_quant8< T >::value >	BroadcastAddDispatch (const BinaryArithmeticOpParam &params, const Shape &input1_shape, const T input1_data, const Shape &input2_shape, const T input2_data, const Shape &output_shape, T *output_data)

void	BroadcastAddDispatch (const BinaryArithmeticOpParam &params, const Shape &input1_shape, const float input1_data, const Shape &input2_shape, const float input2_data, const Shape &output_shape, float *output_data)

void	Sub (const BinaryArithmeticOpParam &params, const Shape &input1_shape, const float input1_data, const Shape &input2_shape, const float input2_data, const Shape &output_shape, float *output_data)

void	BroadcastSubDispatch (const BinaryArithmeticOpParam &params, const Shape &input1_shape, const float input1_data, const Shape &input2_shape, const float input2_data, const Shape &output_shape, float *output_data)

template<typename T >
std::enable_if_t< is_quant8< T >::value, int32_t >	quant8_mul (const BinaryArithmeticOpParam &params, const T input1_data, const T input2_data)

void	MulElementwise (int size, const BinaryArithmeticOpParam &params, const uint8_t input1_data, const uint8_t input2_data, uint8_t *output_data)

void	MulElementwise (int size, const BinaryArithmeticOpParam &params, const int8_t input1_data, const int8_t input2_data, int8_t *output_data)

template<typename T >
std::enable_if_t< is_quant8< T >::value >	Mul (const BinaryArithmeticOpParam &params, const Shape &input1_shape, const T input1_data, const Shape &input2_shape, const T input2_data, const Shape &output_shape, T *output_data)

void	Mul (const BinaryArithmeticOpParam &params, const Shape &input1_shape, const float input1_data, const Shape &input2_shape, const float input2_data, const Shape &output_shape, float *output_data)

void	MulSimpleBroadcast (int size, const BinaryArithmeticOpParam &params, const uint8_t broadcast_value, const uint8_t input2_data, uint8_t output_data)

void	MulSimpleBroadcast (int size, const BinaryArithmeticOpParam &params, const int8_t broadcast_value, const int8_t input2_data, int8_t output_data)

template<typename T >
std::enable_if_t< is_quant8< T >::value >	BroadcastMulDispatch (const BinaryArithmeticOpParam &params, const Shape &input1_shape, const T input1_data, const Shape &input2_shape, const T input2_data, const Shape &output_shape, T *output_data)

void	BroadcastMulDispatch (const BinaryArithmeticOpParam &params, const Shape &input1_shape, const float input1_data, const Shape &input2_shape, const float input2_data, const Shape &output_shape, float *output_data)

void	Div (const BinaryArithmeticOpParam &params, const Shape &input1_shape, const float input1_data, const Shape &input2_shape, const float input2_data, const Shape &output_shape, float *output_data)

void	BroadcastDivDispatch (const BinaryArithmeticOpParam &params, const Shape &input1_shape, const float input1_data, const Shape &input2_shape, const float input2_data, const Shape &output_shape, float *output_data)

void	AddBiasAndEvalActivationFunction (float output_activation_min, float output_activation_max, const Shape &bias_shape, const float bias_data, const Shape &array_shape, float array_data)

void	Conv (const ConvParams &params, const Shape &input_shape, const uint8_t input_data, const Shape &filter_shape, const uint8_t filter_data, const Shape &bias_shape, const int32_t bias_data, const Shape &output_shape, uint8_t output_data, const Shape &im2col_shape, uint8_t *im2col_data)

template<bool kAllowStrided, int kFixedInputDepth, int kFixedDepthMultiplier>
void	FloatDepthwiseConvAccumRow (int stride, int dilation_factor, int input_depth, int input_width, const float input_data, int pad_width, int depth_multiplier, int filter_width, const float filter_data, int out_x_buffer_start, int out_x_buffer_end, int output_depth, float *acc_buffer)

void	FloatDepthwiseConvAccumRowGeneric (int stride, int dilation_factor, int input_depth, int input_width, const float input_data, int pad_width, int depth_multiplier, int filter_width, const float filter_data, int out_x_buffer_start, int out_x_buffer_end, int output_depth, float *acc_buffer)

void	DepthwiseConvInitAccBuffer (int num_output_pixels, int output_depth, const float bias_data, float acc_buffer)

void	DepthwiseConvImpl (const DepthwiseConvParams &params, const Shape &input_shape, const float input_data, const Shape &filter_shape, const float filter_data, const Shape &bias_shape, const float bias_data, const Shape &output_shape, float output_data, int thread_start, int thread_end, int thread_dim)

void	DepthwiseConvWithRounding (const DepthwiseConvParams &params, const Shape &input_shape, const uint8_t input_data, const Shape &filter_shape, const uint8_t filter_data, const Shape &bias_shape, const int32_t bias_data, const Shape &output_shape, uint8_t output_data, int thread_start, int thread_end, int thread_dim)

void	DepthwiseConvImpl (const DepthwiseConvParams &params, const Shape &input_shape, const uint8_t input_data, const Shape &filter_shape, const uint8_t filter_data, const Shape &bias_shape, const int32_t bias_data, const Shape &output_shape, uint8_t output_data, int thread_start, int thread_end, int thread_dim)

template<typename T >
void	ExtractPatchIntoBufferColumn (const Shape &input_shape, int w, int h, int b, int kheight, int kwidth, int stride_width, int stride_height, int pad_width, int pad_height, int in_width, int in_height, int in_depth, int single_buffer_length, int buffer_id, const T in_data, T conv_buffer_data, uint8_t zero_byte)

template<typename T >
void	DilatedIm2col (const ConvParams &params, const Shape &input_shape, const T input_data, const Shape &filter_shape, const Shape &output_shape, T im2col_data, const int32_t *zero_bytes, const int zero_bytes_len)

template<typename T >
void	DilatedIm2col (const ConvParams &params, uint8_t zero_byte, const Shape &input_shape, const T input_data, const Shape &filter_shape, const Shape &output_shape, T im2col_data)

template<typename T >
void	Im2col (const ConvParams &params, int kheight, int kwidth, uint8_t zero_byte, const Shape &input_shape, const T input_data, const Shape &output_shape, T output_data)

Variables
std::mutex	_gemmlowp_mutex

Typedef Documentation

◆ BinaryOpImplFloatFuncs

using nnfw::cker::optimized::BinaryOpImplFloatFuncs = typedef std::pair<void (*)(int, const BinaryArithmeticOpParam &, const float *, const float *, float *), void (*)(int, const BinaryArithmeticOpParam &, const float, const float *, float *)>

Definition at line 670 of file BinaryArithmeticOps.h.

Function Documentation

◆ Add() [1/2]

void nnfw::cker::optimized::Add	(	const BinaryArithmeticOpParam &	params,
		const Shape &	input1_shape,
		const float *	input1_data,
		const Shape &	input2_shape,
		const float *	input2_data,
		const Shape &	output_shape,
		float *	output_data
	)

inline

Definition at line 699 of file BinaryArithmeticOps.h.

{
  const int flat_size = MatchingElementsSize(input1_shape, input2_shape, output_shape);
  auto implFuncs = getBinaryOpWithActivationImplFloat<BinaryOpFuncAddFloat>(params);
  (*implFuncs.first)(flat_size, params, input1_data, input2_data, output_data);
}

References nnfw::cker::MatchingElementsSize(), and output_shape.

◆ Add() [2/2]

template<typename T >

std::enable_if_t< is_quant8< T >::value > nnfw::cker::optimized::Add	(	const BinaryArithmeticOpParam &	params,
		const Shape &	input1_shape,
		const T *	input1_data,
		const Shape &	input2_shape,
		const T *	input2_data,
		const Shape &	output_shape,
		T *	output_data
	)

inline

Definition at line 692 of file BinaryArithmeticOps.h.

{
  const int flat_size = MatchingElementsSize(input1_shape, input2_shape, output_shape);
  AddElementwise(flat_size, params, input1_data, input2_data, output_data);
}

References AddElementwise(), nnfw::cker::MatchingElementsSize(), and output_shape.

Referenced by nnfw::cker::BinaryArithmeticOp(), and nnfw::cker::BinaryArithmeticOp().

◆ AddBiasAndEvalActivationFunction()

void nnfw::cker::optimized::AddBiasAndEvalActivationFunction	(	float	output_activation_min,
		float	output_activation_max,
		const Shape &	bias_shape,
		const float *	bias_data,
		const Shape &	array_shape,
		float *	array_data
	)

inline

Definition at line 74 of file Conv.h.

{
  BiasAndClamp(output_activation_min, output_activation_max, bias_shape.FlatSize(), bias_data,
               array_shape.FlatSize(), array_data);
}

References nnfw::cker::BiasAndClamp(), and nnfw::cker::Shape::FlatSize().

Referenced by nnfw::cker::multithreaded::Conv().

◆ AddElementwise() [1/2]

void nnfw::cker::optimized::AddElementwise	(	int	size,
		const BinaryArithmeticOpParam &	params,
		const int8_t *	input1_data,
		const int8_t *	input2_data,
		int8_t *	output_data
	)

inline

Definition at line 322 of file BinaryArithmeticOps.h.

{
  int i = 0;
#ifdef USE_NEON
  const int8x16_t output_activation_min_vector = vdupq_n_s8(params.quantized_activation_min);
  const int8x16_t output_activation_max_vector = vdupq_n_s8(params.quantized_activation_max);
 
  const int input1_left_shift = params.left_shift + params.input1_shift;
  const int input2_left_shift = params.left_shift + params.input2_shift;
  const int32x4_t input1_left_dup = vdupq_n_s32(input1_left_shift);
  const int32x4_t input2_left_dup = vdupq_n_s32(input2_left_shift);
 
  const int16x8_t input1_offset_dup = vdupq_n_s16(params.input1_offset);
  const int16x8_t input2_offset_dup = vdupq_n_s16(params.input2_offset);
 
  for (; i <= size - 16; i += 16)
  {
    const int8x16_t input1_val_original = vld1q_s8(input1_data + i);
    const int8x16_t input2_val_original = vld1q_s8(input2_data + i);
 
    const int16x8_t input1_val_s16_high = vmovl_s8(vget_high_s8(input1_val_original));
    const int16x8_t input1_val_s16_low = vmovl_s8(vget_low_s8(input1_val_original));
 
    const int16x8_t input2_val_s16_high = vmovl_s8(vget_high_s8(input2_val_original));
    const int16x8_t input2_val_s16_low = vmovl_s8(vget_low_s8(input2_val_original));
    const int16x8_t input1_val_high = vaddq_s16(input1_val_s16_high, input1_offset_dup);
    const int16x8_t input2_val_high = vaddq_s16(input2_val_s16_high, input2_offset_dup);
    const int16x8_t input1_val_low = vaddq_s16(input1_val_s16_low, input1_offset_dup);
    const int16x8_t input2_val_low = vaddq_s16(input2_val_s16_low, input2_offset_dup);
    const int16x4_t input1_val_high_high = vget_high_s16(input1_val_high);
    const int16x4_t input1_val_high_low = vget_low_s16(input1_val_high);
    const int16x4_t input1_val_low_high = vget_high_s16(input1_val_low);
    const int16x4_t input1_val_low_low = vget_low_s16(input1_val_low);
    const int16x4_t input2_val_high_high = vget_high_s16(input2_val_high);
    const int16x4_t input2_val_high_low = vget_low_s16(input2_val_high);
    const int16x4_t input2_val_low_high = vget_high_s16(input2_val_low);
    const int16x4_t input2_val_low_low = vget_low_s16(input2_val_low);
    int32x4_t x111 = vmovl_s16(input1_val_low_low);
    int32x4_t x112 = vmovl_s16(input1_val_low_high);
    int32x4_t x121 = vmovl_s16(input1_val_high_low);
    int32x4_t x122 = vmovl_s16(input1_val_high_high);
    int32x4_t x211 = vmovl_s16(input2_val_low_low);
    int32x4_t x212 = vmovl_s16(input2_val_low_high);
    int32x4_t x221 = vmovl_s16(input2_val_high_low);
    int32x4_t x222 = vmovl_s16(input2_val_high_high);
 
    x111 = vshlq_s32(x111, input1_left_dup);
    x112 = vshlq_s32(x112, input1_left_dup);
    x121 = vshlq_s32(x121, input1_left_dup);
    x122 = vshlq_s32(x122, input1_left_dup);
    x211 = vshlq_s32(x211, input2_left_dup);
    x212 = vshlq_s32(x212, input2_left_dup);
    x221 = vshlq_s32(x221, input2_left_dup);
    x222 = vshlq_s32(x222, input2_left_dup);
    x111 = vqrdmulhq_n_s32(x111, params.input1_multiplier);
    x112 = vqrdmulhq_n_s32(x112, params.input1_multiplier);
    x121 = vqrdmulhq_n_s32(x121, params.input1_multiplier);
    x122 = vqrdmulhq_n_s32(x122, params.input1_multiplier);
    x211 = vqrdmulhq_n_s32(x211, params.input2_multiplier);
    x212 = vqrdmulhq_n_s32(x212, params.input2_multiplier);
    x221 = vqrdmulhq_n_s32(x221, params.input2_multiplier);
    x222 = vqrdmulhq_n_s32(x222, params.input2_multiplier);
    int32x4_t s11 = vaddq_s32(x111, x211);
    int32x4_t s12 = vaddq_s32(x112, x212);
    int32x4_t s21 = vaddq_s32(x121, x221);
    int32x4_t s22 = vaddq_s32(x122, x222);
    s11 = vqrdmulhq_n_s32(s11, params.output_multiplier);
    s12 = vqrdmulhq_n_s32(s12, params.output_multiplier);
    s21 = vqrdmulhq_n_s32(s21, params.output_multiplier);
    s22 = vqrdmulhq_n_s32(s22, params.output_multiplier);
    using gemmlowp::RoundingDivideByPOT;
    s11 = RoundingDivideByPOT(s11, -params.output_shift);
    s12 = RoundingDivideByPOT(s12, -params.output_shift);
    s21 = RoundingDivideByPOT(s21, -params.output_shift);
    s22 = RoundingDivideByPOT(s22, -params.output_shift);
    const int16x4_t s11_narrowed = vmovn_s32(s11);
    const int16x4_t s12_narrowed = vmovn_s32(s12);
    const int16x4_t s21_narrowed = vmovn_s32(s21);
    const int16x4_t s22_narrowed = vmovn_s32(s22);
    const int16x8_t s1 =
      vaddq_s16(vcombine_s16(s11_narrowed, s12_narrowed), vdupq_n_s16(params.output_offset));
    const int16x8_t s2 =
      vaddq_s16(vcombine_s16(s21_narrowed, s22_narrowed), vdupq_n_s16(params.output_offset));
    const int8x16_t s = vcombine_s8(vqmovn_s16(s1), vqmovn_s16(s2));
 
    const int8x16_t clamped =
      vmaxq_s8(output_activation_min_vector, vminq_s8(output_activation_max_vector, s));
    vst1q_s8(output_data + i, clamped);
  }
#endif // NEON
 
  for (; i < size; ++i)
  {
    const int32_t input1_val = params.input1_offset + input1_data[i];
    const int32_t input2_val = params.input2_offset + input2_data[i];
    const int32_t shifted_input1_val = input1_val * (1 << params.left_shift);
    const int32_t shifted_input2_val = input2_val * (1 << params.left_shift);
    const int32_t scaled_input1_val = MultiplyByQuantizedMultiplierSmallerThanOneExp(
      shifted_input1_val, params.input1_multiplier, params.input1_shift);
    const int32_t scaled_input2_val = MultiplyByQuantizedMultiplierSmallerThanOneExp(
      shifted_input2_val, params.input2_multiplier, params.input2_shift);
    const int32_t raw_sum = scaled_input1_val + scaled_input2_val;
    const int32_t raw_output = MultiplyByQuantizedMultiplierSmallerThanOneExp(
                                 raw_sum, params.output_multiplier, params.output_shift) +
                               params.output_offset;
    const int32_t clamped_output = std::min(params.quantized_activation_max,
                                            std::max(params.quantized_activation_min, raw_output));
    output_data[i] = static_cast<int8_t>(clamped_output);
  }
}

◆ AddElementwise() [2/2]

void nnfw::cker::optimized::AddElementwise	(	int	size,
		const BinaryArithmeticOpParam &	params,
		const uint8_t *	input1_data,
		const uint8_t *	input2_data,
		uint8_t *	output_data
	)

inline

Definition at line 246 of file BinaryArithmeticOps.h.

{
  int i = 0;
 
#ifdef USE_NEON
  const uint8x8_t output_activation_min_vector = vdup_n_u8(params.quantized_activation_min);
  const uint8x8_t output_activation_max_vector = vdup_n_u8(params.quantized_activation_max);
  for (; i <= size - 8; i += 8)
  {
    const uint8x8_t input1_val_original = vld1_u8(input1_data + i);
    const uint8x8_t input2_val_original = vld1_u8(input2_data + i);
    const int16x8_t input1_val_s16 = vreinterpretq_s16_u16(vmovl_u8(input1_val_original));
    const int16x8_t input2_val_s16 = vreinterpretq_s16_u16(vmovl_u8(input2_val_original));
    const int16x8_t input1_val = vaddq_s16(input1_val_s16, vdupq_n_s16(params.input1_offset));
    const int16x8_t input2_val = vaddq_s16(input2_val_s16, vdupq_n_s16(params.input2_offset));
    const int16x4_t input1_val_high = vget_high_s16(input1_val);
    const int16x4_t input1_val_low = vget_low_s16(input1_val);
    const int16x4_t input2_val_high = vget_high_s16(input2_val);
    const int16x4_t input2_val_low = vget_low_s16(input2_val);
    int32x4_t x11 = vmovl_s16(input1_val_low);
    int32x4_t x12 = vmovl_s16(input1_val_high);
    int32x4_t x21 = vmovl_s16(input2_val_low);
    int32x4_t x22 = vmovl_s16(input2_val_high);
    const int32x4_t left_shift_dup = vdupq_n_s32(params.left_shift);
    x11 = vshlq_s32(x11, left_shift_dup);
    x12 = vshlq_s32(x12, left_shift_dup);
    x21 = vshlq_s32(x21, left_shift_dup);
    x22 = vshlq_s32(x22, left_shift_dup);
    x11 = vqrdmulhq_n_s32(x11, params.input1_multiplier);
    x12 = vqrdmulhq_n_s32(x12, params.input1_multiplier);
    x21 = vqrdmulhq_n_s32(x21, params.input2_multiplier);
    x22 = vqrdmulhq_n_s32(x22, params.input2_multiplier);
    const int32x4_t input1_shift_dup = vdupq_n_s32(params.input1_shift);
    const int32x4_t input2_shift_dup = vdupq_n_s32(params.input2_shift);
    x11 = vshlq_s32(x11, input1_shift_dup);
    x12 = vshlq_s32(x12, input1_shift_dup);
    x21 = vshlq_s32(x21, input2_shift_dup);
    x22 = vshlq_s32(x22, input2_shift_dup);
    int32x4_t s1 = vaddq_s32(x11, x21);
    int32x4_t s2 = vaddq_s32(x12, x22);
    s1 = vqrdmulhq_n_s32(s1, params.output_multiplier);
    s2 = vqrdmulhq_n_s32(s2, params.output_multiplier);
    using gemmlowp::RoundingDivideByPOT;
    s1 = RoundingDivideByPOT(s1, -params.output_shift);
    s2 = RoundingDivideByPOT(s2, -params.output_shift);
    const int16x4_t s1_narrowed = vmovn_s32(s1);
    const int16x4_t s2_narrowed = vmovn_s32(s2);
    const int16x8_t s =
      vaddq_s16(vcombine_s16(s1_narrowed, s2_narrowed), vdupq_n_s16(params.output_offset));
    const uint8x8_t clamped =
      vmax_u8(output_activation_min_vector, vmin_u8(output_activation_max_vector, vqmovun_s16(s)));
    vst1_u8(output_data + i, clamped);
  }
#endif // NEON
  for (; i < size; ++i)
  {
    const int32_t input1_val = params.input1_offset + input1_data[i];
    const int32_t input2_val = params.input2_offset + input2_data[i];
    const int32_t shifted_input1_val = input1_val * (1 << params.left_shift);
    const int32_t shifted_input2_val = input2_val * (1 << params.left_shift);
    const int32_t scaled_input1_val = MultiplyByQuantizedMultiplierSmallerThanOneExp(
      shifted_input1_val, params.input1_multiplier, params.input1_shift);
    const int32_t scaled_input2_val = MultiplyByQuantizedMultiplierSmallerThanOneExp(
      shifted_input2_val, params.input2_multiplier, params.input2_shift);
    const int32_t raw_sum = scaled_input1_val + scaled_input2_val;
    const int32_t raw_output = MultiplyByQuantizedMultiplierSmallerThanOneExp(
                                 raw_sum, params.output_multiplier, params.output_shift) +
                               params.output_offset;
    const int32_t clamped_output = std::min(params.quantized_activation_max,
                                            std::max(params.quantized_activation_min, raw_output));
    output_data[i] = static_cast<uint8_t>(clamped_output);
  }
}

Referenced by Add(), and BroadcastAddDispatch().

◆ AddScalarBroadcast() [1/2]

void nnfw::cker::optimized::AddScalarBroadcast	(	int	size,
		const BinaryArithmeticOpParam &	params,
		int8_t	input1_data,
		const int8_t *	input2_data,
		int8_t *	output_data
	)

inline

Definition at line 727 of file BinaryArithmeticOps.h.

{
  using gemmlowp::RoundingDivideByPOT;
  int i = 0;
#ifdef USE_NEON
  const int32x4_t left_shift_dup = vdupq_n_s32(params.left_shift);
  const int8x8_t output_activation_min_vector = vdup_n_s8(params.quantized_activation_min);
  const int8x8_t output_activation_max_vector = vdup_n_s8(params.quantized_activation_max);
 
  // Process broadcast scalar.
  const int8x8_t input1_val_original = vdup_n_s8(input1_data);
  const int16x8_t input1_val_s16 = vmovl_s8(input1_val_original);
  const int16x8_t input1_val = vaddq_s16(input1_val_s16, vdupq_n_s16(params.input1_offset));
  const int16x4_t input1_val_high = vget_high_s16(input1_val);
  const int16x4_t input1_val_low = vget_low_s16(input1_val);
  int32x4_t x11 = vmovl_s16(input1_val_low);
  int32x4_t x12 = vmovl_s16(input1_val_high);
  x11 = vshlq_s32(x11, left_shift_dup);
  x12 = vshlq_s32(x12, left_shift_dup);
  x11 = vqrdmulhq_n_s32(x11, params.input1_multiplier);
  x12 = vqrdmulhq_n_s32(x12, params.input1_multiplier);
  const int32x4_t input1_shift_dup = vdupq_n_s32(params.input1_shift);
  x11 = vshlq_s32(x11, input1_shift_dup);
  x12 = vshlq_s32(x12, input1_shift_dup);
 
  for (; i <= size - 8; i += 8)
  {
    const int8x8_t input2_val_original = vld1_s8(input2_data + i);
    const int16x8_t input2_val_s16 = vmovl_s8(input2_val_original);
    const int16x8_t input2_val = vaddq_s16(input2_val_s16, vdupq_n_s16(params.input2_offset));
    const int16x4_t input2_val_high = vget_high_s16(input2_val);
    const int16x4_t input2_val_low = vget_low_s16(input2_val);
    int32x4_t x21 = vmovl_s16(input2_val_low);
    int32x4_t x22 = vmovl_s16(input2_val_high);
    x21 = vshlq_s32(x21, left_shift_dup);
    x22 = vshlq_s32(x22, left_shift_dup);
    x21 = vqrdmulhq_n_s32(x21, params.input2_multiplier);
    x22 = vqrdmulhq_n_s32(x22, params.input2_multiplier);
    const int32x4_t input2_shift_dup = vdupq_n_s32(params.input2_shift);
    x21 = vshlq_s32(x21, input2_shift_dup);
    x22 = vshlq_s32(x22, input2_shift_dup);
    int32x4_t s1 = vaddq_s32(x11, x21);
    int32x4_t s2 = vaddq_s32(x12, x22);
    s1 = vqrdmulhq_n_s32(s1, params.output_multiplier);
    s2 = vqrdmulhq_n_s32(s2, params.output_multiplier);
    s1 = RoundingDivideByPOT(s1, -params.output_shift);
    s2 = RoundingDivideByPOT(s2, -params.output_shift);
    const int16x4_t s1_narrowed = vmovn_s32(s1);
    const int16x4_t s2_narrowed = vmovn_s32(s2);
    const int16x8_t s =
      vaddq_s16(vcombine_s16(s1_narrowed, s2_narrowed), vdupq_n_s16(params.output_offset));
    const int8x8_t clamped =
      vmax_s8(output_activation_min_vector, vmin_s8(output_activation_max_vector, vqmovn_s16(s)));
    vst1_s8(output_data + i, clamped);
  }
#endif // NEON
 
  if (i < size)
  {
    // Process broadcast scalar.
    const int32_t input1_val = params.input1_offset + input1_data;
    const int32_t shifted_input1_val = input1_val * (1 << params.left_shift);
    const int32_t scaled_input1_val = MultiplyByQuantizedMultiplierSmallerThanOneExp(
      shifted_input1_val, params.input1_multiplier, params.input1_shift);
 
    for (; i < size; ++i)
    {
      const int32_t input2_val = params.input2_offset + input2_data[i];
      const int32_t shifted_input2_val = input2_val * (1 << params.left_shift);
      const int32_t scaled_input2_val = MultiplyByQuantizedMultiplierSmallerThanOneExp(
        shifted_input2_val, params.input2_multiplier, params.input2_shift);
      const int32_t raw_sum = scaled_input1_val + scaled_input2_val;
      const int32_t raw_output = MultiplyByQuantizedMultiplierSmallerThanOneExp(
                                   raw_sum, params.output_multiplier, params.output_shift) +
                                 params.output_offset;
      const int32_t clamped_output = std::min(
        params.quantized_activation_max, std::max(params.quantized_activation_min, raw_output));
      output_data[i] = static_cast<int8_t>(clamped_output);
    }
  }
}

◆ AddScalarBroadcast() [2/2]

void nnfw::cker::optimized::AddScalarBroadcast	(	int	size,
		const BinaryArithmeticOpParam &	params,
		uint8_t	broadcast_value,
		const uint8_t *	input2_data,
		uint8_t *	output_data
	)

inline

Definition at line 711 of file BinaryArithmeticOps.h.

{
  int i = 0;
  int32_t clamped_output;
  for (; i < size; ++i)
  {
    clamped_output = quant8_sum(params, broadcast_value, input2_data[i]);
    output_data[i] = static_cast<uint8_t>(clamped_output);
  }
}

References quant8_sum(), and size.

Referenced by BroadcastAddDispatch().

◆ BinaryBroadcastFiveFold() [1/2]

template<typename ElementwiseF , typename ScalarBroadcastF , typename T >

void nnfw::cker::optimized::BinaryBroadcastFiveFold	(	const BinaryArithmeticOpParam &	params,
		bool	switch_inputs,
		const Shape &	,
		const T *	unswitched_input1_data,
		const Shape &	,
		const T *	unswitched_input2_data,
		const Shape &	,
		T *	output_data,
		ElementwiseF	elementwise_f,
		ScalarBroadcastF	scalar_broadcast_f
	)

inline

Definition at line 40 of file BinaryArithmeticOps.h.

{
  const T *input1_data = switch_inputs ? unswitched_input2_data : unswitched_input1_data;
  const T *input2_data = switch_inputs ? unswitched_input1_data : unswitched_input2_data;
 
  // Fivefold nested loops. The second input resets its position for each
  // iteration of the second loop. The first input resets its position at the
  // beginning of the fourth loop. The innermost loop is an elementwise add of
  // sections of the arrays.
  T *output_data_ptr = output_data;
  const T *input1_data_ptr = input1_data;
  const T *input2_data_reset = input2_data;
  // In the fivefold pattern, y0, y2 and y4 are not broadcast, and so shared
  // between input shapes. y3 for input 1 is always broadcast, and so the
  // dimension there is 1, whereas optionally y1 might be broadcast for input 2.
  // Put another way,
  // input1.shape.FlatSize = y0 * y1 * y2 * y4,
  // input2.shape.FlatSize = y0 * y2 * y3 * y4.
  int y0 = params.broadcast_shape[0];
  int y1 = params.broadcast_shape[1];
  int y2 = params.broadcast_shape[2];
  int y3 = params.broadcast_shape[3];
  int y4 = params.broadcast_shape[4];
  if (y4 > 1)
  {
    // General fivefold pattern, with y4 > 1 so there is a non-broadcast inner
    // dimension.
    for (int i0 = 0; i0 < y0; ++i0)
    {
      const T *input2_data_ptr = nullptr;
      for (int i1 = 0; i1 < y1; ++i1)
      {
        input2_data_ptr = input2_data_reset;
        for (int i2 = 0; i2 < y2; ++i2)
        {
          for (int i3 = 0; i3 < y3; ++i3)
          {
            elementwise_f(y4, params, input1_data_ptr, input2_data_ptr, output_data_ptr);
            input2_data_ptr += y4;
            output_data_ptr += y4;
          }
          // We have broadcast y4 of input1 data y3 times, and now move on.
          input1_data_ptr += y4;
        }
      }
      // We have broadcast y2*y3*y4 of input2 data y1 times, and now move on.
      input2_data_reset = input2_data_ptr;
    }
  }
  else
  {
    // Special case of y4 == 1, in which the innermost loop is a single element
    // and can be combined with the next (y3) as an inner broadcast.
    //
    // Note that this handles the case of pure scalar broadcast when
    // y0 == y1 == y2 == 1. With low overhead it handles cases such as scalar
    // broadcast with batch (as y2 > 1).
    //
    // NOTE The process is the same as the above general case except simplified
    // for y4 == 1 and the loop over y3 is contained within the
    // AddScalarBroadcast function.
    for (int i0 = 0; i0 < y0; ++i0)
    {
      const T *input2_data_ptr = nullptr;
      for (int i1 = 0; i1 < y1; ++i1)
      {
        input2_data_ptr = input2_data_reset;
        for (int i2 = 0; i2 < y2; ++i2)
        {
          scalar_broadcast_f(y3, params, *input1_data_ptr, input2_data_ptr, output_data_ptr);
          input2_data_ptr += y3;
          output_data_ptr += y3;
          input1_data_ptr += 1;
        }
      }
      input2_data_reset = input2_data_ptr;
    }
  }
}

References nnfw::cker::BinaryArithmeticOpParam::broadcast_shape.

Referenced by BroadcastAddDispatch(), BroadcastAddDispatch(), BroadcastDivDispatch(), BroadcastMulDispatch(), BroadcastMulDispatch(), and BroadcastSubDispatch().

◆ BinaryBroadcastFiveFold() [2/2]

template<typename ElementwiseF , typename ScalarBroadcastF , typename T >

void nnfw::cker::optimized::BinaryBroadcastFiveFold	(	const BinaryArithmeticOpParam &	unswitched_params,
		const Shape &	,
		const T *	unswitched_input1_data,
		const Shape &	,
		const T *	unswitched_input2_data,
		const Shape &	,
		T *	output_data,
		ElementwiseF	elementwise_f,
		ScalarBroadcastF	scalar_broadcast_f
	)

inline

Definition at line 128 of file BinaryArithmeticOps.h.

{
  BinaryArithmeticOpParam switched_params = unswitched_params;
  switched_params.input1_offset = unswitched_params.input2_offset;
  switched_params.input1_multiplier = unswitched_params.input2_multiplier;
  switched_params.input1_shift = unswitched_params.input2_shift;
  switched_params.input2_offset = unswitched_params.input1_offset;
  switched_params.input2_multiplier = unswitched_params.input1_multiplier;
  switched_params.input2_shift = unswitched_params.input1_shift;
 
  const bool use_unswitched =
    unswitched_params.broadcast_category == BroadcastableOpCategory::kFirstInputBroadcastsFast;
 
  const BinaryArithmeticOpParam &params = use_unswitched ? unswitched_params : switched_params;
  const T *input1_data = use_unswitched ? unswitched_input1_data : unswitched_input2_data;
  const T *input2_data = use_unswitched ? unswitched_input2_data : unswitched_input1_data;
 
  // Fivefold nested loops. The second input resets its position for each
  // iteration of the second loop. The first input resets its position at the
  // beginning of the fourth loop. The innermost loop is an elementwise add of
  // sections of the arrays.
  T *output_data_ptr = output_data;
  const T *input1_data_ptr = input1_data;
  const T *input2_data_reset = input2_data;
  // In the fivefold pattern, y0, y2 and y4 are not broadcast, and so shared
  // between input shapes. y3 for input 1 is always broadcast, and so the
  // dimension there is 1, whereas optionally y1 might be broadcast for
  // input 2. Put another way, input1.shape.FlatSize = y0 * y1 * y2 * y4,
  // input2.shape.FlatSize = y0 * y2 * y3 * y4.
  int y0 = params.broadcast_shape[0];
  int y1 = params.broadcast_shape[1];
  int y2 = params.broadcast_shape[2];
  int y3 = params.broadcast_shape[3];
  int y4 = params.broadcast_shape[4];
  if (y4 > 1)
  {
    // General fivefold pattern, with y4 > 1 so there is a non-broadcast inner
    // dimension.
    for (int i0 = 0; i0 < y0; ++i0)
    {
      const T *input2_data_ptr = nullptr;
      for (int i1 = 0; i1 < y1; ++i1)
      {
        input2_data_ptr = input2_data_reset;
        for (int i2 = 0; i2 < y2; ++i2)
        {
          for (int i3 = 0; i3 < y3; ++i3)
          {
            elementwise_f(y4, params, input1_data_ptr, input2_data_ptr, output_data_ptr);
            input2_data_ptr += y4;
            output_data_ptr += y4;
          }
          // We have broadcast y4 of input1 data y3 times, and now move on.
          input1_data_ptr += y4;
        }
      }
      // We have broadcast y2*y3*y4 of input2 data y1 times, and now move on.
      input2_data_reset = input2_data_ptr;
    }
  }
  else
  {
    // Special case of y4 == 1, in which the innermost loop is a single
    // element and can be combined with the next (y3) as an inner broadcast.
    //
    // Note that this handles the case of pure scalar broadcast when
    // y0 == y1 == y2 == 1. With low overhead it handles cases such as scalar
    // broadcast with batch (as y2 > 1).
    //
    // NOTE The process is the same as the above general case except
    // simplified for y4 == 1 and the loop over y3 is contained within the
    // AddScalarBroadcast function.
    for (int i0 = 0; i0 < y0; ++i0)
    {
      const T *input2_data_ptr = nullptr;
      for (int i1 = 0; i1 < y1; ++i1)
      {
        input2_data_ptr = input2_data_reset;
        for (int i2 = 0; i2 < y2; ++i2)
        {
          scalar_broadcast_f(y3, params, *input1_data_ptr, input2_data_ptr, output_data_ptr);
          input2_data_ptr += y3;
          output_data_ptr += y3;
          input1_data_ptr += 1;
        }
      }
      input2_data_reset = input2_data_ptr;
    }
  }
}

References nnfw::cker::BinaryArithmeticOpParam::broadcast_category, nnfw::cker::BinaryArithmeticOpParam::broadcast_shape, nnfw::cker::BinaryArithmeticOpParam::input1_multiplier, nnfw::cker::BinaryArithmeticOpParam::input1_offset, nnfw::cker::BinaryArithmeticOpParam::input1_shift, nnfw::cker::BinaryArithmeticOpParam::input2_multiplier, nnfw::cker::BinaryArithmeticOpParam::input2_offset, nnfw::cker::BinaryArithmeticOpParam::input2_shift, and nnfw::cker::kFirstInputBroadcastsFast.

◆ BinaryOpElementwise()

template<class OPERATOR , class ACTIVATION >

void nnfw::cker::optimized::BinaryOpElementwise	(	int	size,
		const BinaryArithmeticOpParam &	params,
		const float *	input1_data,
		const float *	input2_data,
		float *	output_data
	)

inline

Definition at line 562 of file BinaryArithmeticOps.h.

{
  int i = 0;
 
#ifdef USE_NEON
  const auto activation_min = vdupq_n_f32(params.float_activation_min);
  const auto activation_max = vdupq_n_f32(params.float_activation_max);
  for (; i <= size - 16; i += 16)
  {
    auto a10 = vld1q_f32(input1_data + i);
    auto a11 = vld1q_f32(input1_data + i + 4);
    auto a12 = vld1q_f32(input1_data + i + 8);
    auto a13 = vld1q_f32(input1_data + i + 12);
    auto a20 = vld1q_f32(input2_data + i);
    auto a21 = vld1q_f32(input2_data + i + 4);
    auto a22 = vld1q_f32(input2_data + i + 8);
    auto a23 = vld1q_f32(input2_data + i + 12);
    auto x0 = OPERATOR::calculate(a10, a20);
    auto x1 = OPERATOR::calculate(a11, a21);
    auto x2 = OPERATOR::calculate(a12, a22);
    auto x3 = OPERATOR::calculate(a13, a23);
    x0 = ACTIVATION::applyFloor(x0, activation_min);
    x1 = ACTIVATION::applyFloor(x1, activation_min);
    x2 = ACTIVATION::applyFloor(x2, activation_min);
    x3 = ACTIVATION::applyFloor(x3, activation_min);
    x0 = ACTIVATION::applyCeiling(x0, activation_max);
    x1 = ACTIVATION::applyCeiling(x1, activation_max);
    x2 = ACTIVATION::applyCeiling(x2, activation_max);
    x3 = ACTIVATION::applyCeiling(x3, activation_max);
    vst1q_f32(output_data + i, x0);
    vst1q_f32(output_data + i + 4, x1);
    vst1q_f32(output_data + i + 8, x2);
    vst1q_f32(output_data + i + 12, x3);
  }
  for (; i <= size - 4; i += 4)
  {
    auto a1 = vld1q_f32(input1_data + i);
    auto a2 = vld1q_f32(input2_data + i);
    auto x = OPERATOR::calculate(a1, a2); // vaddq
    auto x_clamped =
      ACTIVATION::applyCeiling(ACTIVATION::applyFloor(x, activation_min), activation_max);
    vst1q_f32(output_data + i, x_clamped);
  }
#endif // USE_NEON
  for (; i < size; i++)
  {
    auto x = OPERATOR::calculate(input1_data[i], input2_data[i]);
    output_data[i] = ACTIVATION::applyCeiling(
      ACTIVATION::applyFloor(x, params.float_activation_min), params.float_activation_max);
  }
}

References nnfw::cker::BinaryArithmeticOpParam::float_activation_max, nnfw::cker::BinaryArithmeticOpParam::float_activation_min, and size.

◆ BinaryOpScalarBroadcast()

template<class OPERATOR , class ACTIVATION >

void nnfw::cker::optimized::BinaryOpScalarBroadcast	(	int	size,
		const BinaryArithmeticOpParam &	params,
		const float	broadcast_value,
		const float *	input2_data,
		float *	output_data
	)

inline

Definition at line 620 of file BinaryArithmeticOps.h.

{
  int i = 0;
 
#ifdef USE_NEON
  const auto activation_min = vdupq_n_f32(params.float_activation_min);
  const auto activation_max = vdupq_n_f32(params.float_activation_max);
  const auto broadcast_value_dup = vdupq_n_f32(broadcast_value);
  for (; i <= size - 16; i += 16)
  {
    auto a20 = vld1q_f32(input2_data + i);
    auto a21 = vld1q_f32(input2_data + i + 4);
    auto a22 = vld1q_f32(input2_data + i + 8);
    auto a23 = vld1q_f32(input2_data + i + 12);
    auto x0 = OPERATOR::calculate(broadcast_value_dup, a20);
    auto x1 = OPERATOR::calculate(broadcast_value_dup, a21);
    auto x2 = OPERATOR::calculate(broadcast_value_dup, a22);
    auto x3 = OPERATOR::calculate(broadcast_value_dup, a23);
    x0 = ACTIVATION::applyFloor(x0, activation_min);
    x1 = ACTIVATION::applyFloor(x1, activation_min);
    x2 = ACTIVATION::applyFloor(x2, activation_min);
    x3 = ACTIVATION::applyFloor(x3, activation_min);
    x0 = ACTIVATION::applyCeiling(x0, activation_max);
    x1 = ACTIVATION::applyCeiling(x1, activation_max);
    x2 = ACTIVATION::applyCeiling(x2, activation_max);
    x3 = ACTIVATION::applyCeiling(x3, activation_max);
    vst1q_f32(output_data + i, x0);
    vst1q_f32(output_data + i + 4, x1);
    vst1q_f32(output_data + i + 8, x2);
    vst1q_f32(output_data + i + 12, x3);
  }
  for (; i <= size - 4; i += 4)
  {
    auto a2 = vld1q_f32(input2_data + i);
    auto x = OPERATOR::calculate(broadcast_value_dup, a2);
    auto x_clamped =
      ACTIVATION::applyCeiling(ACTIVATION::applyFloor(x, activation_min), activation_max);
    vst1q_f32(output_data + i, x_clamped);
  }
#endif // USE_NEON
  for (; i < size; i++)
  {
    auto x = OPERATOR::calculate(broadcast_value, input2_data[i]);
    output_data[i] = ACTIVATION::applyCeiling(
      ACTIVATION::applyFloor(x, params.float_activation_min), params.float_activation_max);
  }
}

References nnfw::cker::BinaryArithmeticOpParam::float_activation_max, nnfw::cker::BinaryArithmeticOpParam::float_activation_min, and size.

◆ BroadcastAddDispatch() [1/2]

void nnfw::cker::optimized::BroadcastAddDispatch	(	const BinaryArithmeticOpParam &	params,
		const Shape &	input1_shape,
		const float *	input1_data,
		const Shape &	input2_shape,
		const float *	input2_data,
		const Shape &	output_shape,
		float *	output_data
	)

inline

Definition at line 835 of file BinaryArithmeticOps.h.

{
  if (params.broadcast_category == BroadcastableOpCategory::kGenericBroadcast)
  {
    const std::function<float(const float &, const float &)> fn =
      [](const float &a, const float &b) -> float { return a + b; };
    reference::BroadcastBinaryArithmeticOpSlow(params, input1_shape, input1_data, input2_shape,
                                               input2_data, output_shape, output_data, fn);
  }
  else
  {
    auto implFuncs = getBinaryOpWithActivationImplFloat<BinaryOpFuncAddFloat>(params);
 
    BinaryBroadcastFiveFold(
      params, params.broadcast_category == BroadcastableOpCategory::kSecondInputBroadcastsFast,
      input1_shape, input1_data, input2_shape, input2_data, output_shape, output_data,
      implFuncs.first, implFuncs.second);
  }
}

References BinaryBroadcastFiveFold(), nnfw::cker::BinaryArithmeticOpParam::broadcast_category, nnfw::cker::reference::BroadcastBinaryArithmeticOpSlow(), nnfw::cker::kGenericBroadcast, nnfw::cker::kSecondInputBroadcastsFast, and output_shape.

◆ BroadcastAddDispatch() [2/2]

template<typename T >

std::enable_if_t< is_quant8< T >::value > nnfw::cker::optimized::BroadcastAddDispatch	(	const BinaryArithmeticOpParam &	params,
		const Shape &	input1_shape,
		const T *	input1_data,
		const Shape &	input2_shape,
		const T *	input2_data,
		const Shape &	output_shape,
		T *	output_data
	)

inline

Definition at line 812 of file BinaryArithmeticOps.h.

{
  if (params.broadcast_category == BroadcastableOpCategory::kGenericBroadcast)
  {
    const std::function<T(const BinaryArithmeticOpParam &, const T &, const T &)> fn =
      [](const BinaryArithmeticOpParam &params, const T &a, const T &b) {
        return static_cast<T>(quant8_sum(params, a, b));
      };
    reference::BroadcastBinaryArithmeticOpSlow(params, input1_shape, input1_data, input2_shape,
                                               input2_data, output_shape, output_data, fn);
    return;
  }
 
  BinaryBroadcastFiveFold(
    params, input1_shape, input1_data, input2_shape, input2_data, output_shape, output_data,
    static_cast<void (*)(int, const BinaryArithmeticOpParam &, const T *, const T *, T *)>(
      AddElementwise),
    static_cast<void (*)(int, const BinaryArithmeticOpParam &, T, const T *, T *)>(
      AddScalarBroadcast));
}

References AddElementwise(), AddScalarBroadcast(), BinaryBroadcastFiveFold(), nnfw::cker::BinaryArithmeticOpParam::broadcast_category, nnfw::cker::reference::BroadcastBinaryArithmeticOpSlow(), nnfw::cker::kGenericBroadcast, output_shape, and quant8_sum().

Referenced by nnfw::cker::BroadcastBinaryArithmeticOp(), and nnfw::cker::BroadcastBinaryArithmeticOp().

◆ BroadcastDivDispatch()

void nnfw::cker::optimized::BroadcastDivDispatch	(	const BinaryArithmeticOpParam &	params,
		const Shape &	input1_shape,
		const float *	input1_data,
		const Shape &	input2_shape,
		const float *	input2_data,
		const Shape &	output_shape,
		float *	output_data
	)

inline

Definition at line 1234 of file BinaryArithmeticOps.h.

{
#ifdef __aarch64__
  if (params.broadcast_category == BroadcastableOpCategory::kFirstInputBroadcastsFast)
  {
    auto implFuncs = getBinaryOpWithActivationImplFloat<BinaryOpFuncDivFloat>(params);
    BinaryBroadcastFiveFold(params, false, input1_shape, input1_data, input2_shape, input2_data,
                            output_shape, output_data, implFuncs.first, implFuncs.second);
  }
  else if (params.broadcast_category == BroadcastableOpCategory::kSecondInputBroadcastsFast)
  {
    auto implFuncs =
      getBinaryOpWithActivationImplFloat<BinaryOpFuncSwapArgs<BinaryOpFuncDivFloat>>(params);
    BinaryBroadcastFiveFold(params, true, input1_shape, input1_data, input2_shape, input2_data,
                            output_shape, output_data, implFuncs.first, implFuncs.second);
  }
  else
#endif // __aarch64__
  {
    const std::function<float(const float &, const float &)> fn =
      [](const float &a, const float &b) -> float { return a / b; };
    reference::BroadcastBinaryArithmeticOpSlow(params, input1_shape, input1_data, input2_shape,
                                               input2_data, output_shape, output_data, fn);
  }
}

References BinaryBroadcastFiveFold(), nnfw::cker::BinaryArithmeticOpParam::broadcast_category, nnfw::cker::reference::BroadcastBinaryArithmeticOpSlow(), nnfw::cker::kFirstInputBroadcastsFast, nnfw::cker::kSecondInputBroadcastsFast, and output_shape.

Referenced by nnfw::cker::BroadcastBinaryArithmeticOp().

◆ BroadcastMulDispatch() [1/2]

void nnfw::cker::optimized::BroadcastMulDispatch	(	const BinaryArithmeticOpParam &	params,
		const Shape &	input1_shape,
		const float *	input1_data,
		const Shape &	input2_shape,
		const float *	input2_data,
		const Shape &	output_shape,
		float *	output_data
	)

inline

Definition at line 1199 of file BinaryArithmeticOps.h.

{
  if (params.broadcast_category == BroadcastableOpCategory::kGenericBroadcast)
  {
    // TODO: Use GetBinaryArithmeticFn
    const std::function<float(const float &, const float &)> fn =
      [](const float &a, const float &b) -> float { return a * b; };
    reference::BroadcastBinaryArithmeticOpSlow(params, input1_shape, input1_data, input2_shape,
                                               input2_data, output_shape, output_data, fn);
    return;
  }
  auto implFuncs = getBinaryOpWithActivationImplFloat<BinaryOpFuncMulFloat>(params);
  BinaryBroadcastFiveFold(params, input1_shape, input1_data, input2_shape, input2_data,
                          output_shape, output_data, implFuncs.first, implFuncs.second);
}

References BinaryBroadcastFiveFold(), nnfw::cker::BinaryArithmeticOpParam::broadcast_category, nnfw::cker::reference::BroadcastBinaryArithmeticOpSlow(), nnfw::cker::kGenericBroadcast, and output_shape.

◆ BroadcastMulDispatch() [2/2]

template<typename T >

std::enable_if_t< is_quant8< T >::value > nnfw::cker::optimized::BroadcastMulDispatch	(	const BinaryArithmeticOpParam &	params,
		const Shape &	input1_shape,
		const T *	input1_data,
		const Shape &	input2_shape,
		const T *	input2_data,
		const Shape &	output_shape,
		T *	output_data
	)

inline

Definition at line 1177 of file BinaryArithmeticOps.h.

{
  if (params.broadcast_category == BroadcastableOpCategory::kGenericBroadcast)
  {
    const std::function<T(const BinaryArithmeticOpParam &, const T &, const T &)> fn =
      [](const BinaryArithmeticOpParam &params, const T &a, const T &b) {
        return static_cast<T>(quant8_mul(params, a, b));
      };
    reference::BroadcastBinaryArithmeticOpSlow(params, input1_shape, input1_data, input2_shape,
                                               input2_data, output_shape, output_data, fn);
    return;
  }
  BinaryBroadcastFiveFold(
    params, input1_shape, input1_data, input2_shape, input2_data, output_shape, output_data,
    static_cast<void (*)(int, const BinaryArithmeticOpParam &, const T *, const T *, T *)>(
      MulElementwise),
    static_cast<void (*)(int, const BinaryArithmeticOpParam &, T, const T *, T *)>(
      MulSimpleBroadcast));
}

References BinaryBroadcastFiveFold(), nnfw::cker::BinaryArithmeticOpParam::broadcast_category, nnfw::cker::reference::BroadcastBinaryArithmeticOpSlow(), nnfw::cker::kGenericBroadcast, MulElementwise(), MulSimpleBroadcast(), output_shape, and quant8_mul().

Referenced by nnfw::cker::BroadcastBinaryArithmeticOp(), and nnfw::cker::BroadcastBinaryArithmeticOp().

◆ BroadcastSubDispatch()

void nnfw::cker::optimized::BroadcastSubDispatch	(	const BinaryArithmeticOpParam &	params,
		const Shape &	input1_shape,
		const float *	input1_data,
		const Shape &	input2_shape,
		const float *	input2_data,
		const Shape &	output_shape,
		float *	output_data
	)

inline

Definition at line 867 of file BinaryArithmeticOps.h.

{
  if (params.broadcast_category == BroadcastableOpCategory::kFirstInputBroadcastsFast)
  {
    auto implFuncs = getBinaryOpWithActivationImplFloat<BinaryOpFuncSubFloat>(params);
    BinaryBroadcastFiveFold(params, false, input1_shape, input1_data, input2_shape, input2_data,
                            output_shape, output_data, implFuncs.first, implFuncs.second);
  }
  else if (params.broadcast_category == BroadcastableOpCategory::kSecondInputBroadcastsFast)
  {
    auto implFuncs =
      getBinaryOpWithActivationImplFloat<BinaryOpFuncSwapArgs<BinaryOpFuncSubFloat>>(params);
    BinaryBroadcastFiveFold(params, true, input1_shape, input1_data, input2_shape, input2_data,
                            output_shape, output_data, implFuncs.first, implFuncs.second);
  }
  else
  {
    const std::function<float(const float &, const float &)> fn =
      [](const float &a, const float &b) -> float { return a - b; };
    reference::BroadcastBinaryArithmeticOpSlow(params, input1_shape, input1_data, input2_shape,
                                               input2_data, output_shape, output_data, fn);
  }
}

References BinaryBroadcastFiveFold(), nnfw::cker::BinaryArithmeticOpParam::broadcast_category, nnfw::cker::reference::BroadcastBinaryArithmeticOpSlow(), nnfw::cker::kFirstInputBroadcastsFast, nnfw::cker::kSecondInputBroadcastsFast, and output_shape.

Referenced by nnfw::cker::BroadcastBinaryArithmeticOp().

◆ Conv()

void nnfw::cker::optimized::Conv	(	const ConvParams &	params,
		const Shape &	input_shape,
		const uint8_t *	input_data,
		const Shape &	filter_shape,
		const uint8_t *	filter_data,
		const Shape &	bias_shape,
		const int32_t *	bias_data,
		const Shape &	output_shape,
		uint8_t *	output_data,
		const Shape &	im2col_shape,
		uint8_t *	im2col_data
	)

inline

Definition at line 83 of file Conv.h.

{
  gemmlowp::GemmContext *gemm_context = gemm_support::GetGemmLowpContext();
 
  const int stride_width = params.stride_width;
  const int stride_height = params.stride_height;
  const int dilation_width_factor = params.dilation_width_factor;
  const int dilation_height_factor = params.dilation_height_factor;
  const int32_t input_offset = params.input_offset;
  const int32_t filter_offset = params.weights_offset;
  const int32_t output_offset = params.output_offset;
  const int32_t output_multiplier = params.output_multiplier;
  const int output_shift = params.output_shift;
  const int32_t output_activation_min = params.quantized_activation_min;
  const int32_t output_activation_max = params.quantized_activation_max;
  assert(input_shape.DimensionsCount() == 4);
  assert(filter_shape.DimensionsCount() == 4);
  assert(output_shape.DimensionsCount() == 4);
 
  const uint8_t *gemm_input_data = nullptr;
  const Shape *gemm_input_shape = nullptr;
  const int filter_width = filter_shape.Dims(2);
  const int filter_height = filter_shape.Dims(1);
  const bool need_dilated_im2col = dilation_width_factor != 1 || dilation_height_factor != 1;
  const bool need_im2col =
    stride_width != 1 || stride_height != 1 || filter_width != 1 || filter_height != 1;
  if (need_dilated_im2col)
  {
    assert(im2col_data);
    const int input_zero_point = -input_offset;
    assert(input_zero_point >= 0);
    assert(input_zero_point <= 255);
    DilatedIm2col(params, input_zero_point, input_shape, input_data, filter_shape, output_shape,
                  im2col_data);
    gemm_input_data = im2col_data;
    gemm_input_shape = &im2col_shape;
  }
  else if (need_im2col)
  {
    assert(im2col_data);
    const int input_zero_point = -input_offset;
    assert(input_zero_point >= 0);
    assert(input_zero_point <= 255);
    Im2col(params, filter_height, filter_width, input_zero_point, input_shape, input_data,
           im2col_shape, im2col_data);
    gemm_input_data = im2col_data;
    gemm_input_shape = &im2col_shape;
  }
  else
  {
    gemm_input_data = input_data;
    gemm_input_shape = &input_shape;
  }
 
  const int gemm_input_rows = gemm_input_shape->Dims(3);
  // Using FlatSizeSkipDim causes segfault in some contexts (see b/79927784).
  // The root cause has not yet been identified though. Same applies below for
  // the other calls commented out. This is a partial rollback of cl/196819423.
  // const int gemm_input_cols = FlatSizeSkipDim(*gemm_input_shape, 3);
  const int gemm_input_cols =
    gemm_input_shape->Dims(0) * gemm_input_shape->Dims(1) * gemm_input_shape->Dims(2);
  const int filter_rows = filter_shape.Dims(0);
  // See b/79927784.
  // const int filter_cols = FlatSizeSkipDim(filter_shape, 0);
  const int filter_cols = filter_shape.Dims(1) * filter_shape.Dims(2) * filter_shape.Dims(3);
  const int output_rows = output_shape.Dims(3);
  // See b/79927784.
  // const int output_cols = FlatSizeSkipDim(output_shape, 3);
  const int output_cols = output_shape.Dims(0) * output_shape.Dims(1) * output_shape.Dims(2);
  assert(output_rows == filter_rows);
  assert(output_cols == gemm_input_cols);
  assert(filter_cols == gemm_input_rows);
  assert(bias_shape.FlatSize() == output_rows);
  gemmlowp::MatrixMap<const uint8_t, gemmlowp::MapOrder::RowMajor> filter_matrix(
    filter_data, filter_rows, filter_cols);
  gemmlowp::MatrixMap<const uint8_t, gemmlowp::MapOrder::ColMajor> input_matrix(
    gemm_input_data, gemm_input_rows, gemm_input_cols);
  gemmlowp::MatrixMap<uint8_t, gemmlowp::MapOrder::ColMajor> output_matrix(output_data, output_rows,
                                                                           output_cols);
  const auto &output_pipeline =
    GemmlowpOutputPipeline::MakeExp(bias_data, output_rows, output_offset, output_multiplier,
                                    output_shift, output_activation_min, output_activation_max);
 
  std::lock_guard<std::mutex> lock_guard(_gemmlowp_mutex);
  gemmlowp::GemmWithOutputPipeline<uint8_t, uint8_t, gemmlowp::L8R8WithLhsNonzeroBitDepthParams>(
    gemm_context, filter_matrix, input_matrix, &output_matrix, filter_offset, input_offset,
    output_pipeline);
}

Referenced by nnfw::cker::Conv::operator()().

◆ DepthwiseConvImpl() [1/2]

void nnfw::cker::optimized::DepthwiseConvImpl	(	const DepthwiseConvParams &	params,
		const Shape &	input_shape,
		const float *	input_data,
		const Shape &	filter_shape,
		const float *	filter_data,
		const Shape &	bias_shape,
		const float *	bias_data,
		const Shape &	output_shape,
		float *	output_data,
		int	thread_start,
		int	thread_end,
		int	thread_dim
	)

inline

Definition at line 1033 of file DepthwiseConvFloat.h.

{
  const int stride_width = params.stride_width;
  const int stride_height = params.stride_height;
  const int pad_width = params.padding_values.width;
  const int pad_height = params.padding_values.height;
  const int depth_multiplier = params.depth_multiplier;
  const float output_activation_min = params.float_activation_min;
  const float output_activation_max = params.float_activation_max;
  const int dilation_width_factor = params.dilation_width_factor;
  const int dilation_height_factor = params.dilation_height_factor;
  assert(input_shape.DimensionsCount() == 4);
  assert(filter_shape.DimensionsCount() == 4);
  assert(output_shape.DimensionsCount() == 4);
  assert(thread_dim == 0 || thread_dim == 1);
 
  const int batches = MatchingDim(input_shape, 0, output_shape, 0);
  const int output_depth = MatchingDim(filter_shape, 3, output_shape, 3);
  const int input_height = input_shape.Dims(1);
  const int input_width = input_shape.Dims(2);
  const int input_depth = input_shape.Dims(3);
  const int filter_height = filter_shape.Dims(1);
  const int filter_width = filter_shape.Dims(2);
  const int output_height = output_shape.Dims(1);
  const int output_width = output_shape.Dims(2);
  assert(output_depth == input_depth * depth_multiplier);
  assert(bias_shape.FlatSize() == output_depth);
 
  static const int kAccBufferMaxSize = 4832;
  float acc_buffer[kAccBufferMaxSize];
  assert(kAccBufferMaxSize >= output_depth);
  const int kOutputPixelsInAccBuffer = kAccBufferMaxSize / output_depth;
  [[maybe_unused]] const int kAccBufferActualSize = kOutputPixelsInAccBuffer * output_depth;
  assert(kOutputPixelsInAccBuffer * output_depth <= kAccBufferActualSize);
  assert(kAccBufferActualSize <= kAccBufferMaxSize);
  assert(kOutputPixelsInAccBuffer >= 1);
 
  // row_accum_func will point to the core accumulation function to be used
  // for this DepthwiseConv op.
  using row_accum_func_t = decltype(&FloatDepthwiseConvAccumRowGeneric);
  row_accum_func_t row_accum_func = nullptr;
 
#define TFMINI_USE_DEPTHWISECONV_KERNEL(ALLOW_STRIDED, FIXED_INPUT_DEPTH, FIXED_DEPTH_MULTIPLIER) \
  if (!row_accum_func && (stride_width == 1 || ALLOW_STRIDED) &&                                  \
      (input_depth == FIXED_INPUT_DEPTH || FIXED_INPUT_DEPTH == 0) &&                             \
      depth_multiplier == FIXED_DEPTH_MULTIPLIER)                                                 \
  {                                                                                               \
    row_accum_func =                                                                              \
      FloatDepthwiseConvAccumRow<ALLOW_STRIDED, FIXED_INPUT_DEPTH, FIXED_DEPTH_MULTIPLIER>;       \
  }
 
#ifdef USE_NEON
  // We go over our list of kernels by decreasing order of preference
  // for the cases where multiple kernels could apply.
 
  // Start with the fastest kernels: AllowStrided=false, fixed input depth.
 
  TFMINI_USE_DEPTHWISECONV_KERNEL(false, 8, 1)
  TFMINI_USE_DEPTHWISECONV_KERNEL(false, 2, 1)
 
  // Next come the strided kernels: AllowStrided=true, fixed input depth.
  // They are a bit less efficient, but allow stride!=1.
 
  TFMINI_USE_DEPTHWISECONV_KERNEL(true, 8, 1)
  TFMINI_USE_DEPTHWISECONV_KERNEL(true, 1, 8)
  TFMINI_USE_DEPTHWISECONV_KERNEL(true, 1, 20)
  TFMINI_USE_DEPTHWISECONV_KERNEL(true, 1, 32)
  TFMINI_USE_DEPTHWISECONV_KERNEL(true, 2, 1)
  TFMINI_USE_DEPTHWISECONV_KERNEL(true, 3, 2)
  TFMINI_USE_DEPTHWISECONV_KERNEL(true, 3, 4)
  TFMINI_USE_DEPTHWISECONV_KERNEL(true, 4, 1)
 
  // Finally, the kernels allowing a variable input depth,
  // these are the least efficient but most general kernels.
 
  TFMINI_USE_DEPTHWISECONV_KERNEL(true, 0, 1)
  TFMINI_USE_DEPTHWISECONV_KERNEL(true, 0, 2)
  TFMINI_USE_DEPTHWISECONV_KERNEL(true, 0, 8)
  TFMINI_USE_DEPTHWISECONV_KERNEL(true, 0, 16)
 
#endif // USE_NEON
 
#undef TFMINI_USE_DEPTHWISECONV_KERNEL
 
  // No matching fast kernel found, use slow fallback.
  if (!row_accum_func)
  {
    row_accum_func = FloatDepthwiseConvAccumRowGeneric;
  }
 
  const int input_height_stride = input_shape.Dims(3) * input_shape.Dims(2);
  const int input_batch_stride = input_height_stride * input_shape.Dims(1);
  const int filter_height_stride = filter_shape.Dims(3) * filter_shape.Dims(2);
 
  // Now that we have determined row_accum_func, we can start work.
  int batch_start = 0;
  int batch_end = batches;
  int row_start = 0;
  int row_end = output_height;
  int output_ptr_offset = 0;
 
  switch (thread_dim)
  {
    case 0:
      // Multithread along with the batch axis
      assert(thread_start >= 0);
      assert(thread_end <= batches);
      batch_start = thread_start;
      batch_end = thread_end;
      output_ptr_offset = batch_start * FlatSizeSkipDim(output_shape, 0);
      break;
    case 1:
      // Multithread along with the row axis
      assert(thread_start >= 0);
      assert(thread_end <= output_height);
      row_start = thread_start;
      row_end = thread_end;
      output_ptr_offset = row_start * output_width * output_depth;
      break;
  }
 
  float *output_ptr = output_data + output_ptr_offset;
  int batch_step = (output_height + row_start - row_end) * output_width * output_depth;
 
  for (int b = batch_start; b < batch_end; ++b)
  {
    for (int out_y = row_start; out_y < row_end; ++out_y)
    {
      const int in_y_origin = (out_y * stride_height) - pad_height;
      const int filter_y_start =
        std::max(0, (-in_y_origin + dilation_height_factor - 1) / dilation_height_factor);
      const int filter_y_end =
        std::min(filter_height, (input_height - in_y_origin + dilation_height_factor - 1) /
                                  dilation_height_factor);
      for (int out_x_buffer_start = 0; out_x_buffer_start < output_width;
           out_x_buffer_start += kOutputPixelsInAccBuffer)
      {
        const int out_x_buffer_end =
          std::min(output_width, out_x_buffer_start + kOutputPixelsInAccBuffer);
        // We call a 'pixel' a group of activation that share all but the
        // 'depth'/'channel' coordinate. num_output_pixels is the number of
        // output pixels that we will accumulate in this loop iteration.
        const int num_output_pixels = out_x_buffer_end - out_x_buffer_start;
        // Initialize our local accumulator with the bias values, so we don't
        // have to add them later.
        DepthwiseConvInitAccBuffer(num_output_pixels, output_depth, bias_data, acc_buffer);
        // Accumulation loop. Most of the time should be spent in here.
        for (int filter_y = filter_y_start; filter_y < filter_y_end; ++filter_y)
        {
          const int in_y = in_y_origin + dilation_height_factor * filter_y;
          row_accum_func(stride_width, dilation_width_factor, input_depth, input_width,
                         input_data + in_y * input_height_stride + b * input_batch_stride,
                         pad_width, depth_multiplier, filter_width,
                         filter_data + filter_y * filter_height_stride, out_x_buffer_start,
                         out_x_buffer_end, output_depth, acc_buffer);
        }
        // Finished accumulating. Now store to destination.
        const int num_output_values = output_depth * num_output_pixels;
        int i = 0;
// TODO(benoitjacob) optimized code goes here
#ifdef USE_NEON
        // Handle 16 values at a time
        for (; i <= num_output_values - 16; i += 16)
        {
          float32x4_t acc[4];
          for (int k = 0; k < 4; k++)
          {
            acc[k] = vld1q_f32(acc_buffer + i + 4 * k);
          }
          for (int k = 0; k < 4; k++)
          {
            acc[k] = vmaxq_f32(vdupq_n_f32(output_activation_min),
                               vminq_f32(vdupq_n_f32(output_activation_max), acc[k]));
          }
          for (int k = 0; k < 4; k++)
          {
            vst1q_f32(output_ptr + 4 * k, acc[k]);
          }
          output_ptr += 16;
        }
        // Handle 4 values at a time
        for (; i <= num_output_values - 4; i += 4)
        {
          float32x4_t acc = vld1q_f32(acc_buffer + i);
 
          acc = vmaxq_f32(vdupq_n_f32(output_activation_min),
                          vminq_f32(vdupq_n_f32(output_activation_max), acc));
 
          vst1q_f32(output_ptr, acc);
          output_ptr += 4;
        }
#endif
        // Handle leftover values, one by one. This is very slow.
        for (; i < num_output_values; i++)
        {
          float acc = acc_buffer[i];
          acc = std::max(output_activation_min, std::min(output_activation_max, acc));
 
          *output_ptr++ = acc;
        }
      }
    }
    output_ptr += batch_step;
  }
}

Referenced by nnfw::cker::DepthwiseConv(), and nnfw::cker::DepthwiseConvWorkerTask< T, TS >::Run().

◆ DepthwiseConvImpl() [2/2]

void nnfw::cker::optimized::DepthwiseConvImpl	(	const DepthwiseConvParams &	params,
		const Shape &	input_shape,
		const uint8_t *	input_data,
		const Shape &	filter_shape,
		const uint8_t *	filter_data,
		const Shape &	bias_shape,
		const int32_t *	bias_data,
		const Shape &	output_shape,
		uint8_t *	output_data,
		int	thread_start,
		int	thread_end,
		int	thread_dim
	)

inline

Definition at line 2231 of file DepthwiseConvUint8.h.

{
  return DepthwiseConvWithRounding(params, input_shape, input_data, filter_shape, filter_data,
                                   bias_shape, bias_data, output_shape, output_data, thread_start,
                                   thread_end, thread_dim);
}

References DepthwiseConvWithRounding(), and output_shape.

◆ DepthwiseConvInitAccBuffer()

void nnfw::cker::optimized::DepthwiseConvInitAccBuffer	(	int	num_output_pixels,
		int	output_depth,
		const float *	bias_data,
		float *	acc_buffer
	)

inline

Definition at line 1016 of file DepthwiseConvFloat.h.

{
  // TODO(benoitjacob): This might need optimized specializations
  // for small output_depth values, if that ever becomes an important
  // case (like it was for some quantized DepthwiseConv cases).
  for (int i = 0; i < num_output_pixels; i++)
  {
    memcpy(acc_buffer + i * output_depth, bias_data, sizeof(acc_buffer[0]) * output_depth);
  }
}

Referenced by DepthwiseConvImpl().

◆ DepthwiseConvWithRounding()

void nnfw::cker::optimized::DepthwiseConvWithRounding	(	const DepthwiseConvParams &	params,
		const Shape &	input_shape,
		const uint8_t *	input_data,
		const Shape &	filter_shape,
		const uint8_t *	filter_data,
		const Shape &	bias_shape,
		const int32_t *	bias_data,
		const Shape &	output_shape,
		uint8_t *	output_data,
		int	thread_start,
		int	thread_end,
		int	thread_dim
	)

inline

Definition at line 2153 of file DepthwiseConvUint8.h.

{
  [[maybe_unused]] const int depth_multiplier = params.depth_multiplier;
  [[maybe_unused]] const int32_t output_activation_min = params.quantized_activation_min;
  [[maybe_unused]] const int32_t output_activation_max = params.quantized_activation_max;
  [[maybe_unused]] const int dilation_width_factor = params.dilation_width_factor;
  [[maybe_unused]] const int dilation_height_factor = params.dilation_height_factor;
  assert(dilation_width_factor >= 1);
  assert(dilation_height_factor >= 1);
  assert(input_shape.DimensionsCount() == 4);
  assert(filter_shape.DimensionsCount() == 4);
  assert(output_shape.DimensionsCount() == 4);
  assert(output_activation_min <= output_activation_max);
  [[maybe_unused]] const int output_depth = MatchingDim(filter_shape, 3, output_shape, 3);
  [[maybe_unused]] const int input_depth = input_shape.Dims(3);
  assert(output_depth == input_depth * depth_multiplier);
  assert(bias_shape.FlatSize() == output_depth);
 
// Enable for arm64 except for the Nvidia Linux 4 Tegra (L4T) running on
// Jetson TX-2. This compiler does not support the offsetof() macro.
#if defined(__aarch64__) && !defined(GOOGLE_L4T)
//  TODO Use below codes
//  // Dispatch to dot-product 3x3 kernels when supported.
//
//  ruy::Context *ruy_context = cpu_backend_context->ruy_context();
//  const bool has_dot_product_instructions =
//      ruy_context != nullptr &&
//      (ruy_context->GetRuntimeEnabledPaths() & ruy::Path::kNeonDotprod) != ruy::Path::kNone;
//  if (has_dot_product_instructions)
//  {
//    using optimized_ops::depthwise_conv::DotProduct3x3KernelType;
//    DotProduct3x3KernelType kernel_type =
//    optimized_ops::depthwise_conv::CategorizeDotProductKernel(
//        input_shape, filter_shape, params);
//    if (kernel_type != DotProduct3x3KernelType::kNone)
//    {
//      optimized_ops::depthwise_conv::DepthwiseConvDotProduct3x3<
//          DepthwiseConvImplementation::kUseNeon3x3DotProduct>(params, input_shape, input_data,
//                                                              filter_shape, filter_data,
//                                                              bias_shape,
//                                                              bias_data, output_shape,
//                                                              output_data);
//      return;
//    }
//  }
//
//  // Dispatch to non-dot-product 3x3 kernels when supported.
//
//  const int stride_width = params.stride_width;
//  const int stride_height = params.stride_height;
//  const int pad_width = params.padding_values.width;
//  const int pad_height = params.padding_values.height;
//  const int output_shift = params.output_shift;
//
//  // Call kernel optimized for depthwise convolutions using 3x3 filters if
//  // parameters are supported.
//  if (depthwise_conv::Fast3x3FilterKernelSupported(input_shape, filter_shape, stride_width,
//                                                   stride_height, dilation_width_factor,
//                                                   dilation_height_factor, pad_width, pad_height,
//                                                   depth_multiplier, output_shape, output_shift))
//  {
//    depthwise_conv::DepthwiseConv3x3Filter<kOutputRounding>(
//        params, input_shape, input_data, filter_shape, filter_data, bias_shape, bias_data,
//        output_shape, output_data, thread_start, thread_end, thread_dim);
//    return;
//  }
#endif
 
  depthwise_conv::DepthwiseConvGeneral(params, input_shape, input_data, filter_shape, filter_data,
                                       bias_shape, bias_data, output_shape, output_data,
                                       thread_start, thread_end, thread_dim);
}

References nnfw::cker::DepthwiseConvParams::depth_multiplier, nnfw::cker::optimized::depthwise_conv::DepthwiseConvGeneral(), nnfw::cker::DepthwiseConvParams::dilation_height_factor, nnfw::cker::DepthwiseConvParams::dilation_width_factor, nnfw::cker::Shape::DimensionsCount(), nnfw::cker::Shape::Dims(), nnfw::cker::Shape::FlatSize(), nnfw::cker::MatchingDim(), output_shape, nnfw::cker::DepthwiseConvParams::quantized_activation_max, and nnfw::cker::DepthwiseConvParams::quantized_activation_min.

Referenced by DepthwiseConvImpl().

◆ DilatedIm2col() [1/2]

template<typename T >

void nnfw::cker::optimized::DilatedIm2col	(	const ConvParams &	params,
		const Shape &	input_shape,
		const T *	input_data,
		const Shape &	filter_shape,
		const Shape &	output_shape,
		T *	im2col_data,
		const int32_t *	zero_bytes,
		const int	zero_bytes_len
	)

Definition at line 121 of file OptimizedUtils.h.

{
  const int stride_width = params.stride_width;
  const int stride_height = params.stride_height;
  const int dilation_width_factor = params.dilation_width_factor;
  const int dilation_height_factor = params.dilation_height_factor;
  const int pad_width = params.padding_values.width;
  const int pad_height = params.padding_values.height;
  assert(input_shape.DimensionsCount() == 4);
  assert(filter_shape.DimensionsCount() == 4);
  assert(output_shape.DimensionsCount() == 4);
 
  // For dilated convolution, the input pixels are not contiguous therefore we
  // can't use the same optimizations as Im2Col(). Though note this code would
  // work fine for the non-dilated case too (though likely a bit slower).
  assert(dilation_width_factor != 1 || dilation_height_factor != 1);
  assert(im2col_data);
  const int batches = MatchingDim(input_shape, 0, output_shape, 0);
  const int input_height = input_shape.Dims(1);
  const int input_width = input_shape.Dims(2);
  const int input_depth = MatchingDim(input_shape, 3, filter_shape, 3);
  const int filter_height = filter_shape.Dims(1);
  const int filter_width = filter_shape.Dims(2);
  const int output_height = output_shape.Dims(1);
  const int output_width = output_shape.Dims(2);
  MatchingDim(output_shape, 3, filter_shape, 0);
 
  // Construct the MxN sized im2col matrix.
  // The rows M, are sub-ordered B x H x W
  const Shape row_shape({1, batches, output_height, output_width});
  // The columns, N, are sub-ordered Kh x Kw x Din
  const Shape col_shape({1, filter_height, filter_width, input_depth});
  // Use dimensions M and N to construct dims for indexing directly into im2col
  const Shape im2col_shape({1, 1, row_shape.FlatSize(), col_shape.FlatSize()});
 
  // Loop through the output rows (B x H x W)
  for (int batch = 0; batch < batches; ++batch)
  {
    const T zero_byte =
      zero_bytes_len > 1 ? static_cast<T>(zero_bytes[batch]) : static_cast<T>(zero_bytes[0]);
    for (int out_y = 0; out_y < output_height; ++out_y)
    {
      for (int out_x = 0; out_x < output_width; ++out_x)
      {
        // Each im2col row is an output pixel. Arrange the input data in this
        // row in an order we can conveniently multiply with the filter data.
        int row_offset = Offset(row_shape, 0, batch, out_y, out_x);
        const int in_x_origin = (out_x * stride_width) - pad_width;
        const int in_y_origin = (out_y * stride_height) - pad_height;
        // Loop through all the pixels of the filter (Kh x Kw)
        for (int filter_y = 0; filter_y < filter_height; ++filter_y)
        {
          const int in_y = in_y_origin + dilation_height_factor * filter_y;
          if ((in_y >= 0) && (in_y < input_height))
          {
            // Filter row is within the input data.
            // Loop through all the filter pixels in this row.
            for (int filter_x = 0; filter_x < filter_width; ++filter_x)
            {
              const int in_x = in_x_origin + dilation_width_factor * filter_x;
              int col_offset = Offset(col_shape, 0, filter_y, filter_x, 0);
              T *dst = im2col_data + Offset(im2col_shape, 0, 0, row_offset, col_offset);
              if ((in_x >= 0) && (in_x < input_width))
              {
                // Filter pixel is within the input, copy the input data.
                T const *src = input_data + Offset(input_shape, batch, in_y, in_x, 0);
                memcpy(dst, src, input_depth * sizeof(T));
              }
              else
              {
                // Filter pixel is outside the input, zero it out.
                memset(dst, zero_byte, input_depth * sizeof(T));
              }
            }
          }
          else
          {
            // Filter row is outside the input, zero out the entire filter row.
            int col_offset = Offset(col_shape, 0, filter_y, 0, 0);
            T *dst = im2col_data + Offset(im2col_shape, 0, 0, row_offset, col_offset);
            memset(dst, zero_byte, filter_width * input_depth * sizeof(T));
          }
        }
      }
    }
  }
}

References nnfw::cker::ConvParams::dilation_height_factor, nnfw::cker::ConvParams::dilation_width_factor, nnfw::cker::Shape::DimensionsCount(), nnfw::cker::Shape::Dims(), nnfw::cker::PaddingValues::height, nnfw::cker::MatchingDim(), nnfw::cker::Offset(), output_shape, nnfw::cker::ConvParams::padding_values, nnfw::cker::ConvParams::stride_height, nnfw::cker::ConvParams::stride_width, and nnfw::cker::PaddingValues::width.

Referenced by Conv().

◆ DilatedIm2col() [2/2]

template<typename T >

void nnfw::cker::optimized::DilatedIm2col	(	const ConvParams &	params,
		uint8_t	zero_byte,
		const Shape &	input_shape,
		const T *	input_data,
		const Shape &	filter_shape,
		const Shape &	output_shape,
		T *	im2col_data
	)

Definition at line 212 of file OptimizedUtils.h.

{
  const int32_t zero_point = static_cast<int32_t>(zero_byte);
  DilatedIm2col<T>(params, input_shape, input_data, filter_shape, output_shape, im2col_data,
                   &zero_point, 1);
}

References output_shape.

◆ Div()

void nnfw::cker::optimized::Div	(	const BinaryArithmeticOpParam &	params,
		const Shape &	input1_shape,
		const float *	input1_data,
		const Shape &	input2_shape,
		const float *	input2_data,
		const Shape &	output_shape,
		float *	output_data
	)

inline

Definition at line 1218 of file BinaryArithmeticOps.h.

{
#ifdef __aarch64__
  const int flat_size = MatchingElementsSize(input1_shape, input2_shape, output_shape);
  auto implFuncs = getBinaryOpWithActivationImplFloat<BinaryOpFuncDivFloat>(params);
  (*implFuncs.first)(flat_size, params, input1_data, input2_data, output_data);
#else
  const std::function<float(const float &, const float &)> fn =
    [](const float &a, const float &b) -> float { return a / b; };
  reference::BinaryArithmeticOp(params, input1_shape, input1_data, input2_shape, input2_data,
                                output_shape, output_data, fn);
#endif // __aarch64__
}

References nnfw::cker::reference::BinaryArithmeticOp(), nnfw::cker::MatchingElementsSize(), and output_shape.

Referenced by nnfw::cker::BinaryArithmeticOp().

◆ ExtractPatchIntoBufferColumn()

template<typename T >

void nnfw::cker::optimized::ExtractPatchIntoBufferColumn	(	const Shape &	input_shape,
		int	w,
		int	h,
		int	b,
		int	kheight,
		int	kwidth,
		int	stride_width,
		int	stride_height,
		int	pad_width,
		int	pad_height,
		int	in_width,
		int	in_height,
		int	in_depth,
		int	single_buffer_length,
		int	buffer_id,
		const T *	in_data,
		T *	conv_buffer_data,
		uint8_t	zero_byte
	)

inline

Definition at line 34 of file OptimizedUtils.h.

{
  assert(input_shape.DimensionsCount() == 4);
  // This chunk of code reshapes all the inputs corresponding to
  // output (b, h, w) to a column vector in conv_buffer(:, buffer_id).
  const int kwidth_times_indepth = kwidth * in_depth;
  const int inwidth_times_indepth = in_width * in_depth;
  const int ih_ungated_start = h * stride_height - pad_height;
  const int ih_ungated_end = (ih_ungated_start + kheight);
  const int ih_end = std::min(ih_ungated_end, in_height);
  const int iw_ungated_start = w * stride_width - pad_width;
  const int iw_ungated_end = (iw_ungated_start + kwidth);
  const int iw_end = std::min(iw_ungated_end, in_width);
  // If the patch is off the edge of the input image, skip writing those rows
  // and columns from the patch into the output array.
  const int h_offset = std::max(0, -ih_ungated_start);
  const int w_offset = std::max(0, -iw_ungated_start);
  const int ih_start = std::max(0, ih_ungated_start);
  const int iw_start = std::max(0, iw_ungated_start);
  const int single_row_num = std::min(kwidth - w_offset, in_width - iw_start) * in_depth;
  const int output_row_offset = (buffer_id * single_buffer_length);
  int out_offset = output_row_offset + (h_offset * kwidth + w_offset) * in_depth;
  int in_offset = Offset(input_shape, b, ih_start, iw_start, 0);
 
  // Express all of the calculations as padding around the input patch.
  const int top_padding = h_offset;
  const int bottom_padding = (ih_ungated_end - ih_end);
  const int left_padding = w_offset;
  const int right_padding = (iw_ungated_end - iw_end);
  assert(single_row_num == ((kwidth - (left_padding + right_padding)) * in_depth));
 
  // Write out zeroes to the elements representing the top rows of the input
  // patch that are off the edge of the input image.
  if (top_padding > 0)
  {
    const int top_row_elements = (top_padding * kwidth * in_depth);
    memset(conv_buffer_data + output_row_offset, zero_byte, (top_row_elements * sizeof(T)));
  }
 
  // If the patch is on the interior of the input image horizontally, just copy
  // over the rows sequentially, otherwise add zero padding at the start or end.
  if ((left_padding == 0) && (right_padding == 0))
  {
    for (int ih = ih_start; ih < ih_end; ++ih)
    {
      memcpy(conv_buffer_data + out_offset, in_data + in_offset, single_row_num * sizeof(T));
      out_offset += kwidth_times_indepth;
      in_offset += inwidth_times_indepth;
    }
  }
  else
  {
    for (int ih = ih_start; ih < ih_end; ++ih)
    {
      if (left_padding > 0)
      {
        const int left_start = (out_offset - (left_padding * in_depth));
        memset(conv_buffer_data + left_start, zero_byte, (left_padding * in_depth * sizeof(T)));
      }
      memcpy(conv_buffer_data + out_offset, in_data + in_offset, single_row_num * sizeof(T));
      if (right_padding > 0)
      {
        const int right_start = (out_offset + single_row_num);
        memset(conv_buffer_data + right_start, zero_byte, (right_padding * in_depth * sizeof(T)));
      }
      out_offset += kwidth_times_indepth;
      in_offset += inwidth_times_indepth;
    }
  }
 
  // If the bottom of the patch falls off the input image, pad the values
  // representing those input rows with zeroes.
  if (bottom_padding > 0)
  {
    const int bottom_row_elements = (bottom_padding * kwidth * in_depth);
    const int bottom_start =
      output_row_offset + ((top_padding + (ih_end - ih_start)) * kwidth * in_depth);
    memset(conv_buffer_data + bottom_start, zero_byte, (bottom_row_elements * sizeof(T)));
  }
}

References nnfw::cker::Shape::DimensionsCount(), and nnfw::cker::Offset().

Referenced by Im2col().

◆ FloatDepthwiseConvAccumRow()

template<bool kAllowStrided, int kFixedInputDepth, int kFixedDepthMultiplier>

void nnfw::cker::optimized::FloatDepthwiseConvAccumRow	(	int	stride,
		int	dilation_factor,
		int	input_depth,
		int	input_width,
		const float *	input_data,
		int	pad_width,
		int	depth_multiplier,
		int	filter_width,
		const float *	filter_data,
		int	out_x_buffer_start,
		int	out_x_buffer_end,
		int	output_depth,
		float *	acc_buffer
	)

Definition at line 908 of file DepthwiseConvFloat.h.

{
  // Sanity check parameters. This is important in particular to ensure
  // that we keep the number of template instantiations minimal, so we don't
  // increase binary size unnecessarily.
  static_assert(kFixedDepthMultiplier || !kFixedInputDepth, "");
  static_assert(kFixedInputDepth || kAllowStrided, "");
  assert(stride == 1 || kAllowStrided);
  if (kFixedInputDepth)
  {
    assert(input_depth == kFixedInputDepth);
  }
  if (kFixedDepthMultiplier)
  {
    assert(depth_multiplier == kFixedDepthMultiplier);
  }
  assert(output_depth == input_depth * depth_multiplier);
  const int input_ptr_increment = stride * input_depth;
  const float *filter_base_ptr = filter_data;
  for (int filter_x = 0; filter_x < filter_width; ++filter_x)
  {
    // For the current (filter_x, filter_y) point in the filter,
    // compute the boundaries of the corresponding output row segment.
    int out_x_loop_start_unclamped = 0;
    int out_x_loop_end_unclamped = 0;
    if (kAllowStrided)
    {
      if (stride == 2)
      {
        out_x_loop_start_unclamped = (pad_width - dilation_factor * filter_x + 1) / 2;
        out_x_loop_end_unclamped = (pad_width + input_width - dilation_factor * filter_x + 1) / 2;
      }
      else if (stride == 4)
      {
        out_x_loop_start_unclamped = (pad_width - dilation_factor * filter_x + 3) / 4;
        out_x_loop_end_unclamped = (pad_width + input_width - dilation_factor * filter_x + 3) / 4;
      }
      else
      {
        out_x_loop_start_unclamped = (pad_width - dilation_factor * filter_x + stride - 1) / stride;
        out_x_loop_end_unclamped =
          (pad_width + input_width - dilation_factor * filter_x + stride - 1) / stride;
      }
    }
    else
    {
      out_x_loop_start_unclamped = pad_width - dilation_factor * filter_x;
      out_x_loop_end_unclamped = pad_width + input_width - dilation_factor * filter_x;
    }
    // The kernel will have to iterate on the segment of the
    // output row that starts at out_x_loop_start and out_x_loop_end.
    const int out_x_loop_start = std::max(out_x_buffer_start, out_x_loop_start_unclamped);
    const int out_x_loop_end = std::min(out_x_buffer_end, out_x_loop_end_unclamped);
 
    float *acc_buffer_ptr = acc_buffer + (out_x_loop_start - out_x_buffer_start) * output_depth;
    const int in_x_origin = (out_x_loop_start * stride) - pad_width + dilation_factor * filter_x;
    const float *input_ptr = input_data + in_x_origin * input_depth;
    const int num_output_pixels = out_x_loop_end - out_x_loop_start;
    FloatDepthwiseConvKernel<kAllowStrided, kFixedInputDepth, kFixedDepthMultiplier>::Run(
      num_output_pixels, input_depth, depth_multiplier, input_ptr, input_ptr_increment,
      filter_base_ptr, acc_buffer_ptr);
    filter_base_ptr += output_depth;
  }
}

◆ FloatDepthwiseConvAccumRowGeneric()

void nnfw::cker::optimized::FloatDepthwiseConvAccumRowGeneric	(	int	stride,
		int	dilation_factor,
		int	input_depth,
		int	input_width,
		const float *	input_data,
		int	pad_width,
		int	depth_multiplier,
		int	filter_width,
		const float *	filter_data,
		int	out_x_buffer_start,
		int	out_x_buffer_end,
		int	output_depth,
		float *	acc_buffer
	)

inline

Definition at line 977 of file DepthwiseConvFloat.h.

{
  const float *filter_base_ptr = filter_data;
  for (int filter_x = 0; filter_x < filter_width; ++filter_x)
  {
    const int out_x_loop_start =
      std::max(out_x_buffer_start, (pad_width - dilation_factor * filter_x + stride - 1) / stride);
    const int out_x_loop_end =
      std::min(out_x_buffer_end,
               (pad_width + input_width - dilation_factor * filter_x + stride - 1) / stride);
 
    float *acc_buffer_ptr = acc_buffer + (out_x_loop_start - out_x_buffer_start) * output_depth;
    const int in_x_origin = (out_x_loop_start * stride) - pad_width + dilation_factor * filter_x;
    const float *input_ptr = input_data + in_x_origin * input_depth;
    const int input_ptr_increment = (stride - 1) * input_depth;
    for (int out_x = out_x_loop_start; out_x < out_x_loop_end; out_x++)
    {
      const float *filter_ptr = filter_base_ptr;
      for (int ic = 0; ic < input_depth; ++ic)
      {
        const float input_val = *input_ptr++;
        for (int m = 0; m < depth_multiplier; m++)
        {
          const float filter_val = *filter_ptr++;
          *acc_buffer_ptr++ += filter_val * input_val;
        }
      }
      input_ptr += input_ptr_increment;
    }
    filter_base_ptr += output_depth;
  }
}

References m.

Referenced by DepthwiseConvImpl().

◆ getBinaryOpWithActivationImplFloat()

template<class FUNC >

BinaryOpImplFloatFuncs nnfw::cker::optimized::getBinaryOpWithActivationImplFloat ( const BinaryArithmeticOpParam & params )

inline

Definition at line 676 of file BinaryArithmeticOps.h.

{
  if (params.float_activation_max == std::numeric_limits<float>::max())
    if (params.float_activation_min == std::numeric_limits<float>::lowest())
      return BinaryOpImplFloatFuncs(BinaryOpElementwise<FUNC, BinaryOpActivationFloatNone>,
                                    BinaryOpScalarBroadcast<FUNC, BinaryOpActivationFloatNone>);
    else
      return BinaryOpImplFloatFuncs(BinaryOpElementwise<FUNC, BinaryOpActivationFloatMax>,
                                    BinaryOpScalarBroadcast<FUNC, BinaryOpActivationFloatMax>);
  else
    return BinaryOpImplFloatFuncs(BinaryOpElementwise<FUNC, BinaryOpActivationFloatMinMax>,
                                  BinaryOpScalarBroadcast<FUNC, BinaryOpActivationFloatMinMax>);
}

References nnfw::cker::BinaryArithmeticOpParam::float_activation_max, and nnfw::cker::BinaryArithmeticOpParam::float_activation_min.

◆ Im2col()

template<typename T >

void nnfw::cker::optimized::Im2col	(	const ConvParams &	params,
		int	kheight,
		int	kwidth,
		uint8_t	zero_byte,
		const Shape &	input_shape,
		const T *	input_data,
		const Shape &	output_shape,
		T *	output_data
	)

Definition at line 222 of file OptimizedUtils.h.

{
  const int stride_width = params.stride_width;
  const int stride_height = params.stride_height;
  const int pad_width = params.padding_values.width;
  const int pad_height = params.padding_values.height;
  assert(input_shape.DimensionsCount() == 4);
  assert(output_shape.DimensionsCount() == 4);
 
  const int batches = MatchingDim(input_shape, 0, output_shape, 0);
  const int input_depth = input_shape.Dims(3);
  const int input_width = input_shape.Dims(2);
  const int input_height = input_shape.Dims(1);
  const int output_depth = output_shape.Dims(3);
  const int output_width = output_shape.Dims(2);
  const int output_height = output_shape.Dims(1);
 
  int buffer_id = 0;
  // Loop over the output nodes.
  for (int b = 0; b < batches; ++b)
  {
    for (int h = 0; h < output_height; ++h)
    {
      for (int w = 0; w < output_width; ++w)
      {
        ExtractPatchIntoBufferColumn(input_shape, w, h, b, kheight, kwidth, stride_width,
                                     stride_height, pad_width, pad_height, input_width,
                                     input_height, input_depth, output_depth, buffer_id, input_data,
                                     output_data, zero_byte);
        ++buffer_id;
      }
    }
  }
}

References nnfw::cker::Shape::DimensionsCount(), nnfw::cker::Shape::Dims(), ExtractPatchIntoBufferColumn(), nnfw::cker::PaddingValues::height, nnfw::cker::MatchingDim(), output_shape, nnfw::cker::ConvParams::padding_values, nnfw::cker::ConvParams::stride_height, nnfw::cker::ConvParams::stride_width, and nnfw::cker::PaddingValues::width.

Referenced by Conv().

◆ Mul() [1/2]

void nnfw::cker::optimized::Mul	(	const BinaryArithmeticOpParam &	params,
		const Shape &	input1_shape,
		const float *	input1_data,
		const Shape &	input2_shape,
		const float *	input2_data,
		const Shape &	output_shape,
		float *	output_data
	)

inline

Definition at line 1072 of file BinaryArithmeticOps.h.

{
  const int flat_size = MatchingElementsSize(input1_shape, input2_shape, output_shape);
  auto implFuncs = getBinaryOpWithActivationImplFloat<BinaryOpFuncMulFloat>(params);
  (*implFuncs.first)(flat_size, params, input1_data, input2_data, output_data);
}

References nnfw::cker::MatchingElementsSize(), and output_shape.

◆ Mul() [2/2]

template<typename T >

std::enable_if_t< is_quant8< T >::value > nnfw::cker::optimized::Mul	(	const BinaryArithmeticOpParam &	params,
		const Shape &	input1_shape,
		const T *	input1_data,
		const Shape &	input2_shape,
		const T *	input2_data,
		const Shape &	output_shape,
		T *	output_data
	)

inline

Definition at line 1065 of file BinaryArithmeticOps.h.

{
  const int flat_size = MatchingElementsSize(input1_shape, input2_shape, output_shape);
  MulElementwise(flat_size, params, input1_data, input2_data, output_data);
}

References nnfw::cker::MatchingElementsSize(), MulElementwise(), and output_shape.

Referenced by nnfw::cker::BinaryArithmeticOp(), and nnfw::cker::BinaryArithmeticOp().

◆ MulElementwise() [1/2]

void nnfw::cker::optimized::MulElementwise	(	int	size,
		const BinaryArithmeticOpParam &	params,
		const int8_t *	input1_data,
		const int8_t *	input2_data,
		int8_t *	output_data
	)

inline

Definition at line 974 of file BinaryArithmeticOps.h.

{
  int i = 0;
#ifdef USE_NEON
  const int16x8_t input1_offset_vector = vdupq_n_s16(params.input1_offset);
  const int16x8_t input2_offset_vector = vdupq_n_s16(params.input2_offset);
  const int16x8_t output_offset_vector = vdupq_n_s16(params.output_offset);
  const auto output_activation_min_vector = vdupq_n_s8(params.quantized_activation_min);
  const auto output_activation_max_vector = vdupq_n_s8(params.quantized_activation_max);
  const int left_shift = std::max(0, params.output_shift);
  const int right_shift = std::max(0, -params.output_shift);
  const int32x4_t left_shift_vec = vdupq_n_s32(left_shift);
  for (; i <= size - 16; i += 16)
  {
    // We load / store 16 at a time, multiplying as four sets of 4 int32s.
    const int8x16_t input1_val_original = vld1q_s8(input1_data + i);
    const int8x16_t input2_val_original = vld1q_s8(input2_data + i);
 
    const int16x8_t input1_val_s16_high = vmovl_s8(vget_high_s8(input1_val_original));
    const int16x8_t input1_val_s16_low = vmovl_s8(vget_low_s8(input1_val_original));
 
    const int16x8_t input2_val_s16_high = vmovl_s8(vget_high_s8(input2_val_original));
    const int16x8_t input2_val_s16_low = vmovl_s8(vget_low_s8(input2_val_original));
    const int16x8_t input1_val_high = vaddq_s16(input1_val_s16_high, input1_offset_vector);
    const int16x8_t input2_val_high = vaddq_s16(input2_val_s16_high, input2_offset_vector);
    const int16x8_t input1_val_low = vaddq_s16(input1_val_s16_low, input1_offset_vector);
    const int16x8_t input2_val_low = vaddq_s16(input2_val_s16_low, input2_offset_vector);
    const int16x4_t input1_val_high_high = vget_high_s16(input1_val_high);
    const int16x4_t input1_val_high_low = vget_low_s16(input1_val_high);
    const int16x4_t input1_val_low_high = vget_high_s16(input1_val_low);
    const int16x4_t input1_val_low_low = vget_low_s16(input1_val_low);
    const int16x4_t input2_val_high_high = vget_high_s16(input2_val_high);
    const int16x4_t input2_val_high_low = vget_low_s16(input2_val_high);
    const int16x4_t input2_val_low_high = vget_high_s16(input2_val_low);
    const int16x4_t input2_val_low_low = vget_low_s16(input2_val_low);
 
    auto p1 = vmull_s16(input2_val_high_high, input1_val_high_high);
    auto p2 = vmull_s16(input2_val_high_low, input1_val_high_low);
    auto p3 = vmull_s16(input2_val_low_high, input1_val_low_high);
    auto p4 = vmull_s16(input2_val_low_low, input1_val_low_low);
 
    p1 = vshlq_s32(p1, left_shift_vec);
    p2 = vshlq_s32(p2, left_shift_vec);
    p3 = vshlq_s32(p3, left_shift_vec);
    p4 = vshlq_s32(p4, left_shift_vec);
 
    p1 = vqrdmulhq_n_s32(p1, params.output_multiplier);
    p2 = vqrdmulhq_n_s32(p2, params.output_multiplier);
    p3 = vqrdmulhq_n_s32(p3, params.output_multiplier);
    p4 = vqrdmulhq_n_s32(p4, params.output_multiplier);
    using gemmlowp::RoundingDivideByPOT;
    p1 = RoundingDivideByPOT(p1, right_shift);
    p2 = RoundingDivideByPOT(p2, right_shift);
    p3 = RoundingDivideByPOT(p3, right_shift);
    p4 = RoundingDivideByPOT(p4, right_shift);
 
    const auto p1_narrowed = vqmovn_s32(p1);
    const auto p2_narrowed = vqmovn_s32(p2);
    const auto p3_narrowed = vqmovn_s32(p3);
    const auto p4_narrowed = vqmovn_s32(p4);
 
    const int16x8_t p_part1 =
      vaddq_s16(vcombine_s16(p2_narrowed, p1_narrowed), output_offset_vector);
    const int16x8_t p_part2 =
      vaddq_s16(vcombine_s16(p4_narrowed, p3_narrowed), output_offset_vector);
    const int8x16_t p = vcombine_s8(vqmovn_s16(p_part2), vqmovn_s16(p_part1));
 
    const auto clamped =
      vmaxq_s8(output_activation_min_vector, vminq_s8(output_activation_max_vector, p));
    vst1q_s8(output_data + i, clamped);
  }
#endif // NEON
 
  for (; i < size; ++i)
  {
    const int32_t input1_val = params.input1_offset + input1_data[i];
    const int32_t input2_val = params.input2_offset + input2_data[i];
    const int32_t unclamped_result =
      params.output_offset + MultiplyByQuantizedMultiplier(input1_val * input2_val,
                                                           params.output_multiplier,
                                                           params.output_shift);
    const int32_t clamped_output = std::min(
      params.quantized_activation_max, std::max(params.quantized_activation_min, unclamped_result));
    output_data[i] = static_cast<int8_t>(clamped_output);
  }
}

References nnfw::cker::BinaryArithmeticOpParam::input1_offset, nnfw::cker::BinaryArithmeticOpParam::input2_offset, nnfw::cker::MultiplyByQuantizedMultiplier(), nnfw::cker::BinaryArithmeticOpParam::output_multiplier, nnfw::cker::BinaryArithmeticOpParam::output_offset, nnfw::cker::BinaryArithmeticOpParam::output_shift, nnfw::cker::BinaryArithmeticOpParam::quantized_activation_max, nnfw::cker::BinaryArithmeticOpParam::quantized_activation_min, and size.

◆ MulElementwise() [2/2]

void nnfw::cker::optimized::MulElementwise	(	int	size,
		const BinaryArithmeticOpParam &	params,
		const uint8_t *	input1_data,
		const uint8_t *	input2_data,
		uint8_t *	output_data
	)

inline

Definition at line 910 of file BinaryArithmeticOps.h.

{
  int i = 0;
 
#ifdef USE_NEON
  const auto input1_offset_vector = vdupq_n_s16(params.input1_offset);
  const auto input2_offset_vector = vdupq_n_s16(params.input2_offset);
  const auto output_offset_vector = vdupq_n_s16(params.output_offset);
  const auto output_activation_min_vector = vdup_n_u8(params.quantized_activation_min);
  const auto output_activation_max_vector = vdup_n_u8(params.quantized_activation_max);
  const int left_shift = std::max(0, params.output_shift);
  const int right_shift = std::max(0, -params.output_shift);
  const int32x4_t left_shift_vec = vdupq_n_s32(left_shift);
  for (; i <= size - 8; i += 8)
  {
    // We load / store 8 at a time, multiplying as two sets of 4 int32s.
    const auto input1_val_original = vld1_u8(input1_data + i);
    const auto input2_val_original = vld1_u8(input2_data + i);
    const auto input1_val_s16 = vreinterpretq_s16_u16(vmovl_u8(input1_val_original));
    const auto input2_val_s16 = vreinterpretq_s16_u16(vmovl_u8(input2_val_original));
    const auto input1_val = vaddq_s16(input1_val_s16, input1_offset_vector);
    const auto input2_val = vaddq_s16(input2_val_s16, input2_offset_vector);
 
    const auto input1_val_low = vget_low_s16(input1_val);
    const auto input1_val_high = vget_high_s16(input1_val);
    const auto input2_val_low = vget_low_s16(input2_val);
    const auto input2_val_high = vget_high_s16(input2_val);
 
    auto p1 = vmull_s16(input2_val_low, input1_val_low);
    auto p2 = vmull_s16(input2_val_high, input1_val_high);
 
    p1 = vshlq_s32(p1, left_shift_vec);
    p2 = vshlq_s32(p2, left_shift_vec);
    p1 = vqrdmulhq_n_s32(p1, params.output_multiplier);
    p2 = vqrdmulhq_n_s32(p2, params.output_multiplier);
    using gemmlowp::RoundingDivideByPOT;
    p1 = RoundingDivideByPOT(p1, right_shift);
    p2 = RoundingDivideByPOT(p2, right_shift);
 
    const auto p1_narrowed = vqmovn_s32(p1);
    const auto p2_narrowed = vqmovn_s32(p2);
    const auto p = vaddq_s16(vcombine_s16(p1_narrowed, p2_narrowed), output_offset_vector);
    const auto clamped =
      vmax_u8(output_activation_min_vector, vmin_u8(output_activation_max_vector, vqmovun_s16(p)));
    vst1_u8(output_data + i, clamped);
  }
#endif // NEON
 
  for (; i < size; ++i)
  {
    const int32_t input1_val = params.input1_offset + input1_data[i];
    const int32_t input2_val = params.input2_offset + input2_data[i];
    const int32_t unclamped_result =
      params.output_offset + MultiplyByQuantizedMultiplier(input1_val * input2_val,
                                                           params.output_multiplier,
                                                           params.output_shift);
    const int32_t clamped_output = std::min(
      params.quantized_activation_max, std::max(params.quantized_activation_min, unclamped_result));
    output_data[i] = static_cast<uint8_t>(clamped_output);
  }
}

References nnfw::cker::BinaryArithmeticOpParam::input1_offset, nnfw::cker::BinaryArithmeticOpParam::input2_offset, nnfw::cker::MultiplyByQuantizedMultiplier(), nnfw::cker::BinaryArithmeticOpParam::output_multiplier, nnfw::cker::BinaryArithmeticOpParam::output_offset, nnfw::cker::BinaryArithmeticOpParam::output_shift, nnfw::cker::BinaryArithmeticOpParam::quantized_activation_max, nnfw::cker::BinaryArithmeticOpParam::quantized_activation_min, and size.

Referenced by BroadcastMulDispatch(), and Mul().

◆ MulSimpleBroadcast() [1/2]

void nnfw::cker::optimized::MulSimpleBroadcast	(	int	size,
		const BinaryArithmeticOpParam &	params,
		const int8_t	broadcast_value,
		const int8_t *	input2_data,
		int8_t *	output_data
	)

inline

Definition at line 1095 of file BinaryArithmeticOps.h.

{
  const int16_t input1_val = params.input1_offset + broadcast_value;
 
  int i = 0;
#ifdef USE_NEON
  const auto input2_offset_vector = vdupq_n_s16(params.input2_offset);
  const auto output_offset_vector = vdupq_n_s16(params.output_offset);
  const auto output_activation_min_vector = vdupq_n_s8(params.quantized_activation_min);
  const auto output_activation_max_vector = vdupq_n_s8(params.quantized_activation_max);
  const int left_shift = std::max(0, params.output_shift);
  const int right_shift = std::max(0, -params.output_shift);
  const int32x4_t left_shift_vec = vdupq_n_s32(left_shift);
  for (; i <= size - 16; i += 16)
  {
    // We load / store 16 at a time, multiplying as four sets of 4 int32s.
    const auto input2_val_original = vld1q_s8(input2_data + i);
    const auto input2_val_s16_high = vmovl_s8(vget_high_s8(input2_val_original));
    const auto input2_val_s16_low = vmovl_s8(vget_low_s8(input2_val_original));
 
    const auto input2_val_high = vaddq_s16(input2_val_s16_high, input2_offset_vector);
    const auto input2_val_low = vaddq_s16(input2_val_s16_low, input2_offset_vector);
 
    const auto input2_val_low_low = vget_low_s16(input2_val_low);
    const auto input2_val_low_high = vget_high_s16(input2_val_low);
    const auto input2_val_high_low = vget_low_s16(input2_val_high);
    const auto input2_val_high_high = vget_high_s16(input2_val_high);
 
    auto p1 = vmull_n_s16(input2_val_high_high, input1_val);
    auto p2 = vmull_n_s16(input2_val_high_low, input1_val);
    auto p3 = vmull_n_s16(input2_val_low_high, input1_val);
    auto p4 = vmull_n_s16(input2_val_low_low, input1_val);
 
    p1 = vshlq_s32(p1, left_shift_vec);
    p2 = vshlq_s32(p2, left_shift_vec);
    p3 = vshlq_s32(p3, left_shift_vec);
    p4 = vshlq_s32(p4, left_shift_vec);
 
    p1 = vqrdmulhq_n_s32(p1, params.output_multiplier);
    p2 = vqrdmulhq_n_s32(p2, params.output_multiplier);
    p3 = vqrdmulhq_n_s32(p3, params.output_multiplier);
    p4 = vqrdmulhq_n_s32(p4, params.output_multiplier);
    using gemmlowp::RoundingDivideByPOT;
    p1 = RoundingDivideByPOT(p1, right_shift);
    p2 = RoundingDivideByPOT(p2, right_shift);
    p3 = RoundingDivideByPOT(p3, right_shift);
    p4 = RoundingDivideByPOT(p4, right_shift);
 
    const auto p1_narrowed = vqmovn_s32(p1);
    const auto p2_narrowed = vqmovn_s32(p2);
    const auto p3_narrowed = vqmovn_s32(p3);
    const auto p4_narrowed = vqmovn_s32(p4);
 
    const int16x8_t p_part1 =
      vaddq_s16(vcombine_s16(p2_narrowed, p1_narrowed), output_offset_vector);
    const int16x8_t p_part2 =
      vaddq_s16(vcombine_s16(p4_narrowed, p3_narrowed), output_offset_vector);
    const int8x16_t p = vcombine_s8(vqmovn_s16(p_part2), vqmovn_s16(p_part1));
 
    const auto clamped =
      vmaxq_s8(output_activation_min_vector, vminq_s8(output_activation_max_vector, p));
    vst1q_s8(output_data + i, clamped);
  }
#endif // NEON
 
  for (; i < size; ++i)
  {
    const int32_t input2_val = params.input2_offset + input2_data[i];
    const int32_t unclamped_result =
      params.output_offset + MultiplyByQuantizedMultiplier(input1_val * input2_val,
                                                           params.output_multiplier,
                                                           params.output_shift);
    const int32_t clamped_output = std::min(
      params.quantized_activation_max, std::max(params.quantized_activation_min, unclamped_result));
    output_data[i] = static_cast<int8_t>(clamped_output);
  }
}

References nnfw::cker::BinaryArithmeticOpParam::input1_offset, nnfw::cker::BinaryArithmeticOpParam::input2_offset, nnfw::cker::MultiplyByQuantizedMultiplier(), nnfw::cker::BinaryArithmeticOpParam::output_multiplier, nnfw::cker::BinaryArithmeticOpParam::output_offset, nnfw::cker::BinaryArithmeticOpParam::output_shift, nnfw::cker::BinaryArithmeticOpParam::quantized_activation_max, nnfw::cker::BinaryArithmeticOpParam::quantized_activation_min, and size.

◆ MulSimpleBroadcast() [2/2]

void nnfw::cker::optimized::MulSimpleBroadcast	(	int	size,
		const BinaryArithmeticOpParam &	params,
		const uint8_t	broadcast_value,
		const uint8_t *	input2_data,
		uint8_t *	output_data
	)

inline

Definition at line 1081 of file BinaryArithmeticOps.h.

{
  int i = 0;
  int32_t clamped_output;
  for (; i < size; ++i)
  {
    clamped_output = quant8_mul(params, broadcast_value, input2_data[i]);
    output_data[i] = static_cast<uint8_t>(clamped_output);
  }
}

References quant8_mul(), and size.

Referenced by BroadcastMulDispatch().

◆ quant8_mul()

template<typename T >

std::enable_if_t< is_quant8< T >::value, int32_t > nnfw::cker::optimized::quant8_mul	(	const BinaryArithmeticOpParam &	params,
		const T	input1_data,
		const T	input2_data
	)

inline

Definition at line 896 of file BinaryArithmeticOps.h.

{
  const int32_t input1_val = params.input1_offset + input1_data;
  const int32_t input2_val = params.input2_offset + input2_data;
  const int32_t unclamped_result =
    params.output_offset + MultiplyByQuantizedMultiplier(input1_val * input2_val,
                                                         params.output_multiplier,
                                                         params.output_shift);
  const int32_t clamped_output = std::min(
    params.quantized_activation_max, std::max(params.quantized_activation_min, unclamped_result));
 
  return clamped_output;
}

References nnfw::cker::BinaryArithmeticOpParam::input1_offset, nnfw::cker::BinaryArithmeticOpParam::input2_offset, nnfw::cker::MultiplyByQuantizedMultiplier(), nnfw::cker::BinaryArithmeticOpParam::output_multiplier, nnfw::cker::BinaryArithmeticOpParam::output_offset, nnfw::cker::BinaryArithmeticOpParam::output_shift, nnfw::cker::BinaryArithmeticOpParam::quantized_activation_max, and nnfw::cker::BinaryArithmeticOpParam::quantized_activation_min.

Referenced by BroadcastMulDispatch(), and MulSimpleBroadcast().

◆ quant8_sum()

template<typename T >

std::enable_if_t< is_quant8< T >::value, int32_t > nnfw::cker::optimized::quant8_sum	(	const BinaryArithmeticOpParam &	params,
		const T	input1_data,
		const T	input2_data
	)

inline

Definition at line 227 of file BinaryArithmeticOps.h.

{
  const int32_t input1_val = params.input1_offset + input1_data;
  const int32_t input2_val = params.input2_offset + input2_data;
  const int32_t shifted_input1_val = input1_val * (1 << params.left_shift);
  const int32_t shifted_input2_val = input2_val * (1 << params.left_shift);
  const int32_t scaled_input1_val = MultiplyByQuantizedMultiplierSmallerThanOneExp(
    shifted_input1_val, params.input1_multiplier, params.input1_shift);
  const int32_t scaled_input2_val = MultiplyByQuantizedMultiplierSmallerThanOneExp(
    shifted_input2_val, params.input2_multiplier, params.input2_shift);
  const int32_t raw_sum = scaled_input1_val + scaled_input2_val;
  const int32_t raw_output = MultiplyByQuantizedMultiplierSmallerThanOneExp(
                               raw_sum, params.output_multiplier, params.output_shift) +
                             params.output_offset;
  const int32_t clamped_output = std::min(params.quantized_activation_max,
                                          std::max(params.quantized_activation_min, raw_output));
  return clamped_output;
}

Referenced by AddScalarBroadcast(), and BroadcastAddDispatch().

◆ Sub()

void nnfw::cker::optimized::Sub	(	const BinaryArithmeticOpParam &	params,
		const Shape &	input1_shape,
		const float *	input1_data,
		const Shape &	input2_shape,
		const float *	input2_data,
		const Shape &	output_shape,
		float *	output_data
	)

inline

Definition at line 858 of file BinaryArithmeticOps.h.

{
  const int flat_size = MatchingElementsSize(input1_shape, input2_shape, output_shape);
  auto implFuncs = getBinaryOpWithActivationImplFloat<BinaryOpFuncSubFloat>(params);
  (*implFuncs.first)(flat_size, params, input1_data, input2_data, output_data);
}

References nnfw::cker::MatchingElementsSize(), and output_shape.

Referenced by nnfw::cker::BinaryArithmeticOp().

Variable Documentation

◆ _gemmlowp_mutex

std::mutex nnfw::cker::optimized::_gemmlowp_mutex

Definition at line 45 of file Conv.h.

Referenced by Conv().

Namespaces

Data Structures

Typedefs

Functions

Variables

Typedef Documentation

◆ BinaryOpImplFloatFuncs

Function Documentation

◆ Add() [1/2]

◆ Add() [2/2]

◆ AddBiasAndEvalActivationFunction()

◆ AddElementwise() [1/2]

◆ AddElementwise() [2/2]

◆ AddScalarBroadcast() [1/2]

◆ AddScalarBroadcast() [2/2]

◆ BinaryBroadcastFiveFold() [1/2]

◆ BinaryBroadcastFiveFold() [2/2]

◆ BinaryOpElementwise()

◆ BinaryOpScalarBroadcast()

◆ BroadcastAddDispatch() [1/2]

◆ BroadcastAddDispatch() [2/2]

◆ BroadcastDivDispatch()

◆ BroadcastMulDispatch() [1/2]

◆ BroadcastMulDispatch() [2/2]

◆ BroadcastSubDispatch()

◆ Conv()

◆ DepthwiseConvImpl() [1/2]

◆ DepthwiseConvImpl() [2/2]

◆ DepthwiseConvInitAccBuffer()

◆ DepthwiseConvWithRounding()

◆ DilatedIm2col() [1/2]

◆ DilatedIm2col() [2/2]

◆ Div()

◆ ExtractPatchIntoBufferColumn()

◆ FloatDepthwiseConvAccumRow()

◆ FloatDepthwiseConvAccumRowGeneric()

◆ getBinaryOpWithActivationImplFloat()

◆ Im2col()

◆ Mul() [1/2]

◆ Mul() [2/2]

◆ MulElementwise() [1/2]

◆ MulElementwise() [2/2]

◆ MulSimpleBroadcast() [1/2]

◆ MulSimpleBroadcast() [2/2]

◆ quant8_mul()

◆ quant8_sum()

◆ Sub()

Variable Documentation

◆ _gemmlowp_mutex