Data Structures
struct	FloatDepthwiseConvKernel

Functions
template<bool kAllowStrided, int kFixedInputDepth, int kFixedDepthMultiplier>
void	FloatDepthwiseConvAccumRow (int stride, int input_depth, int input_width, const float input_data, int pad_width, int depth_multiplier, int filter_width, const float filter_data, int out_x_buffer_start, int out_x_buffer_end, int output_depth, float *acc_buffer)

void	FloatDepthwiseConvAccumRowGeneric (int stride, int input_depth, int input_width, const float input_data, int pad_width, int depth_multiplier, int filter_width, const float filter_data, int out_x_buffer_start, int out_x_buffer_end, int output_depth, float *acc_buffer)

void	DepthwiseConvInitAccBuffer (int num_output_pixels, int output_depth, const float bias_data, float acc_buffer)

template<FusedActivationFunctionType Ac>
void	DepthwiseConv (const float input_data, const Dims< 4 > &input_dims, const float filter_data, const Dims< 4 > &filter_dims, const float bias_data, const Dims< 4 > &bias_dims, int stride_width, int stride_height, int pad_width, int pad_height, int depth_multiplier, float output_data, const Dims< 4 > &output_dims)

Function Documentation

◆ DepthwiseConv()

template<FusedActivationFunctionType Ac>

void optimized_ops::DepthwiseConv	(	const float *	input_data,
		const Dims< 4 > &	input_dims,
		const float *	filter_data,
		const Dims< 4 > &	filter_dims,
		const float *	bias_data,
		const Dims< 4 > &	bias_dims,
		int	stride_width,
		int	stride_height,
		int	pad_width,
		int	pad_height,
		int	depth_multiplier,
		float *	output_data,
		const Dims< 4 > &	output_dims
	)

Definition at line 165 of file DepthwiseConv2D.float.cpp.

{
  static_assert(
      Ac == FusedActivationFunctionType::kNone || Ac == FusedActivationFunctionType::kRelu ||
          Ac == FusedActivationFunctionType::kRelu6 || Ac == FusedActivationFunctionType::kRelu1,
      "");
  const int batches = MatchingArraySize(input_dims, 3, output_dims, 3);
  const int output_depth = MatchingArraySize(filter_dims, 0, output_dims, 0);
  const int input_height = ArraySize(input_dims, 2);
  const int input_width = ArraySize(input_dims, 1);
  const int input_depth = ArraySize(input_dims, 0);
  const int filter_height = ArraySize(filter_dims, 2);
  const int filter_width = ArraySize(filter_dims, 1);
  const int output_height = ArraySize(output_dims, 2);
  const int output_width = ArraySize(output_dims, 1);
#if 0 // TODO-NNRT : Check if assertion is needed, output depth some times not equal to input *
      // depthmultiplier
  DCHECK(output_depth == input_depth * depth_multiplier);
#endif
 
  static const int kAccBufferMaxSize = 1024;
  float acc_buffer[kAccBufferMaxSize];
  DCHECK_GE(kAccBufferMaxSize, output_depth);
  const int kOutputPixelsInAccBuffer = kAccBufferMaxSize / output_depth;
  const int kAccBufferActualSize = kOutputPixelsInAccBuffer * output_depth;
  DCHECK_LE(kOutputPixelsInAccBuffer * output_depth, kAccBufferActualSize);
  DCHECK_LE(kAccBufferActualSize, kAccBufferMaxSize);
  DCHECK_GE(kOutputPixelsInAccBuffer, 1);
 
  // row_accum_func will point to the core accumulation function to be used
  // for this DepthwiseConv op.
  auto *row_accum_func = FloatDepthwiseConvAccumRowGeneric;
 
  const int kMaxFixedDepthMultiplier = 16;
  int fixed_depth_multiplier = 0;
  if (depth_multiplier <= kMaxFixedDepthMultiplier)
  {
    fixed_depth_multiplier = depth_multiplier;
  }
  // kMaxUnrolling is the max number of output values that we aim to handle
  // in one unrolled iteration of the inner loop. For practical performance
  // reasons, it is limited by the number of available registers. We could
  // fine-tune it depending on the architecture, but that's not worth doing
  // since this whole code is not very optimized to begin with. The
  // present value reflects what's realistic on ARM 32bit NEON with 16 128-bit
  // vector registers.
  const int kMaxUnrolling = 8;
  int fixed_input_depth = 0;
  if (fixed_depth_multiplier && input_depth * fixed_depth_multiplier <= kMaxUnrolling)
  {
    fixed_input_depth = input_depth;
  }
 
  // Now that we have determined row_accum_func, we can start work.
  float *output_ptr = output_data;
  for (int b = 0; b < batches; ++b)
  {
    for (int out_y = 0; out_y < output_height; ++out_y)
    {
      const int in_y_origin = (out_y * stride_height) - pad_height;
      const int filter_y_start = std::max(0, -in_y_origin);
      const int filter_y_end = std::min(filter_height, input_height - in_y_origin);
      for (int out_x_buffer_start = 0; out_x_buffer_start < output_width;
           out_x_buffer_start += kOutputPixelsInAccBuffer)
      {
        const int out_x_buffer_end =
            std::min(output_width, out_x_buffer_start + kOutputPixelsInAccBuffer);
        // We call a 'pixel' a group of activation that share all but the
        // 'depth'/'channel' coordinate. num_output_pixels is the number of
        // output pixels that we will accumulate in this loop iteration.
        const int num_output_pixels = out_x_buffer_end - out_x_buffer_start;
        // Initialize our local accumulator with the bias values, so we don't
        // have to add them later.
        DepthwiseConvInitAccBuffer(num_output_pixels, output_depth, bias_data, acc_buffer);
        // Accumulation loop. Most of the time should be spent in here.
        for (int filter_y = filter_y_start; filter_y < filter_y_end; ++filter_y)
        {
          const int in_y = in_y_origin + filter_y;
          row_accum_func(stride_width, input_depth, input_width,
                         input_data + in_y * input_dims.strides[2] + b * input_dims.strides[3],
                         pad_width, depth_multiplier, filter_width,
                         filter_data + filter_y * filter_dims.strides[2], out_x_buffer_start,
                         out_x_buffer_end, output_depth, acc_buffer);
        }
        // Finished accumulating. Now store to destination.
        const int num_output_values = output_depth * num_output_pixels;
        int i = 0;
        // Handle leftover values, one by one. This is very slow.
        for (; i < num_output_values; i++)
        {
          float acc = acc_buffer[i];
          if (Ac == FusedActivationFunctionType::kRelu)
          {
            acc = std::max(0.f, acc);
          }
          else if (Ac == FusedActivationFunctionType::kRelu6)
          {
            acc = std::max(0.f, std::min(6.f, acc));
          }
          else if (Ac == FusedActivationFunctionType::kRelu1)
          {
            acc = std::max(-1.f, std::min(1.f, acc));
          }
          *output_ptr++ = acc;
        }
      }
    }
  }
}

References ArraySize(), DCHECK, DCHECK_GE, DCHECK_LE, DepthwiseConvInitAccBuffer(), FloatDepthwiseConvAccumRowGeneric(), kNone, kRelu, kRelu1, kRelu6, MatchingArraySize(), and Dims< N >::strides.

◆ DepthwiseConvInitAccBuffer()

void optimized_ops::DepthwiseConvInitAccBuffer	(	int	num_output_pixels,
		int	output_depth,
		const float *	bias_data,
		float *	acc_buffer
	)

inline

Definition at line 154 of file DepthwiseConv2D.float.cpp.

{
  for (int i = 0; i < num_output_pixels; i++)
  {
    memcpy(acc_buffer + i * output_depth, bias_data, sizeof(acc_buffer[0]) * output_depth);
  }
}

Referenced by DepthwiseConv().

◆ FloatDepthwiseConvAccumRow()

template<bool kAllowStrided, int kFixedInputDepth, int kFixedDepthMultiplier>

void optimized_ops::FloatDepthwiseConvAccumRow	(	int	stride,
		int	input_depth,
		int	input_width,
		const float *	input_data,
		int	pad_width,
		int	depth_multiplier,
		int	filter_width,
		const float *	filter_data,
		int	out_x_buffer_start,
		int	out_x_buffer_end,
		int	output_depth,
		float *	acc_buffer
	)

Definition at line 44 of file DepthwiseConv2D.float.cpp.

{
  // Sanity check parameters. This is important in particular to ensure
  // that we keep the number of template instantiations minimal, so we don't
  // increase binary size unnecessarily.
  static_assert(kFixedDepthMultiplier || !kFixedInputDepth, "");
  static_assert(kFixedInputDepth || kAllowStrided, "");
  DCHECK(stride == 1 || kAllowStrided);
  if (kFixedInputDepth)
  {
    DCHECK_EQ(input_depth, kFixedInputDepth);
  }
  if (kFixedDepthMultiplier)
  {
    DCHECK_EQ(depth_multiplier, kFixedDepthMultiplier);
  }
  DCHECK_EQ(output_depth, input_depth * depth_multiplier);
  const int input_ptr_increment = stride * input_depth;
  const float *filter_base_ptr = filter_data;
  for (int filter_x = 0; filter_x < filter_width; ++filter_x)
  {
    // For the current (filter_x, filter_y) point in the filter,
    // compute the boundaries of the corresponding output row segment.
    int out_x_loop_start_unclampled = 0;
    int out_x_loop_end_unclampled = 0;
    if (kAllowStrided)
    {
      if (stride == 2)
      {
        out_x_loop_start_unclampled = (pad_width - filter_x + 1) / 2;
        out_x_loop_end_unclampled = (pad_width + input_width - filter_x + 1) / 2;
      }
      else if (stride == 4)
      {
        out_x_loop_start_unclampled = (pad_width - filter_x + 3) / 4;
        out_x_loop_end_unclampled = (pad_width + input_width - filter_x + 3) / 4;
      }
      else
      {
        out_x_loop_start_unclampled = (pad_width - filter_x + stride - 1) / stride;
        out_x_loop_end_unclampled = (pad_width + input_width - filter_x + stride - 1) / stride;
      }
    }
    else
    {
      out_x_loop_start_unclampled = pad_width - filter_x;
      out_x_loop_end_unclampled = pad_width + input_width - filter_x;
    }
    // The kernel will have to iterate on the segment of the
    // output row that starts at out_x_loop_start and out_x_loop_end.
    const int out_x_loop_start = std::max(out_x_buffer_start, out_x_loop_start_unclampled);
    const int out_x_loop_end = std::min(out_x_buffer_end, out_x_loop_end_unclampled);
 
    float *acc_buffer_ptr = acc_buffer + (out_x_loop_start - out_x_buffer_start) * output_depth;
    const int in_x_origin = (out_x_loop_start * stride) - pad_width + filter_x;
    const float *input_ptr = input_data + in_x_origin * input_depth;
    const int num_output_pixels = out_x_loop_end - out_x_loop_start;
    FloatDepthwiseConvKernel<kAllowStrided, kFixedInputDepth, kFixedDepthMultiplier>::Run(
        num_output_pixels, input_depth, depth_multiplier, input_ptr, input_ptr_increment,
        filter_base_ptr, acc_buffer_ptr);
    filter_base_ptr += output_depth;
  }
}

References DCHECK, and DCHECK_EQ.

◆ FloatDepthwiseConvAccumRowGeneric()

void optimized_ops::FloatDepthwiseConvAccumRowGeneric	(	int	stride,
		int	input_depth,
		int	input_width,
		const float *	input_data,
		int	pad_width,
		int	depth_multiplier,
		int	filter_width,
		const float *	filter_data,
		int	out_x_buffer_start,
		int	out_x_buffer_end,
		int	output_depth,
		float *	acc_buffer
	)

inline

Definition at line 114 of file DepthwiseConv2D.float.cpp.

{
  const float *filter_base_ptr = filter_data;
  for (int filter_x = 0; filter_x < filter_width; ++filter_x)
  {
    const int out_x_loop_start =
        std::max(out_x_buffer_start, (pad_width - filter_x + stride - 1) / stride);
    const int out_x_loop_end =
        std::min(out_x_buffer_end, (pad_width + input_width - filter_x + stride - 1) / stride);
 
    float *acc_buffer_ptr = acc_buffer + (out_x_loop_start - out_x_buffer_start) * output_depth;
    const int in_x_origin = (out_x_loop_start * stride) - pad_width + filter_x;
    const float *input_ptr = input_data + in_x_origin * input_depth;
    const int input_ptr_increment = (stride - 1) * input_depth;
    for (int out_x = out_x_loop_start; out_x < out_x_loop_end; out_x++)
    {
      const float *filter_ptr = filter_base_ptr;
      for (int ic = 0; ic < input_depth; ++ic)
      {
        const float input_val = *input_ptr++;
        for (int m = 0; m < depth_multiplier; m++)
        {
          const float filter_val = *filter_ptr++;
          *acc_buffer_ptr++ += filter_val * input_val;
        }
      }
      input_ptr += input_ptr_increment;
    }
    filter_base_ptr += output_depth;
  }
}

References m.