nnfw::cker::depthwise_conv_op Namespace Reference

Namespaces
namespace	functor

Data Structures
struct	DepthwiseConv2DKernel
struct	LaunchDepthwiseConvBackpropFilterOp
struct	LaunchDepthwiseConvBackpropFilterOp< CPUDevice, T >
struct	LaunchDepthwiseConvBackpropInputOp
struct	LaunchDepthwiseConvBackpropInputOp< CPUDevice, T >
struct	LaunchDepthwiseConvOp
struct	LaunchDepthwiseConvOp< CPUDevice, T >

Typedefs
using	CPUDevice = Eigen::ThreadPoolDevice

Functions
template<typename T >
void	CopyOutputBackpropRegion (int, int, int, int, int filter_rows_, int filter_cols_, int, int stride_, int pad_rows_, int pad_cols_, int out_rows_, int out_cols_, int out_depth, const int64_t padded_filter_inner_dim_size, const int64_t in_r, const int64_t in_c, const T *out_backprop, T *buffer)
template<typename T >
void	ComputeBackpropInput (int, int, int in_cols, int in_depth_, int filter_rows, int filter_cols, int depth_multiplier_, int, int, int, int, int, int out_depth_, const int64_t padded_filter_inner_dim_size, const int64_t in_r, const int64_t in_c, const T *filter, const T *buffer, T *out_buffer, T *output)
template<typename T >
void	DepthwiseConvBackpropInputReference (int batch, int in_rows, int in_cols, int in_depth, int out_rows, int out_cols, int out_depth, int stride, int depth_multiplier, int filter_rows, int filter_cols, int pad_rows, int pad_cols, const T *out_backprop, const T *filter, T *in_backprop)
template<typename T >
void	ComputeBackpropFilter (int, int, int, int, int filter_rows, int filter_cols, int, int, int, int, int out_rows, int out_cols, int out_depth_, const int64_t padded_out_depth_size, const int64_t out_r, const int64_t out_c, const T *out_backprop, const T *input_buffer, T *output_buffer)
template<typename T >
void	DepthwiseConvBackpropFilterReference (int batch, int in_rows, int in_cols, int in_depth, int out_rows, int out_cols, int out_depth, int stride, int depth_multiplier, int filter_rows, int filter_cols, int pad_rows, int pad_cols, const T *out_backprop, const T *input, T *filter_backprop)

Typedef Documentation

CPUDevice

using nnfw::cker::depthwise_conv_op::CPUDevice = typedef Eigen::ThreadPoolDevice

Definition at line 338 of file depthwise_conv_op.h.

Function Documentation

ComputeBackpropFilter()

template<typename T >

void nnfw::cker::depthwise_conv_op::ComputeBackpropFilter	(	int	,
		int	,
		int	,
		int	,
		int	filter_rows,
		int	filter_cols,
		int	,
		int	,
		int	,
		int	,
		int	out_rows,
		int	out_cols,
		int	out_depth_,
		const int64_t	padded_out_depth_size,
		const int64_t	out_r,
		const int64_t	out_c,
		const T *	out_backprop,
		const T *	input_buffer,
		T *	output_buffer
	)

Definition at line 898 of file depthwise_conv_op.h.

{
  typedef typename Eigen::internal::packet_traits<T>::type Packet;
  static const int64_t kPacketSize = (sizeof(Packet) / sizeof(T));
  // Calculate vectorized size of 'padded_out_depth_size'.
  const int64_t out_depth = out_depth_;
  const int64_t filter_spatial_size = filter_rows * filter_cols;
  const int64_t output_vectorized_size = (padded_out_depth_size / kPacketSize) * kPacketSize;
  const int64_t base_output_index = (out_r * out_cols + out_c) * out_depth;
  // Determine whether we can execute fast or slow code path.
  const int64_t output_image_size = out_rows * out_cols * out_depth;
  const int64_t output_last_vector_index =
    output_image_size - (filter_spatial_size * padded_out_depth_size);
  const bool fast_path = base_output_index <= output_last_vector_index;
 
  if (fast_path)
  {
    // TODO(andydavis) Process multiple inputs in 'input_buffer' so we can
    // amortize the cost of 'output_buffer' load store in the loop below.
    for (int i = 0; i < output_vectorized_size; i += kPacketSize)
    {
      // Load vector register from 'out_backprop'.
      const auto out_bprop_block =
        Eigen::internal::ploadu<Packet>(out_backprop + base_output_index + i);
      for (int j = 0; j < filter_spatial_size; ++j)
      {
        const int64_t index = i + j * padded_out_depth_size;
        // Load vector register from 'input_buffer'.
        const auto input_block = Eigen::internal::ploadu<Packet>(input_buffer + index);
        // Load output block into vector register.
        auto out_block_data = output_buffer + index;
        auto out_block = Eigen::internal::ploadu<Packet>(out_block_data);
        // Vector multiply-add.
        out_block = Eigen::internal::pmadd<Packet>(out_bprop_block, input_block, out_block);
        // Store 'out_block' back to memory.
        Eigen::internal::pstoreu<T>(out_block_data, out_block);
      }
    }
  }
  else
  {
    // Slow path (cant do vector reads from non-padded 'out_backprop'.
    for (int i = 0; i < output_vectorized_size; i += kPacketSize)
    {
      // Calculate safe read size from 'out_backprop'.
      const int64_t out_bprop_index = base_output_index + i;
      const int64_t out_bprop_limit = std::min(output_image_size, out_bprop_index + kPacketSize);
      T out_buf[kPacketSize];
      memset(&out_buf, 0, kPacketSize * sizeof(T));
      const int64_t scalar_size = out_bprop_limit - out_bprop_index;
      for (int64_t j = 0; j < scalar_size; ++j)
      {
        out_buf[j] = out_backprop[out_bprop_index + j];
      }
      // Load vector register from 'out_buf'.
      const auto out_bprop_block = Eigen::internal::ploadu<Packet>(out_buf);
      for (int j = 0; j < filter_spatial_size; ++j)
      {
        const int64_t index = i + j * padded_out_depth_size;
        // Load vector register from 'input_buffer'.
        const auto input_block = Eigen::internal::ploadu<Packet>(input_buffer + index);
        // Load output block into vector register.
        auto out_block_data = output_buffer + index;
        auto out_block = Eigen::internal::ploadu<Packet>(out_block_data);
        // Vector multiply-add.
        out_block = Eigen::internal::pmadd<Packet>(out_bprop_block, input_block, out_block);
        // Store 'out_block' back to memory.
        Eigen::internal::pstoreu<T>(out_block_data, out_block);
      }
    }
  }
}

Referenced by nnfw::cker::depthwise_conv_op::LaunchDepthwiseConvBackpropFilterOp< CPUDevice, T >::operator()().

ComputeBackpropInput()

template<typename T >

void nnfw::cker::depthwise_conv_op::ComputeBackpropInput	(	int	,
		int	,
		int	in_cols,
		int	in_depth_,
		int	filter_rows,
		int	filter_cols,
		int	depth_multiplier_,
		int	,
		int	,
		int	,
		int	,
		int	,
		int	out_depth_,
		const int64_t	padded_filter_inner_dim_size,
		const int64_t	in_r,
		const int64_t	in_c,
		const T *	filter,
		const T *	buffer,
		T *	out_buffer,
		T *	output
	)

Definition at line 636 of file depthwise_conv_op.h.

{
  typedef typename Eigen::internal::packet_traits<T>::type Packet;
  static const int64_t kPacketSize = (sizeof(Packet) / sizeof(T));
 
  const int64_t in_depth = in_depth_;
  const int64_t depth_multiplier = depth_multiplier_;
  const int64_t out_depth = out_depth_;
  const int64_t filter_spatial_size = filter_rows * filter_cols;
 
  // Calculate vectorized and scalar lengths of 'out_depth'.
  const int64_t output_vectorized_size = (out_depth / kPacketSize) * kPacketSize;
  const int64_t output_scalar_size = out_depth % kPacketSize;
 
  // Calculate base index at which to begin writing output.
  const int64_t base_output_index = (in_r * in_cols + in_c) * in_depth;
 
  // Calculate vectorized and scalar lengths for 'depth_multiplier'. This is
  // used to efficiently reduce output when 'depth_multiplier' > kPacketSize.
  const int64_t dm_vectorized_size = (depth_multiplier / kPacketSize) * kPacketSize;
  const int64_t dm_scalar_size = depth_multiplier % kPacketSize;
 
  for (int i = 0; i < output_vectorized_size; i += kPacketSize)
  {
    // Reset accumulator.
    auto vaccum = Eigen::internal::pset1<Packet>(static_cast<T>(0));
    for (int j = 0; j < filter_spatial_size; ++j)
    {
      // Calculate index.
      const int64_t index = i + j * padded_filter_inner_dim_size;
      // Load filter.
      const auto filter_block = Eigen::internal::ploadu<Packet>(filter + index);
      // Load input.
      const auto data_block = Eigen::internal::ploadu<Packet>(buffer + index);
      // Vector multiply-add.
      vaccum = Eigen::internal::pmadd<Packet>(filter_block, data_block, vaccum);
    }
    if (depth_multiplier == 1)
    {
      // Write directly to the output.
      Eigen::internal::pstoreu<T>(output + base_output_index + i, vaccum);
    }
    else
    {
      // Buffer output for subsequent reduction step.
      Eigen::internal::pstoreu<T>(out_buffer + i, vaccum);
    }
  }
 
  if (output_scalar_size > 0)
  {
    auto vaccum = Eigen::internal::pset1<Packet>(static_cast<T>(0));
    for (int j = 0; j < filter_spatial_size; ++j)
    {
      const int64_t index = output_vectorized_size + j * padded_filter_inner_dim_size;
      const auto filter_block = Eigen::internal::ploadu<Packet>(filter + index);
      const auto data_block = Eigen::internal::ploadu<Packet>(buffer + index);
      vaccum = Eigen::internal::pmadd<Packet>(filter_block, data_block, vaccum);
    }
    // Load accumulator into an array and loop through output.
    T out_buf[kPacketSize];
    Eigen::internal::pstoreu<T>(out_buf, vaccum);
    if (depth_multiplier == 1)
    {
      // Write directly to the output.
      for (int j = 0; j < output_scalar_size; ++j)
      {
        output[base_output_index + output_vectorized_size + j] = out_buf[j];
      }
    }
    else
    {
      // Buffer output for subsequent reduction step.
      for (int j = 0; j < output_scalar_size; ++j)
      {
        out_buffer[output_vectorized_size + j] = out_buf[j];
      }
    }
  }
 
  // Iterate over 'in_depth', reduce over 'depth_multiplier', write 'output'.
  if (depth_multiplier > 1)
  {
    for (int64_t d = 0; d < in_depth; ++d)
    {
      const int64_t index = d * depth_multiplier;
      T accum = static_cast<T>(0);
      for (int64_t dm = 0; dm < dm_vectorized_size; dm += kPacketSize)
      {
        const auto v = Eigen::internal::ploadu<Packet>(out_buffer + index + dm);
        accum += Eigen::internal::predux(v);
      }
      // Copy scalar portion of replicated output.
      for (int64_t dm = 0; dm < dm_scalar_size; ++dm)
      {
        accum += out_buffer[index + dm_vectorized_size + dm];
      }
      // Copy to output.
      output[base_output_index + d] = accum;
    }
  }
}

CopyOutputBackpropRegion()

template<typename T >

void nnfw::cker::depthwise_conv_op::CopyOutputBackpropRegion	(	int	,
		int	,
		int	,
		int	,
		int	filter_rows_,
		int	filter_cols_,
		int	,
		int	stride_,
		int	pad_rows_,
		int	pad_cols_,
		int	out_rows_,
		int	out_cols_,
		int	out_depth,
		const int64_t	padded_filter_inner_dim_size,
		const int64_t	in_r,
		const int64_t	in_c,
		const T *	out_backprop,
		T *	buffer
	)

Definition at line 537 of file depthwise_conv_op.h.

{
  typedef typename Eigen::internal::packet_traits<T>::type Packet;
  static const int64_t kPacketSize = (sizeof(Packet) / sizeof(T));
 
  const int64_t stride = stride_;
  const int64_t filter_rows = filter_rows_;
  const int64_t filter_cols = filter_cols_;
  const int64_t pad_rows = pad_rows_;
  const int64_t pad_cols = pad_cols_;
  const int64_t out_rows = out_rows_;
  const int64_t out_cols = out_cols_;
 
  // Calculate the output spatial region which used point (in_r, in_c) as input.
  const int64_t out_r_start =
    std::max(static_cast<int64_t>(0), (in_r - filter_rows + pad_rows + stride) / stride);
  const int64_t out_r_end = std::min(out_rows - 1, (in_r + pad_rows) / stride);
  const int64_t out_c_start =
    std::max(static_cast<int64_t>(0), (in_c - filter_cols + pad_cols + stride) / stride);
  const int64_t out_c_end = std::min(out_cols - 1, (in_c + pad_cols) / stride);
 
  // Zero-pad 'buffer' if output region is smaller than filter spatial size.
  const int64_t filter_spatial_size = filter_rows * filter_cols;
  if ((out_r_end - out_r_start + 1) < filter_rows || (out_c_end - out_c_start + 1) < filter_cols)
  {
    memset(buffer, 0, filter_spatial_size * padded_filter_inner_dim_size * sizeof(T));
  }
 
  // Calculate vectorized and scalar (residual) lengths for 'in_depth'.
  const int64_t vectorized_size = (out_depth / kPacketSize) * kPacketSize;
  const int64_t scalar_size = out_depth % kPacketSize;
  const int64_t pad_size = scalar_size > 0 ? kPacketSize - scalar_size : 0;
 
  for (int out_r = out_r_start; out_r <= out_r_end; ++out_r)
  {
    const int64_t f_r = in_r + pad_rows - out_r * stride;
    for (int out_c = out_c_start; out_c <= out_c_end; ++out_c)
    {
      const int64_t f_c = in_c + pad_cols - out_c * stride;
      const int64_t buf_base = (f_r * filter_cols + f_c) * padded_filter_inner_dim_size;
      // Calculate index into 'out_backprop' for coordinate (out_r, out_c).
      auto *out_bprop = out_backprop + (out_r * out_cols + out_c) * out_depth;
 
      // Copy vectorized portion of inner dimension into 'buffer'.
      for (int64_t d = 0; d < vectorized_size; d += kPacketSize)
      {
        auto v = Eigen::internal::ploadu<Packet>(out_bprop + d);
        Eigen::internal::pstoreu<T>(buffer + buf_base + d, v);
      }
      // Copy scalar portion of out_bprop to 'buffer'
      for (int64_t d = 0; d < scalar_size; ++d)
      {
        buffer[buf_base + vectorized_size + d] = out_bprop[vectorized_size + d];
      }
      // Pad to vector-register width (if needed).
      for (int64_t d = 0; d < pad_size; ++d)
      {
        buffer[buf_base + vectorized_size + scalar_size + d] = static_cast<T>(0);
      }
    }
  }
}

DepthwiseConvBackpropFilterReference()

template<typename T >

void nnfw::cker::depthwise_conv_op::DepthwiseConvBackpropFilterReference	(	int	batch,
		int	in_rows,
		int	in_cols,
		int	in_depth,
		int	out_rows,
		int	out_cols,
		int	out_depth,
		int	stride,
		int	depth_multiplier,
		int	filter_rows,
		int	filter_cols,
		int	pad_rows,
		int	pad_cols,
		const T *	out_backprop,
		const T *	input,
		T *	filter_backprop
	)

Definition at line 1085 of file depthwise_conv_op.h.

{
  int num_filter_backprop = filter_rows * filter_cols * in_depth * depth_multiplier;
  memset(filter_backprop, 0, num_filter_backprop * sizeof(T));
  // Naive for loop as a reference point without concerns about performance.
  for (int b = 0; b < batch; ++b)
  {
    for (int out_r = 0; out_r < out_rows; ++out_r)
    {
      for (int out_c = 0; out_c < out_cols; ++out_c)
      {
        for (int out_d = 0; out_d < out_depth; ++out_d)
        {
          const int in_d = out_d / depth_multiplier;
          const int dm = out_d % depth_multiplier;
          const int in_r_start = out_r * stride - pad_rows;
          const int in_c_start = out_c * stride - pad_cols;
 
          for (int f_r = 0; f_r < filter_rows; ++f_r)
          {
            for (int f_c = 0; f_c < filter_cols; ++f_c)
            {
              const int in_r = in_r_start + f_r;
              const int in_c = in_c_start + f_c;
 
              if (in_r >= 0 && in_r < in_rows && in_c >= 0 && in_c < in_cols)
              {
                int out_backprop_offset =
                  out_d + out_depth * (out_c + out_cols * (out_r + out_rows * b));
                int input_offset = in_d + in_depth * (in_c + in_cols * (in_r + in_rows * b));
                int filter_backprop_offset =
                  dm + depth_multiplier * (in_d + in_depth * (f_c + filter_cols * f_r));
                filter_backprop[filter_backprop_offset] +=
                  input[input_offset] * out_backprop[out_backprop_offset];
              }
            }
          }
        }
      }
    }
  }
}

DepthwiseConvBackpropInputReference()

template<typename T >

void nnfw::cker::depthwise_conv_op::DepthwiseConvBackpropInputReference	(	int	batch,
		int	in_rows,
		int	in_cols,
		int	in_depth,
		int	out_rows,
		int	out_cols,
		int	out_depth,
		int	stride,
		int	depth_multiplier,
		int	filter_rows,
		int	filter_cols,
		int	pad_rows,
		int	pad_cols,
		const T *	out_backprop,
		const T *	filter,
		T *	in_backprop
	)

Definition at line 820 of file depthwise_conv_op.h.

{
  // Naive for loop as a reference point without concerns about performance.
  for (int b = 0; b < batch; ++b)
  {
    for (int in_r = 0; in_r < in_rows; ++in_r)
    {
      for (int in_c = 0; in_c < in_cols; ++in_c)
      {
        for (int in_d = 0; in_d < in_depth; ++in_d)
        {
          T sum = 0;
          const int out_d_start = in_d * depth_multiplier;
          const int out_d_end = out_d_start + depth_multiplier;
 
          for (int out_d = out_d_start; out_d < out_d_end; ++out_d)
          {
            const int out_r_start = std::max(0, (in_r - filter_rows + pad_rows + stride) / stride);
            const int out_r_end = std::min(out_rows - 1, (in_r + pad_rows) / stride);
 
            for (int out_r = out_r_start; out_r <= out_r_end; ++out_r)
            {
              const int out_c_start =
                std::max(0, (in_c - filter_cols + pad_cols + stride) / stride);
              const int out_c_end = std::min(out_cols - 1, (in_c + pad_cols) / stride);
 
              for (int out_c = out_c_start; out_c <= out_c_end; ++out_c)
              {
                int f_r = in_r + pad_rows - out_r * stride;
                int f_c = in_c + pad_cols - out_c * stride;
                int filter_dm = out_d - out_d_start;
                int out_backprop_offset =
                  out_d + out_depth * (out_c + out_cols * (out_r + out_rows * b));
                int filter_offset =
                  filter_dm + depth_multiplier * (in_d + in_depth * (f_c + filter_cols * f_r));
                sum += out_backprop[out_backprop_offset] * filter[filter_offset];
              }
            }
          }
 
          int in_backprop_offset = in_d + in_depth * (in_c + in_cols * (in_r + in_rows * b));
          in_backprop[in_backprop_offset] = sum;
        }
      }
    }
  }
}