30#ifndef LUCI_INTERPRETER_PAL_REFERENCE_OPS_H
31#define LUCI_INTERPRETER_PAL_REFERENCE_OPS_H
44#include "third_party/eigen3/Eigen/Core"
45#include "fixedpoint/fixedpoint.h"
46#include "ruy/profiler/instrumentation.h"
47#include "tensorflow/lite/c/common.h"
48#include "tensorflow/lite/kernels/internal/common.h"
49#include "tensorflow/lite/kernels/internal/quantization_util.h"
50#include "tensorflow/lite/kernels/internal/reference/add.h"
51#include "tensorflow/lite/kernels/internal/reference/add_n.h"
52#include "tensorflow/lite/kernels/internal/reference/arg_min_max.h"
53#include "tensorflow/lite/kernels/internal/reference/batch_matmul.h"
54#include "tensorflow/lite/kernels/internal/reference/batch_to_space_nd.h"
55#include "tensorflow/lite/kernels/internal/reference/binary_function.h"
56#include "tensorflow/lite/kernels/internal/reference/cast.h"
57#include "tensorflow/lite/kernels/internal/reference/ceil.h"
58#include "tensorflow/lite/kernels/internal/reference/comparisons.h"
59#include "tensorflow/lite/kernels/internal/reference/concatenation.h"
60#include "tensorflow/lite/kernels/internal/reference/conv.h"
61#include "tensorflow/lite/kernels/internal/reference/depth_to_space.h"
62#include "tensorflow/lite/kernels/internal/reference/dequantize.h"
63#include "tensorflow/lite/kernels/internal/reference/div.h"
64#include "tensorflow/lite/kernels/internal/reference/elu.h"
65#include "tensorflow/lite/kernels/internal/reference/exp.h"
66#include "tensorflow/lite/kernels/internal/reference/fill.h"
67#include "tensorflow/lite/kernels/internal/reference/floor.h"
68#include "tensorflow/lite/kernels/internal/reference/floor_div.h"
69#include "tensorflow/lite/kernels/internal/reference/floor_mod.h"
70#include "tensorflow/lite/kernels/internal/reference/fully_connected.h"
71#include "tensorflow/lite/kernels/internal/reference/gather.h"
72#include "tensorflow/lite/kernels/internal/reference/hard_swish.h"
73#include "tensorflow/lite/kernels/internal/reference/l2normalization.h"
74#include "tensorflow/lite/kernels/internal/reference/leaky_relu.h"
75#include "tensorflow/lite/kernels/internal/reference/log_softmax.h"
76#include "tensorflow/lite/kernels/internal/reference/logistic.h"
77#include "tensorflow/lite/kernels/internal/reference/maximum_minimum.h"
78#include "tensorflow/lite/kernels/internal/reference/mul.h"
79#include "tensorflow/lite/kernels/internal/reference/neg.h"
80#include "tensorflow/lite/kernels/internal/reference/pad.h"
81#include "tensorflow/lite/kernels/internal/reference/pooling.h"
82#include "tensorflow/lite/kernels/internal/reference/prelu.h"
83#include "tensorflow/lite/kernels/internal/reference/process_broadcast_shapes.h"
84#include "tensorflow/lite/kernels/internal/reference/quantize.h"
85#include "tensorflow/lite/kernels/internal/reference/reduce.h"
86#include "tensorflow/lite/kernels/internal/reference/requantize.h"
87#include "tensorflow/lite/kernels/internal/reference/resize_bilinear.h"
88#include "tensorflow/lite/kernels/internal/reference/resize_nearest_neighbor.h"
89#include "tensorflow/lite/kernels/internal/reference/round.h"
90#include "tensorflow/lite/kernels/internal/reference/softmax.h"
91#include "tensorflow/lite/kernels/internal/reference/space_to_batch_nd.h"
92#include "tensorflow/lite/kernels/internal/reference/space_to_depth.h"
93#include "tensorflow/lite/kernels/internal/reference/strided_slice.h"
94#include "tensorflow/lite/kernels/internal/reference/string_comparisons.h"
95#include "tensorflow/lite/kernels/internal/reference/sub.h"
96#include "tensorflow/lite/kernels/internal/reference/tanh.h"
97#include "tensorflow/lite/kernels/internal/reference/transpose.h"
98#include "tensorflow/lite/kernels/internal/reference/transpose_conv.h"
99#include "tensorflow/lite/kernels/internal/strided_slice_logic.h"
100#include "tensorflow/lite/kernels/internal/tensor.h"
101#include "tensorflow/lite/kernels/internal/types.h"
105namespace reference_ops
109inline void Relu(
const RuntimeShape &input_shape,
const T *input_data,
113 for (
int i = 0; i < flat_size; ++i)
115 const T val = input_data[i];
117 const T clamped = val < lower ? lower : val;
118 output_data[i] = clamped;
123inline void Relu1(
const RuntimeShape &input_shape,
const T *input_data,
126 ruy::profiler::ScopeLabel label(
"Relu1 (not fused)");
128 for (
int i = 0; i < flat_size; ++i)
130 const T val = input_data[i];
133 const T clamped = val > upper ? upper : val < lower ? lower : val;
134 output_data[i] = clamped;
138inline void Relu6(
const RuntimeShape &input_shape,
const float *input_data,
141 ruy::profiler::ScopeLabel label(
"Relu6 (not fused)");
143 for (
int i = 0; i < flat_size; ++i)
145 const float val = input_data[i];
146 const float upper = 6;
147 const float lower = 0;
148 const float clamped = val > upper ? upper : val < lower ? lower : val;
149 output_data[i] = clamped;
154inline void ReluX(
const tflite::ReluParams ¶ms,
const RuntimeShape &input_shape,
155 const T *input_data,
const RuntimeShape &
output_shape, T *output_data)
157 ruy::profiler::ScopeLabel label(
"Quantized ReluX (not fused)");
159 for (
int i = 0; i < flat_size; ++i)
161 const int32 val =
static_cast<int32_t
>(input_data[i]);
162 int32 clamped = params.output_offset + MultiplyByQuantizedMultiplier(val - params.input_offset,
163 params.output_multiplier,
164 params.output_shift);
165 clamped = std::max(params.quantized_activation_min, clamped);
166 clamped = std::min(params.quantized_activation_max, clamped);
167 output_data[i] =
static_cast<T
>(clamped);
172inline void ReluX(
const tflite::ActivationParams ¶ms,
const RuntimeShape &input_shape,
173 const T *input_data,
const RuntimeShape &
output_shape, T *output_data)
175 ruy::profiler::ScopeLabel label(
"Quantized ReluX (not fused)");
177 const T max_value = params.quantized_activation_max;
178 const T min_value = params.quantized_activation_min;
179 for (
int i = 0; i < flat_size; ++i)
181 const T val = input_data[i];
182 const T clamped = val > max_value ? max_value : val < min_value ? min_value : val;
183 output_data[i] = clamped;
192 const RuntimeShape &unswitched_input1_shape,
193 const uint8 *unswitched_input1_data,
194 const RuntimeShape &unswitched_input2_shape,
195 const uint8 *unswitched_input2_data,
198 ArithmeticParams switched_params = unswitched_params;
199 switched_params.input1_offset = unswitched_params.input2_offset;
200 switched_params.input2_offset = unswitched_params.input1_offset;
202 const bool use_unswitched = unswitched_params.broadcast_category ==
203 tflite::BroadcastableOpCategory::kFirstInputBroadcastsFast;
205 const ArithmeticParams ¶ms = use_unswitched ? unswitched_params : switched_params;
206 const uint8 *input1_data = use_unswitched ? unswitched_input1_data : unswitched_input2_data;
207 const uint8 *input2_data = use_unswitched ? unswitched_input2_data : unswitched_input1_data;
213 uint8 *output_data_ptr = output_data;
214 const uint8 *input1_data_ptr = input1_data;
215 const uint8 *input2_data_reset = input2_data;
216 int y0 = params.broadcast_shape[0];
217 int y1 = params.broadcast_shape[1];
218 int y2 = params.broadcast_shape[2];
219 int y3 = params.broadcast_shape[3];
220 int y4 = params.broadcast_shape[4];
221 for (
int i0 = 0; i0 < y0; ++i0)
223 const uint8 *input2_data_ptr;
224 for (
int i1 = 0; i1 < y1; ++i1)
226 input2_data_ptr = input2_data_reset;
227 for (
int i2 = 0; i2 < y2; ++i2)
229 for (
int i3 = 0; i3 < y3; ++i3)
231 MulElementwise(y4, params, input1_data_ptr, input2_data_ptr, output_data_ptr);
232 input2_data_ptr += y4;
233 output_data_ptr += y4;
235 input1_data_ptr += y4;
238 input2_data_reset = input2_data_ptr;
242inline void Mul(
const ArithmeticParams ¶ms,
const RuntimeShape &input1_shape,
243 const int16 *input1_data,
const RuntimeShape &input2_shape,
246 ruy::profiler::ScopeLabel label(
"Mul/Int16");
248 const int flat_size = MatchingElementsSize(input1_shape, input2_shape,
output_shape);
250 for (
int i = 0; i < flat_size; i++)
253 using F0 = gemmlowp::FixedPoint<std::int16_t, 0>;
255 F0 unclamped_result = F0::FromRaw(input1_data[i]) * F0::FromRaw(input2_data[i]);
256 output_data[i] = unclamped_result.raw();
260inline void Mul(
const ArithmeticParams ¶ms,
const RuntimeShape &input1_shape,
261 const int16 *input1_data,
const RuntimeShape &input2_shape,
264 ruy::profiler::ScopeLabel label(
"Mul/Int16Uint8");
265 int32 output_offset = params.output_offset;
266 int32 output_activation_min = params.quantized_activation_min;
267 int32 output_activation_max = params.quantized_activation_max;
268 TFLITE_DCHECK_LE(output_activation_min, output_activation_max);
270 const int flat_size = MatchingElementsSize(input1_shape, input2_shape,
output_shape);
272 for (
int i = 0; i < flat_size; i++)
275 using F0 = gemmlowp::FixedPoint<std::int16_t, 0>;
277 F0 unclamped_result = F0::FromRaw(input1_data[i]) * F0::FromRaw(input2_data[i]);
278 int16 rescaled_result = gemmlowp::RoundingDivideByPOT(unclamped_result.raw(), 8);
279 int16 clamped_result = std::min<int16>(output_activation_max - output_offset, rescaled_result);
280 clamped_result = std::max<int16>(output_activation_min - output_offset, clamped_result);
281 output_data[i] = output_offset + clamped_result;
285inline void Sub16(
const ArithmeticParams ¶ms,
const RuntimeShape &input1_shape,
286 const int16_t *input1_data,
const RuntimeShape &input2_shape,
287 const int16_t *input2_data,
const RuntimeShape &
output_shape,
288 int16_t *output_data)
290 ruy::profiler::ScopeLabel label(
"Sub/Int16");
291 const int input1_shift = params.input1_shift;
292 const int flat_size = MatchingElementsSize(input1_shape, input2_shape,
output_shape);
293 const int16 output_activation_min = params.quantized_activation_min;
294 const int16 output_activation_max = params.quantized_activation_max;
296 TFLITE_DCHECK(input1_shift == 0 || params.input2_shift == 0);
297 TFLITE_DCHECK_LE(input1_shift, 0);
298 TFLITE_DCHECK_LE(params.input2_shift, 0);
299 const int16 *not_shift_input = input1_shift == 0 ? input1_data : input2_data;
300 const int16 *shift_input = input1_shift == 0 ? input2_data : input1_data;
301 const int input_right_shift = input1_shift == 0 ? -params.input2_shift : -input1_shift;
303 if (input1_shift == 0)
306 using F0 = gemmlowp::FixedPoint<std::int16_t, 0>;
307 for (
int i = 0; i < flat_size; ++i)
309 F0 input_ready_scaled = F0::FromRaw(not_shift_input[i]);
311 F0::FromRaw(gemmlowp::RoundingDivideByPOT(shift_input[i], input_right_shift));
312 F0 result = SaturatingSub(input_ready_scaled, scaled_input);
313 const int16 raw_output = result.raw();
314 const int16 clamped_output =
315 std::min(output_activation_max, std::max(output_activation_min, raw_output));
316 output_data[i] = clamped_output;
322 using F0 = gemmlowp::FixedPoint<std::int16_t, 0>;
323 for (
int i = 0; i < flat_size; ++i)
325 F0 input_ready_scaled = F0::FromRaw(not_shift_input[i]);
327 F0::FromRaw(gemmlowp::RoundingDivideByPOT(shift_input[i], input_right_shift));
328 F0 result = SaturatingSub(scaled_input, input_ready_scaled);
329 const int16 raw_output = result.raw();
330 const int16 clamped_output =
331 std::min(output_activation_max, std::max(output_activation_min, raw_output));
332 output_data[i] = clamped_output;
337template <
typename Scalar>
338void Pack(
const PackParams ¶ms,
const RuntimeShape *
const *input_shapes,
339 const Scalar *
const *input_data,
const RuntimeShape &
output_shape, Scalar *output_data)
341 ruy::profiler::ScopeLabel label(
"Pack");
343 int axis = params.axis;
344 int inputs_count = params.inputs_count;
347 for (
int i = 0; i < axis; i++)
352 for (
int i = params.axis + 1; i < dimensions; i++)
356 TFLITE_DCHECK_EQ((**input_shapes).FlatSize(), copy_size * outer_size);
358 for (
int i = 0; i < inputs_count; ++i)
360 for (
int k = 0; k < outer_size; k++)
362 const Scalar *input_ptr = input_data[i] + copy_size * k;
363 int loc = k * inputs_count * copy_size + i * copy_size;
364 memcpy(output_data + loc, input_ptr, copy_size *
sizeof(Scalar));
369template <
typename Scalar>
370void Unpack(
const UnpackParams ¶ms,
const RuntimeShape &input_shape,
const Scalar *input_data,
371 const RuntimeShape &
output_shape, Scalar *
const *output_datas)
373 ruy::profiler::ScopeLabel label(
"Unpack");
374 const int dimensions = input_shape.DimensionsCount();
375 const int outputs_count = params.num_split;
378 int axis = params.axis;
383 TFLITE_DCHECK_GE(axis, 0);
384 TFLITE_DCHECK_LT(axis, dimensions);
385 for (
int i = 0; i < axis; ++i)
387 outer_size *= input_shape.Dims(i);
390 for (
int i = axis + 1; i < dimensions; ++i)
392 copy_size *= input_shape.Dims(i);
394 TFLITE_DCHECK_EQ(
output_shape.FlatSize(), copy_size * outer_size);
396 for (
int i = 0; i < outputs_count; ++i)
398 for (
int k = 0; k < outer_size; k++)
400 Scalar *output_ptr = output_datas[i] + copy_size * k;
401 int loc = k * outputs_count * copy_size + i * copy_size;
402 memcpy(output_ptr, input_data + loc, copy_size *
sizeof(Scalar));
407template <
typename Scalar>
408void PackWithScaling(
const PackParams ¶ms,
const RuntimeShape *
const *input_shapes,
412 ruy::profiler::ScopeLabel label(
"PackWithScaling");
414 int axis = params.axis;
415 const int32 *input_zeropoint = params.input_zeropoint;
416 const float *input_scale = params.input_scale;
417 int inputs_count = params.inputs_count;
418 const int32 output_zeropoint = params.output_zeropoint;
419 const float output_scale = params.output_scale;
422 for (
int i = 0; i < axis; i++)
427 for (
int i = axis + 1; i < dimensions; i++)
431 TFLITE_DCHECK_EQ((**input_shapes).FlatSize(), copy_size * outer_size);
433 Scalar *output_ptr = output_data;
434 const float inverse_output_scale = 1.f / output_scale;
435 for (
int k = 0; k < outer_size; k++)
437 for (
int i = 0; i < inputs_count; ++i)
439 if (input_zeropoint[i] == output_zeropoint && input_scale[i] == output_scale)
441 memcpy(output_ptr, input_data[i] + k * copy_size, copy_size *
sizeof(Scalar));
446 const float scale = input_scale[i] * inverse_output_scale;
447 const float bias = -input_zeropoint[i] * scale;
448 auto input_ptr = input_data[i];
449 for (
int j = 0; j < copy_size; ++j)
452 static_cast<int32_t
>(std::round(input_ptr[j] * scale + bias)) + output_zeropoint;
453 output_ptr[j] =
static_cast<uint8_t
>(std::max(std::min(255, value), 0));
456 output_ptr += copy_size;
461template <
typename Scalar>
463 const Scalar *
const *input_data,
const RuntimeShape &
output_shape,
466 ruy::profiler::ScopeLabel label(
"DepthConcatenation");
467 auto params_copy = params;
468 params_copy.axis = 3;
472inline void LstmCell(
const LstmCellParams ¶ms,
const RuntimeShape &unextended_input_shape,
473 const float *input_data,
const RuntimeShape &unextended_prev_activ_shape,
474 const float *prev_activ_data,
const RuntimeShape &weights_shape,
475 const float *weights_data,
const RuntimeShape &unextended_bias_shape,
476 const float *bias_data,
const RuntimeShape &unextended_prev_state_shape,
477 const float *prev_state_data,
478 const RuntimeShape &unextended_output_state_shape,
float *output_state_data,
479 const RuntimeShape &unextended_output_activ_shape,
float *output_activ_data,
480 const RuntimeShape &unextended_concat_temp_shape,
float *concat_temp_data,
481 const RuntimeShape &unextended_activ_temp_shape,
float *activ_temp_data)
483 TFLITE_DCHECK_LE(unextended_input_shape.DimensionsCount(), 4);
484 TFLITE_DCHECK_LE(unextended_prev_activ_shape.DimensionsCount(), 4);
485 TFLITE_DCHECK_LE(unextended_bias_shape.DimensionsCount(), 4);
486 TFLITE_DCHECK_LE(unextended_prev_state_shape.DimensionsCount(), 4);
487 TFLITE_DCHECK_LE(unextended_output_state_shape.DimensionsCount(), 4);
488 TFLITE_DCHECK_LE(unextended_output_activ_shape.DimensionsCount(), 4);
489 TFLITE_DCHECK_LE(unextended_concat_temp_shape.DimensionsCount(), 4);
490 TFLITE_DCHECK_LE(unextended_activ_temp_shape.DimensionsCount(), 4);
491 const RuntimeShape input_shape = RuntimeShape::ExtendedShape(4, unextended_input_shape);
492 const RuntimeShape prev_activ_shape = RuntimeShape::ExtendedShape(4, unextended_prev_activ_shape);
493 const RuntimeShape bias_shape = RuntimeShape::ExtendedShape(4, unextended_bias_shape);
494 const RuntimeShape prev_state_shape = RuntimeShape::ExtendedShape(4, unextended_prev_state_shape);
495 const RuntimeShape output_state_shape =
496 RuntimeShape::ExtendedShape(4, unextended_output_state_shape);
497 const RuntimeShape output_activ_shape =
498 RuntimeShape::ExtendedShape(4, unextended_output_activ_shape);
499 const RuntimeShape concat_temp_shape =
500 RuntimeShape::ExtendedShape(4, unextended_concat_temp_shape);
501 const RuntimeShape activ_temp_shape = RuntimeShape::ExtendedShape(4, unextended_activ_temp_shape);
502 TFLITE_DCHECK_GE(weights_shape.DimensionsCount(), 2);
504 const int weights_dim_count = weights_shape.DimensionsCount();
505 const int batches = MatchingDim(input_shape, 0, prev_activ_shape, 0, prev_state_shape, 0,
506 output_state_shape, 0, output_activ_shape, 0);
507 const int height = MatchingDim(input_shape, 1, prev_activ_shape, 1, prev_state_shape, 1,
508 output_state_shape, 1, output_activ_shape, 1);
509 const int width = MatchingDim(input_shape, 2, prev_activ_shape, 2, prev_state_shape, 2,
510 output_state_shape, 2, output_activ_shape, 2);
511 const int input_depth = input_shape.Dims(3);
512 const int prev_activ_depth = prev_activ_shape.Dims(3);
513 const int total_input_depth = prev_activ_depth + input_depth;
514 TFLITE_DCHECK_EQ(weights_shape.Dims(weights_dim_count - 1), total_input_depth);
515 TFLITE_DCHECK_EQ(FlatSizeSkipDim(bias_shape, 3), 1);
516 const int intern_activ_depth = MatchingDim(weights_shape, weights_dim_count - 2, bias_shape, 3);
517 TFLITE_DCHECK_EQ(weights_shape.FlatSize(), intern_activ_depth * total_input_depth);
518 TFLITE_DCHECK_EQ(intern_activ_depth % 4, 0);
519 const int output_depth = MatchingDim(prev_state_shape, 3, prev_activ_shape, 3, output_state_shape,
520 3, output_activ_shape, 3);
521 TFLITE_DCHECK_EQ(output_depth, intern_activ_depth / 4);
524 std::vector<float const *> concat_input_arrays_data;
525 std::vector<RuntimeShape const *> concat_input_arrays_shapes;
526 concat_input_arrays_data.push_back(input_data);
527 concat_input_arrays_data.push_back(prev_activ_data);
528 concat_input_arrays_shapes.push_back(&input_shape);
529 concat_input_arrays_shapes.push_back(&prev_activ_shape);
530 tflite::ConcatenationParams concat_params;
531 concat_params.axis = 3;
532 concat_params.inputs_count = concat_input_arrays_data.size();
533 Concatenation(concat_params, &(concat_input_arrays_shapes[0]), &(concat_input_arrays_data[0]),
534 concat_temp_shape, concat_temp_data);
537 tflite::FullyConnectedParams fc_params;
538 fc_params.float_activation_min = std::numeric_limits<float>::lowest();
539 fc_params.float_activation_max = std::numeric_limits<float>::max();
540 FullyConnected(fc_params, concat_temp_shape, concat_temp_data, weights_shape, weights_data,
541 bias_shape, bias_data, activ_temp_shape, activ_temp_data);
544 for (
int b = 0; b < batches; ++b)
546 for (
int w = 0; w < width; ++w)
548 for (
int h = 0; h < height; ++h)
550 for (
int c = 0; c < output_depth; ++c)
552 const float input_gate =
555 std::exp(-activ_temp_data[
Offset(activ_temp_shape, b, h, w, 0 * output_depth + c)]));
556 const float new_input =
557 std::tanh(activ_temp_data[
Offset(activ_temp_shape, b, h, w, 1 * output_depth + c)]);
558 const float forget_gate =
561 std::exp(-activ_temp_data[
Offset(activ_temp_shape, b, h, w, 2 * output_depth + c)]));
562 const float output_gate =
565 std::exp(-activ_temp_data[
Offset(activ_temp_shape, b, h, w, 3 * output_depth + c)]));
566 const float new_state =
567 input_gate * new_input +
568 forget_gate * prev_state_data[
Offset(prev_state_shape, b, h, w, c)];
569 output_state_data[
Offset(output_state_shape, b, h, w, c)] = new_state;
570 output_activ_data[
Offset(output_activ_shape, b, h, w, c)] =
571 output_gate * std::tanh(new_state);
662template <
int StateIntegerBits>
664LstmCell(
const LstmCellParams ¶ms,
const RuntimeShape &unextended_input_shape,
665 const uint8 *input_data_uint8,
const RuntimeShape &unextended_prev_activ_shape,
666 const uint8 *prev_activ_data_uint8,
const RuntimeShape &weights_shape,
667 const uint8 *weights_data_uint8,
const RuntimeShape &unextended_bias_shape,
668 const int32 *bias_data_int32,
const RuntimeShape &unextended_prev_state_shape,
669 const int16 *prev_state_data_int16,
const RuntimeShape &unextended_output_state_shape,
670 int16 *output_state_data_int16,
const RuntimeShape &unextended_output_activ_shape,
671 uint8 *output_activ_data_uint8,
const RuntimeShape &unextended_concat_temp_shape,
672 uint8 *concat_temp_data_uint8,
const RuntimeShape &unextended_activ_temp_shape,
673 int16 *activ_temp_data_int16,
void *gemmlowp_context)
675 (void)gemmlowp_context;
676 int32 weights_zero_point = params.weights_zero_point;
677 int32 accum_multiplier = params.accum_multiplier;
678 int accum_shift = params.accum_shift;
679 TFLITE_DCHECK_LE(unextended_input_shape.DimensionsCount(), 4);
680 TFLITE_DCHECK_LE(unextended_prev_activ_shape.DimensionsCount(), 4);
681 TFLITE_DCHECK_LE(unextended_bias_shape.DimensionsCount(), 4);
682 TFLITE_DCHECK_LE(unextended_prev_state_shape.DimensionsCount(), 4);
683 TFLITE_DCHECK_LE(unextended_output_state_shape.DimensionsCount(), 4);
684 TFLITE_DCHECK_LE(unextended_output_activ_shape.DimensionsCount(), 4);
685 TFLITE_DCHECK_LE(unextended_concat_temp_shape.DimensionsCount(), 4);
686 TFLITE_DCHECK_LE(unextended_activ_temp_shape.DimensionsCount(), 4);
687 const RuntimeShape input_shape = RuntimeShape::ExtendedShape(4, unextended_input_shape);
688 const RuntimeShape prev_activ_shape = RuntimeShape::ExtendedShape(4, unextended_prev_activ_shape);
689 const RuntimeShape bias_shape = RuntimeShape::ExtendedShape(4, unextended_bias_shape);
690 const RuntimeShape prev_state_shape = RuntimeShape::ExtendedShape(4, unextended_prev_state_shape);
691 const RuntimeShape output_state_shape =
692 RuntimeShape::ExtendedShape(4, unextended_output_state_shape);
693 const RuntimeShape output_activ_shape =
694 RuntimeShape::ExtendedShape(4, unextended_output_activ_shape);
695 const RuntimeShape concat_temp_shape =
696 RuntimeShape::ExtendedShape(4, unextended_concat_temp_shape);
697 const RuntimeShape activ_temp_shape = RuntimeShape::ExtendedShape(4, unextended_activ_temp_shape);
698 TFLITE_DCHECK_GE(weights_shape.DimensionsCount(), 2);
701 const int weights_dim_count = weights_shape.DimensionsCount();
702 const int outer_size = MatchingFlatSizeSkipDim(input_shape, 3, prev_activ_shape, prev_state_shape,
703 output_state_shape, output_activ_shape);
704 const int input_depth = input_shape.Dims(3);
705 const int prev_activ_depth = prev_activ_shape.Dims(3);
706 const int total_input_depth = prev_activ_depth + input_depth;
707 TFLITE_DCHECK_EQ(weights_shape.Dims(weights_dim_count - 1), total_input_depth);
708 const int intern_activ_depth = MatchingDim(weights_shape, weights_dim_count - 2, bias_shape, 3);
709 TFLITE_DCHECK_EQ(weights_shape.FlatSize(), intern_activ_depth * total_input_depth);
710 TFLITE_DCHECK_EQ(FlatSizeSkipDim(bias_shape, 3), 1);
711 TFLITE_DCHECK_EQ(intern_activ_depth % 4, 0);
712 const int output_depth = MatchingDim(prev_state_shape, 3, prev_activ_shape, 3, output_state_shape,
713 3, output_activ_shape, 3);
714 TFLITE_DCHECK_EQ(output_depth, intern_activ_depth / 4);
715 const int fc_batches = FlatSizeSkipDim(activ_temp_shape, 3);
716 const int fc_output_depth =
717 MatchingDim(weights_shape, weights_dim_count - 2, activ_temp_shape, 3);
718 const int fc_accum_depth = total_input_depth;
719 TFLITE_DCHECK_EQ(fc_output_depth, 4 * output_depth);
722 uint8 const *concat_input_arrays_data[2] = {input_data_uint8, prev_activ_data_uint8};
723 const RuntimeShape *concat_input_arrays_shapes[2] = {&input_shape, &prev_activ_shape};
724 tflite::ConcatenationParams concat_params;
725 concat_params.axis = 3;
726 concat_params.inputs_count = 2;
727 Concatenation(concat_params, concat_input_arrays_shapes, concat_input_arrays_data,
728 concat_temp_shape, concat_temp_data_uint8);
735 for (
int b = 0; b < fc_batches; ++b)
737 for (
int out_c = 0; out_c < fc_output_depth; ++out_c)
741 int32 accum = bias_data_int32[out_c];
743 for (
int d = 0; d < fc_accum_depth; ++d)
745 int16 input_val = concat_temp_data_uint8[b * fc_accum_depth + d] - 128;
746 int16 weights_val = weights_data_uint8[out_c * fc_accum_depth + d] - weights_zero_point;
747 accum += input_val * weights_val;
753 accum = MultiplyByQuantizedMultiplier(accum, accum_multiplier, accum_shift);
755 accum = std::max(-32768, std::min(32767,
static_cast<int>(accum)));
756 activ_temp_data_int16[out_c + fc_output_depth * b] = accum;
762 for (
int b = 0; b < outer_size; ++b)
764 for (
int c = 0; c < output_depth; ++c)
774 using F0 = gemmlowp::FixedPoint<std::int16_t, 0>;
778 using F3 = gemmlowp::FixedPoint<std::int16_t, 3>;
783 using FS = gemmlowp::FixedPoint<std::int16_t, StateIntegerBits>;
785 F3 input_gate_input =
786 F3::FromRaw(activ_temp_data_int16[b * fc_output_depth + 0 * output_depth + c]);
787 F0 input_gate_output = gemmlowp::logistic(input_gate_input);
790 F3 input_modulation_gate_input =
791 F3::FromRaw(activ_temp_data_int16[b * fc_output_depth + 1 * output_depth + c]);
792 F0 input_modulation_gate_output = gemmlowp::tanh(input_modulation_gate_input);
794 F3 forget_gate_input =
795 F3::FromRaw(activ_temp_data_int16[b * fc_output_depth + 2 * output_depth + c]);
796 F0 forget_gate_output = gemmlowp::logistic(forget_gate_input);
798 F3 output_gate_input =
799 F3::FromRaw(activ_temp_data_int16[b * fc_output_depth + 3 * output_depth + c]);
800 F0 output_gate_output = gemmlowp::logistic(output_gate_input);
802 F0 input_times_input_modulation = input_gate_output * input_modulation_gate_output;
803 FS prev_state = FS::FromRaw(prev_state_data_int16[b * output_depth + c]);
804 FS prev_state_times_forget_state = forget_gate_output * prev_state;
807 gemmlowp::SaturatingAdd(gemmlowp::Rescale<StateIntegerBits>(input_times_input_modulation),
808 prev_state_times_forget_state);
817 F3 new_state_f3 = gemmlowp::Rescale<3>(new_state);
818 F0 output_activ_int16 = output_gate_output * gemmlowp::tanh(new_state_f3);
822 output_state_data_int16[b * output_depth + c] = new_state.raw();
825 int16 rescaled_output_activ = gemmlowp::RoundingDivideByPOT(output_activ_int16.raw(), 8);
826 int16 clamped_output_activ =
827 std::max<int16>(-128, std::min<int16>(127, rescaled_output_activ));
828 output_activ_data_uint8[b * output_depth + c] = 128 + clamped_output_activ;
833template <
typename Scalar>
834void Split(
const SplitParams ¶ms,
const RuntimeShape &input_shape,
const Scalar *input_data,
835 const RuntimeShape *
const *output_shapes, Scalar *
const *output_data)
837 ruy::profiler::ScopeLabel label(
"Split");
838 const int split_dimensions = input_shape.DimensionsCount();
839 int axis = params.axis < 0 ? params.axis + split_dimensions : params.axis;
840 int outputs_count = params.num_split;
841 TFLITE_DCHECK_LT(axis, split_dimensions);
843 int64_t split_size = 0;
844 for (
int i = 0; i < outputs_count; i++)
846 TFLITE_DCHECK_EQ(output_shapes[i]->DimensionsCount(), split_dimensions);
847 for (
int j = 0; j < split_dimensions; j++)
851 MatchingDim(*output_shapes[i], j, input_shape, j);
854 split_size += output_shapes[i]->Dims(axis);
856 TFLITE_DCHECK_EQ(split_size, input_shape.Dims(axis));
857 int64_t outer_size = 1;
858 for (
int i = 0; i < axis; ++i)
860 outer_size *= input_shape.Dims(i);
864 int64_t base_inner_size = 1;
865 for (
int i = axis + 1; i < split_dimensions; ++i)
867 base_inner_size *= input_shape.Dims(i);
870 const Scalar *input_ptr = input_data;
871 for (
int k = 0; k < outer_size; k++)
873 for (
int i = 0; i < outputs_count; ++i)
875 const int copy_size = output_shapes[i]->Dims(axis) * base_inner_size;
876 memcpy(output_data[i] + k * copy_size, input_ptr, copy_size *
sizeof(Scalar));
877 input_ptr += copy_size;
882inline int NodeOffset(
int b,
int h,
int w,
int height,
int width)
884 return (b * height + h) * width + w;
888 const RuntimeShape &input_shape,
const float *input_data,
891 const int trailing_dim = input_shape.DimensionsCount() - 1;
892 const int outer_size = MatchingFlatSizeSkipDim(input_shape, trailing_dim,
output_shape);
893 const int depth = MatchingDim(input_shape, trailing_dim,
output_shape, trailing_dim);
895 for (
int i = 0; i < outer_size; ++i)
897 for (
int c = 0; c < depth; ++c)
899 const int begin_input_c = std::max(0,
static_cast<int>(c - op_params.range));
900 const int end_input_c = std::min(depth,
static_cast<int>(c + op_params.range));
902 for (
int input_c = begin_input_c; input_c < end_input_c; ++input_c)
904 const float input_val = input_data[i * depth + input_c];
905 accum += input_val * input_val;
907 const float multiplier = std::pow(op_params.bias + op_params.alpha * accum, -op_params.beta);
908 output_data[i * depth + c] = input_data[i * depth + c] * multiplier;
913inline void Dequantize(
const RuntimeShape &input_shape,
const Eigen::half *input_data,
917 for (
int i = 0; i < flat_size; i++)
919 output_data[i] =
static_cast<float>(input_data[i]);
923inline void FakeQuant(
const tflite::FakeQuantParams &op_params,
const RuntimeShape &input_shape,
924 const float *input_data,
const RuntimeShape &
output_shape,
float *output_data)
926 ruy::profiler::ScopeLabel label(
"FakeQuant");
927 float rmin = op_params.minmax.min;
928 float rmax = op_params.minmax.max;
929 int num_bits = op_params.num_bits;
932 TFLITE_DCHECK_LE(rmin, 0.0f);
933 TFLITE_DCHECK_GE(rmax, 0.0f);
934 TFLITE_DCHECK_LT(rmin, rmax);
938 int quant_max = (1 << num_bits) - 1;
939 float nudged_min, nudged_max, nudged_scale;
940 NudgeQuantizationRange(rmin, rmax, quant_min, quant_max, &nudged_min, &nudged_max, &nudged_scale);
942 FakeQuantizeArray(nudged_scale, nudged_min, nudged_max, input_data, output_data, flat_size);
956 const RuntimeShape &indices_shape)
961 const int indices_dims = indices_shape.DimensionsCount();
962 ret.
indices_nd = indices_shape.Dims(indices_dims - 1);
963 const int params_dims = params_shape.DimensionsCount();
964 for (
int i = 0; i < indices_dims - 1; ++i)
966 ret.
n_slices *= indices_shape.Dims(i);
968 for (
int i = ret.
indices_nd; i < params_dims; ++i)
973 int remain_flat_size = params_shape.FlatSize();
977 ret.
dims_to_count[i] = remain_flat_size / params_shape.Dims(i);
984template <
typename ParamsT,
typename IndicesT =
int32>
985inline void GatherNd(
const RuntimeShape ¶ms_shape,
const ParamsT *params_data,
986 const RuntimeShape &indices_shape,
const IndicesT *indices_data,
989 ruy::profiler::ScopeLabel label(
"GatherNd");
992 for (
int i = 0; i < res.
n_slices; ++i)
999 std::memcpy(output_data + i * res.
slice_size, params_data + from_pos,
1004#ifndef TF_LITE_STATIC_MEMORY
1005template <
typename IndicesT =
int32>
1006inline void GatherNdString(
const RuntimeShape ¶ms_shape,
const TfLiteTensor *params_data,
1007 const RuntimeShape &indices_shape,
const IndicesT *indices_data,
1008 const RuntimeShape &
output_shape, TfLiteTensor *output_data)
1010 ruy::profiler::ScopeLabel label(
"GatherNdString");
1013 DynamicBuffer buffer;
1014 for (
int i = 0; i < res.
n_slices; ++i)
1023 buffer.AddString(GetString(params_data, from_pos + j));
1026 buffer.WriteToTensor(output_data,
nullptr);
1030template <
typename IndicesT,
typename UpdatesT>
1031inline void ScatterNd(
const RuntimeShape &indices_shape,
const IndicesT *indices_data,
1032 const RuntimeShape &updates_shape,
const UpdatesT *updates_data,
1033 const RuntimeShape &
output_shape, UpdatesT *output_data)
1035 ruy::profiler::ScopeLabel label(
"ScatterNd");
1039 const int outer_dims = indices_shape.DimensionsCount() - 1;
1040 const int indices_nd = indices_shape.Dims(outer_dims);
1041 const int updates_dims = updates_shape.DimensionsCount();
1042 for (
int i = 0; i < outer_dims; ++i)
1044 n_slices *= indices_shape.Dims(i);
1046 for (
int i = outer_dims; i < updates_dims; ++i)
1048 slice_size *= updates_shape.Dims(i);
1052 int remain_flat_size = output_flat_size;
1053 std::vector<int> dims_to_count(indices_nd, 0);
1054 for (
int i = 0; i < indices_nd; ++i)
1056 dims_to_count[i] = remain_flat_size /
output_shape.Dims(i);
1057 remain_flat_size = dims_to_count[i];
1060 memset(output_data, 0,
sizeof(UpdatesT) * output_flat_size);
1061 for (
int i = 0; i < n_slices; ++i)
1064 for (
int j = 0; j < indices_nd; ++j)
1066 IndicesT idx = indices_data[i * indices_nd + j];
1068 to_pos += idx * dims_to_count[j];
1070 for (
int j = 0; j < slice_size; j++)
1072 output_data[to_pos + j] += updates_data[i * slice_size + j];
1077template <
typename T>
1078inline void Slice(
const tflite::SliceParams &op_params,
const RuntimeShape &input_shape,
1079 const RuntimeShape &
output_shape, SequentialTensorWriter<T> *writer)
1081 const RuntimeShape ext_shape = RuntimeShape::ExtendedShape(5, input_shape);
1082 TFLITE_DCHECK_LE(op_params.begin_count, 5);
1083 TFLITE_DCHECK_LE(op_params.size_count, 5);
1087 std::array<int, 5> start;
1088 std::array<int, 5> stop;
1089 for (
int i = 0; i < 5; ++i)
1091 int padded_i = 5 - i;
1095 : start[i] + op_params.size[
size_count - padded_i];
1098 for (
int i0 = start[0]; i0 < stop[0]; ++i0)
1100 for (
int i1 = start[1]; i1 < stop[1]; ++i1)
1102 for (
int i2 = start[2]; i2 < stop[2]; ++i2)
1104 for (
int i3 = start[3]; i3 < stop[3]; ++i3)
1106 for (
int i4 = start[4]; i4 < stop[4]; ++i4)
1108 writer->Write(
Offset(ext_shape, i0, i1, i2, i3, i4));
1116template <
typename T>
1117inline void Slice(
const tflite::SliceParams &op_params,
const RuntimeShape &input_shape,
1118 const T *input_data,
const RuntimeShape &
output_shape, T *output_data)
1120 SequentialTensorWriter<T> writer(input_data, output_data);
1124template <
typename T>
1125inline void Slice(
const tflite::SliceParams &op_params,
const RuntimeShape &input_shape,
1126 const TfLiteTensor *input,
const RuntimeShape &
output_shape, TfLiteTensor *output)
1128 SequentialTensorWriter<T> writer(input, output);
1132template <
typename T>
1133void Minimum(
const RuntimeShape &input1_shape,
const T *input1_data,
const T *input2_data,
1138 auto min_value = input2_data[0];
1139 for (
int i = 0; i < flat_size; i++)
1141 output_data[i] = input1_data[i] > min_value ? min_value : input1_data[i];
1147template <
typename T>
1148inline void Minimum(
const RuntimeShape &input1_shape,
const T *input1_data,
const RuntimeShape &,
1149 const T *input2_data,
const RuntimeShape &
output_shape, T *output_data)
1155template <
typename T>
1156void Maximum(
const RuntimeShape &input1_shape,
const T *input1_data,
const T *input2_data,
1161 auto max_value = input2_data[0];
1162 for (
int i = 0; i < flat_size; i++)
1164 output_data[i] = input1_data[i] < max_value ? max_value : input1_data[i];
1170template <
typename T>
1171inline void Maximum(
const RuntimeShape &input1_shape,
const T *input1_data,
const RuntimeShape &,
1172 const T *input2_data,
const RuntimeShape &
output_shape, T *output_data)
1178template <
typename T1,
typename T2,
typename T3>
1179void ArgMax(
const RuntimeShape &input1_shape,
const T1 *input1_data,
const T3 *input2_data,
1182 ArgMinMax(input1_shape, input1_data, input2_data,
output_shape, output_data, std::greater<T1>());
1187template <
typename T1,
typename T2,
typename T3>
1188inline void ArgMax(
const RuntimeShape &input1_shape,
const T1 *input1_data,
1189 const RuntimeShape &input2_shape,
const T3 *input2_data,
1196template <
typename D,
typename T>
1197void Select(
const RuntimeShape &input_condition_shape,
const D *input_condition_data,
1198 const RuntimeShape &input_x_shape,
const T *input_x_data,
1199 const RuntimeShape &input_y_shape,
const T *input_y_data,
1205 if (input_condition_shape.FlatSize() == 1 && input_x_shape.FlatSize() == 1 &&
1206 input_y_shape.FlatSize() == 1 &&
output_shape.FlatSize() == 1)
1214 for (int64_t i = 0; i < flatsize; ++i)
1216 output_data[i] = input_condition_data[i] ? input_x_data[i] : input_y_data[i];
1220template <
typename D,
typename T>
1221void RankOneSelect(
const RuntimeShape &input_condition_shape,
const D *input_condition_data,
1222 const RuntimeShape &input_x_shape,
const T *input_x_data,
1223 const RuntimeShape &input_y_shape,
const T *input_y_data,
1226 const int64_t outer_size = input_condition_shape.FlatSize();
1228 if (input_condition_shape.DimensionsCount() == 0)
1234 TFLITE_DCHECK_EQ(MatchingDim(input_x_shape, 0, input_y_shape, 0,
output_shape, 0), outer_size);
1235 inner_size = MatchingFlatSizeSkipDim(input_x_shape, 0, input_y_shape,
output_shape);
1239 for (int64_t i = 0; i < outer_size; i++)
1241 const T *input_data = input_condition_data[i] ? input_x_data : input_y_data;
1242 memcpy(output_data +
offset, input_data +
offset, inner_size *
sizeof(T));
1247template <
typename D,
typename T>
1249 const RuntimeShape &input_x_shape,
const T *input_x_data,
1250 const RuntimeShape &input_y_shape,
const T *input_y_data,
1253 TFLITE_DCHECK_LE(input_condition_shape.DimensionsCount(), 4);
1254 TFLITE_DCHECK_LE(input_x_shape.DimensionsCount(), 4);
1255 TFLITE_DCHECK_LE(input_y_shape.DimensionsCount(), 4);
1258 const RuntimeShape extended_output_shape = RuntimeShape::ExtendedShape(4,
output_shape);
1264 &desc_condition, &desc_x, &desc_y);
1277 for (
int b = 0; b < extended_output_shape.Dims(0); ++b)
1279 for (
int y = 0; y < extended_output_shape.Dims(1); ++y)
1281 for (
int x = 0; x < extended_output_shape.Dims(2); ++x)
1283 for (
int c = 0; c < extended_output_shape.Dims(3); ++c)
1288 output_data[
Offset(extended_output_shape, b, y, x, c)] =
1289 input_condition_data[condition_index] ? input_x_data[x_index] : input_y_data[y_index];
1296template <
typename D,
typename T>
1300 const size_t size = input_condition_shape.FlatSize();
1306 const size_t cond_rank = input_condition_shape.DimensionsCount();
1308 std::vector<int> dims_to_count(cond_rank, 0);
1309 int cur_flat_size =
size;
1310 for (
int i = 0; i < cond_rank; ++i)
1312 dims_to_count[i] = cur_flat_size / input_condition_shape.Dims(i);
1313 cur_flat_size = dims_to_count[i];
1316 int output_index = 0;
1317 for (
int i = 0; i <
size; ++i)
1319 if (input_condition_data[i])
1323 for (
int j = 0; j < cond_rank; ++j)
1325 int coord_j = flat_index / dims_to_count[j];
1326 output_data[output_index * cond_rank + j] = coord_j;
1327 flat_index %= dims_to_count[j];
1335template <
typename T,
typename TI>
1336inline void SparseToDense(
const std::vector<std::vector<TI>> &indices,
const T *values,
1337 T default_value,
bool value_is_scalar,
1338 const RuntimeShape &unextended_output_shape, T *output_data)
1340 TFLITE_DCHECK_LE(unextended_output_shape.DimensionsCount(), 4);
1341 const RuntimeShape
output_shape = RuntimeShape::ExtendedShape(4, unextended_output_shape);
1342 const int value_count = indices.size();
1346 for (
int i = 0; i < num_elements; ++i)
1348 output_data[i] = default_value;
1353 if (value_is_scalar)
1355 for (
int i = 0; i < value_count; ++i)
1357 const std::vector<TI> &index = indices[i];
1358 TFLITE_DCHECK_EQ(index.size(), 4);
1359 const T value = *values;
1366 for (
int i = 0; i < value_count; ++i)
1368 const std::vector<TI> &index = indices[i];
1369 TFLITE_DCHECK_EQ(index.size(), 4);
1370 const T value = values[i];
1375template <
typename T>
1376inline void Pow(
const RuntimeShape &input1_shape,
const T *input1_data,
1377 const RuntimeShape &input2_shape,
const T *input2_data,
1381 for (
int i = 0; i < flat_size; ++i)
1383 output_data[i] = std::pow(input1_data[i], input2_data[i]);
1387template <
typename T>
1389 const RuntimeShape &unextended_input2_shape,
const T *input2_data,
1390 const RuntimeShape &unextended_output_shape, T *output_data)
1392 TFLITE_DCHECK_LE(unextended_input1_shape.DimensionsCount(), 4);
1393 TFLITE_DCHECK_LE(unextended_input2_shape.DimensionsCount(), 4);
1394 TFLITE_DCHECK_LE(unextended_output_shape.DimensionsCount(), 4);
1395 const RuntimeShape
output_shape = RuntimeShape::ExtendedShape(4, unextended_output_shape);
1413 auto in1_val = input1_data[in1_idx];
1414 auto in2_val = input2_data[in2_idx];
1415 output_data[out_idx] = std::pow(in1_val, in2_val);
1422template <
typename Scalar>
1423void Reverse(
int axis,
const RuntimeShape &input_shape,
const Scalar *input_data,
1426 ruy::profiler::ScopeLabel label(
"Reverse");
1429 for (
int i = 0; i < axis; ++i)
1431 outer_size *= input_shape.Dims(i);
1435 for (
int i = axis + 1; i < input_shape.DimensionsCount(); ++i)
1437 copy_size *= input_shape.Dims(i);
1440 const int dims_at_axis = input_shape.Dims(axis);
1441 for (
int i = 0; i < outer_size; ++i)
1443 for (
int j = 0; j < dims_at_axis; ++j)
1445 const int start_pos = (i * dims_at_axis + j) * copy_size;
1446 Scalar *output_ptr = output_data + start_pos;
1447 int loc = (i * dims_at_axis + dims_at_axis - j - 1) * copy_size;
1448 memcpy(output_ptr, input_data + loc, copy_size *
sizeof(Scalar));
1453template <
typename Scalar,
typename TS>
1455 const RuntimeShape &input_shape,
const Scalar *input_data,
1458 ruy::profiler::ScopeLabel label(
"ReverseSequence");
1461 int outer_dim = std::min(batch_dim, seq_dim);
1462 int medium_dim = std::max(batch_dim, seq_dim);
1463 for (
int i = 0; i < outer_dim; ++i)
1465 outer_size *= input_shape.Dims(i);
1468 int medium_size = 1;
1469 for (
int i = outer_dim + 1; i < medium_dim; ++i)
1471 medium_size *= input_shape.Dims(i);
1475 for (
int i = medium_dim + 1; i < input_shape.DimensionsCount(); ++i)
1477 copy_size *= input_shape.Dims(i);
1480 const int dims_at_outer_dim = input_shape.Dims(outer_dim);
1481 const int dims_at_medium_dim = input_shape.Dims(medium_dim);
1484 if (batch_dim > seq_dim)
1486 for (
int i = 0; i < outer_size; ++i)
1488 for (
int j = 0; j < dims_at_outer_dim; ++j)
1490 const int in_pos_base = (i * dims_at_outer_dim + j) * medium_size;
1491 for (
int p = 0; p < medium_size; ++p)
1493 for (
int q = 0; q < dims_at_medium_dim; ++q)
1495 const int in_pos = ((in_pos_base + p) * dims_at_medium_dim + q) * copy_size;
1496 const Scalar *in_ptr = input_data + in_pos;
1497 int sl = seq_lengths[q] - 1;
1500 output_ptr = output_data + in_pos;
1504 const int out_pos_base = (i * dims_at_outer_dim + sl - j) * medium_size;
1505 const int out_pos = ((out_pos_base + p) * dims_at_medium_dim + q) * copy_size;
1506 output_ptr = output_data + out_pos;
1508 memcpy(output_ptr, in_ptr, copy_size *
sizeof(Scalar));
1514 else if (batch_dim < seq_dim)
1516 for (
int i = 0; i < outer_size; ++i)
1518 for (
int j = 0; j < dims_at_outer_dim; ++j)
1520 const int in_pos_base = (i * dims_at_outer_dim + j) * medium_size;
1521 int sl = seq_lengths[j] - 1;
1522 const int out_pos_base = (i * dims_at_outer_dim + j) * medium_size;
1523 for (
int p = 0; p < medium_size; ++p)
1525 for (
int q = 0; q < dims_at_medium_dim; ++q)
1527 const int in_pos = ((in_pos_base + p) * dims_at_medium_dim + q) * copy_size;
1528 const Scalar *in_ptr = input_data + in_pos;
1531 output_ptr = output_data + in_pos;
1535 const int out_pos = ((out_pos_base + p) * dims_at_medium_dim + sl - q) * copy_size;
1536 output_ptr = output_data + out_pos;
1538 memcpy(output_ptr, in_ptr, copy_size *
sizeof(Scalar));
1546template <
typename T>
1547inline void SegmentSum(
const RuntimeShape &input_shape,
const T *input_data,
1548 const RuntimeShape &segment_ids_shape,
const int32_t *segment_ids_data,
1551 const int segment_flat_size = MatchingFlatSizeSkipDim(input_shape, 0,
output_shape);
1553 memset(output_data, 0,
sizeof(T) *
output_shape.FlatSize());
1555 for (
int i = 0; i < input_shape.Dims(0); i++)
1557 int output_index = segment_ids_data[i];
1558 for (
int j = 0; j < segment_flat_size; ++j)
1560 output_data[output_index * segment_flat_size + j] += input_data[i * segment_flat_size + j];
void Concatenation(int concat_dim, const Scalar *const *input_data, const Dims< 4 > *const *input_dims, int inputs_count, Scalar *output_data, const Dims< 4 > &output_dims)
void FullyConnected(const float *input_data, const Dims< 4 > &input_dims, const float *weights_data, const Dims< 4 > &weights_dims, const float *bias_data, const Dims< 4 > &bias_dims, float *output_data, const Dims< 4 > &output_dims)
void NdArrayDescsForElementwiseBroadcast(const Dims< N > &input0_dims, const Dims< N > &input1_dims, NdArrayDesc< N > *desc0_out, NdArrayDesc< N > *desc1_out)
int SubscriptToIndex(const NdArrayDesc< 4 > &desc, int i0, int i1, int i2, int i3)
int Offset(const Dims< 4 > &dims, int i0, int i1, int i2, int i3)
int MatchingFlatSize(const Dims< N > &dims, const Dims< N > &check_dims_0)
__global uchar * offset(const Image *img, int x, int y)
const luci_interpreter::RuntimeShape output_shape
void GatherNd(const RuntimeShape ¶ms_shape, const ParamsT *params_data, const RuntimeShape &indices_shape, const IndicesT *indices_data, const RuntimeShape &output_shape, ParamsT *output_data)
void LocalResponseNormalization(const tflite::LocalResponseNormalizationParams &op_params, const RuntimeShape &input_shape, const float *input_data, const RuntimeShape &output_shape, float *output_data)
void FakeQuant(const tflite::FakeQuantParams &op_params, const RuntimeShape &input_shape, const float *input_data, const RuntimeShape &output_shape, float *output_data)
void Sub16(const ArithmeticParams ¶ms, const RuntimeShape &input1_shape, const int16_t *input1_data, const RuntimeShape &input2_shape, const int16_t *input2_data, const RuntimeShape &output_shape, int16_t *output_data)
void LstmCell(const LstmCellParams ¶ms, const RuntimeShape &unextended_input_shape, const float *input_data, const RuntimeShape &unextended_prev_activ_shape, const float *prev_activ_data, const RuntimeShape &weights_shape, const float *weights_data, const RuntimeShape &unextended_bias_shape, const float *bias_data, const RuntimeShape &unextended_prev_state_shape, const float *prev_state_data, const RuntimeShape &unextended_output_state_shape, float *output_state_data, const RuntimeShape &unextended_output_activ_shape, float *output_activ_data, const RuntimeShape &unextended_concat_temp_shape, float *concat_temp_data, const RuntimeShape &unextended_activ_temp_shape, float *activ_temp_data)
void Relu1(const RuntimeShape &input_shape, const T *input_data, const RuntimeShape &output_shape, T *output_data)
void Dequantize(const RuntimeShape &input_shape, const Eigen::half *input_data, const RuntimeShape &output_shape, float *output_data)
GatherNdHelperResult GatherNdHelper(const RuntimeShape ¶ms_shape, const RuntimeShape &indices_shape)
void GatherNdString(const RuntimeShape ¶ms_shape, const TfLiteTensor *params_data, const RuntimeShape &indices_shape, const IndicesT *indices_data, const RuntimeShape &output_shape, TfLiteTensor *output_data)
void SparseToDense(const std::vector< std::vector< TI > > &indices, const T *values, T default_value, bool value_is_scalar, const RuntimeShape &unextended_output_shape, T *output_data)
void ArgMax(const RuntimeShape &input1_shape, const T1 *input1_data, const T3 *input2_data, const RuntimeShape &output_shape, T2 *output_data)
void ReluX(const tflite::ReluParams ¶ms, const RuntimeShape &input_shape, const T *input_data, const RuntimeShape &output_shape, T *output_data)
void Select(const RuntimeShape &input_condition_shape, const D *input_condition_data, const RuntimeShape &input_x_shape, const T *input_x_data, const RuntimeShape &input_y_shape, const T *input_y_data, const RuntimeShape &output_shape, T *output_data)
void BroadcastPow4DSlow(const RuntimeShape &unextended_input1_shape, const T *input1_data, const RuntimeShape &unextended_input2_shape, const T *input2_data, const RuntimeShape &unextended_output_shape, T *output_data)
void ReverseSequence(const TS *seq_lengths, const int seq_dim, const int batch_dim, const RuntimeShape &input_shape, const Scalar *input_data, const RuntimeShape &output_shape, Scalar *output_data)
void BroadcastSelect4DSlow(const RuntimeShape &input_condition_shape, const D *input_condition_data, const RuntimeShape &input_x_shape, const T *input_x_data, const RuntimeShape &input_y_shape, const T *input_y_data, const RuntimeShape &output_shape, T *output_data)
void Minimum(const RuntimeShape &input1_shape, const T *input1_data, const T *input2_data, const RuntimeShape &output_shape, T *output_data)
void RankOneSelect(const RuntimeShape &input_condition_shape, const D *input_condition_data, const RuntimeShape &input_x_shape, const T *input_x_data, const RuntimeShape &input_y_shape, const T *input_y_data, const RuntimeShape &output_shape, T *output_data)
void Pow(const RuntimeShape &input1_shape, const T *input1_data, const RuntimeShape &input2_shape, const T *input2_data, const RuntimeShape &output_shape, T *output_data)
void Mul(const ArithmeticParams ¶ms, const RuntimeShape &input1_shape, const int16 *input1_data, const RuntimeShape &input2_shape, const int16 *input2_data, const RuntimeShape &output_shape, int16 *output_data)
void SelectTrueCoords(const RuntimeShape &input_condition_shape, const D *input_condition_data, T *output_data)
void Slice(const tflite::SliceParams &op_params, const RuntimeShape &input_shape, const RuntimeShape &output_shape, SequentialTensorWriter< T > *writer)
void SegmentSum(const RuntimeShape &input_shape, const T *input_data, const RuntimeShape &segment_ids_shape, const int32_t *segment_ids_data, const RuntimeShape &output_shape, T *output_data)
void DepthConcatenation(const ConcatenationParams ¶ms, const RuntimeShape *const *input_shapes, const Scalar *const *input_data, const RuntimeShape &output_shape, Scalar *output_data)
void PackWithScaling(const PackParams ¶ms, const RuntimeShape *const *input_shapes, const uint8 *const *input_data, const RuntimeShape &output_shape, uint8 *output_data)
void Pack(const PackParams ¶ms, const RuntimeShape *const *input_shapes, const Scalar *const *input_data, const RuntimeShape &output_shape, Scalar *output_data)
void BroadcastMulFivefold(const ArithmeticParams &unswitched_params, const RuntimeShape &unswitched_input1_shape, const uint8 *unswitched_input1_data, const RuntimeShape &unswitched_input2_shape, const uint8 *unswitched_input2_data, const RuntimeShape &output_shape, uint8 *output_data)
int NodeOffset(int b, int h, int w, int height, int width)
void Reverse(int axis, const RuntimeShape &input_shape, const Scalar *input_data, const RuntimeShape &output_shape, Scalar *output_data)
void Relu6(const RuntimeShape &input_shape, const float *input_data, const RuntimeShape &output_shape, float *output_data)
void Unpack(const UnpackParams ¶ms, const RuntimeShape &input_shape, const Scalar *input_data, const RuntimeShape &output_shape, Scalar *const *output_datas)
void Split(const SplitParams ¶ms, const RuntimeShape &input_shape, const Scalar *input_data, const RuntimeShape *const *output_shapes, Scalar *const *output_data)
void Maximum(const RuntimeShape &input1_shape, const T *input1_data, const T *input2_data, const RuntimeShape &output_shape, T *output_data)
void ScatterNd(const RuntimeShape &indices_shape, const IndicesT *indices_data, const RuntimeShape &updates_shape, const UpdatesT *updates_data, const RuntimeShape &output_shape, UpdatesT *output_data)
std::vector< int > dims_to_count