onert::exec::SingleModelExecutors Class Reference

Class to gather executor set for single model NN package. More...

#include <SingleModelExecutors.h>

Collaboration diagram for onert::exec::SingleModelExecutors:

Public Member Functions
	SingleModelExecutors (void)=default
	Construct a new SingleModelExecutors object.
	SingleModelExecutors (const SingleModelExecutors &)=delete
	SingleModelExecutors (SingleModelExecutors &&)=default
	~SingleModelExecutors ()=default
	Destroy the SingleModelExecutors object.
void	emplace (const ir::ModelIndex &model_index, const ir::SubgraphIndex &subg_index, std::unique_ptr< IExecutor > exec) override
	Insert executor in executor set.
IExecutor *	at (const ir::ModelIndex &model_index, const ir::SubgraphIndex &subg_index) const override
	Return executor of index.
uint32_t	inputSize () const override
	Return executor set's number of input.
uint32_t	outputSize () const override
	Return executor set's number of output.
const ir::OperandInfo &	inputInfo (const ir::IOIndex &index) const override
	Return NN package input tensor info.
const ir::OperandInfo &	outputInfo (const ir::IOIndex &index) const override
	Return NN package output tensor info.
void	execute (const ExecutionContext &ctx) override
	Execute NN package executor set.
Public Member Functions inherited from onert::exec::IExecutors
virtual	~IExecutors ()=default
	Virtual IExecutors destructor.
IExecutor *	entryExecutor () const

Detailed Description

Class to gather executor set for single model NN package.

Definition at line 29 of file SingleModelExecutors.h.

Constructor & Destructor Documentation

SingleModelExecutors() [1/3]

onert::exec::SingleModelExecutors::SingleModelExecutors ( void  )

default

Construct a new SingleModelExecutors object.

SingleModelExecutors() [2/3]

onert::exec::SingleModelExecutors::SingleModelExecutors ( const SingleModelExecutors &  )

delete

SingleModelExecutors() [3/3]

onert::exec::SingleModelExecutors::SingleModelExecutors ( SingleModelExecutors &&  )

default

~SingleModelExecutors()

onert::exec::SingleModelExecutors::~SingleModelExecutors ( )

default

Destroy the SingleModelExecutors object.

Member Function Documentation

at()

IExecutor * onert::exec::SingleModelExecutors::at ( const ir::ModelIndex &  model_index, const ir::SubgraphIndex &  subg_index  ) const

overridevirtual

Return executor of index.

Parameters

[in]	model_index	Model index
[in]	subg_index	Subgraph index

Returns

Executor

Implements onert::exec::IExecutors.

Definition at line 32 of file SingleModelExecutors.cc.

{
  return _executors.at(subg_index).get();
}

emplace()

void onert::exec::SingleModelExecutors::emplace ( const ir::ModelIndex &  model_index, const ir::SubgraphIndex &  subg_index, std::unique_ptr< IExecutor >  exec  )

overridevirtual

Insert executor in executor set.

Parameters

[in]	model_index	Model index
[in]	subg_index	Subgraph index
[in]	exec	Executor to insert

Implements onert::exec::IExecutors.

Definition at line 26 of file SingleModelExecutors.cc.

{
  _executors.emplace(subg_index, std::move(exec));
}

execute()

void onert::exec::SingleModelExecutors::execute ( const ExecutionContext &  ctx )

overridevirtual

Execute NN package executor set.

Parameters

[in]

ctx

Execution context

Implements onert::exec::IExecutors.

Definition at line 52 of file SingleModelExecutors.cc.

{
  // UserTensor for Input/Output
  std::vector<std::unique_ptr<backend::builtin::UserTensor>> tensorpool;
 
  // EdgeTensor for Input Quantization / Output Dequantization
  std::vector<std::unique_ptr<EdgeTensor>> qtensorpool;
 
  // Input/Output Tensor vector for executor
  std::vector<backend::IPortableTensor *> inputs(ctx.desc.inputs.size());
  std::vector<backend::IPortableTensor *> outputs(ctx.desc.outputs.size());
 
  // Vector for input quantization I/O
  std::vector<backend::ITensor *> input_tensors;
  std::vector<backend::ITensor *> input_qtensors;
  std::vector<ir::PermuteType> input_permute_types;
 
  // Vector for output dequantization I/O
  std::vector<backend::ITensor *> output_qtensors;
  std::vector<backend::ITensor *> output_tensors;
  std::vector<ir::PermuteType> output_permute_types;
 
  // Prepare UserTensor and EdgeTensor for input quantization
  for (uint32_t i = 0; i < inputs.size(); i++)
  {
    auto &desc = ctx.desc.inputs[i];
 
    // Input is optional if buffer is nullptr, and optional input's size is 0
    if (desc->buffer == nullptr && (desc->size != 0 || desc->info.total_size() != 0))
      throw std::runtime_error{"Input " + std::to_string(i) + "'s buffer is not set."};
 
    tensorpool.emplace_back(std::make_unique<backend::builtin::UserTensor>(
      desc->info, desc->layout, const_cast<uint8_t *>(static_cast<const uint8_t *>(desc->buffer)),
      desc->size));
 
    auto user_type = desc->info.typeInfo().type();
    auto &model_info = entryExecutor()->inputInfo(i).typeInfo();
    auto model_type = model_info.type();
    if ((user_type != model_type && user_type == ir::DataType::FLOAT32) ||
        (desc->layout == ir::Layout::NCHW))
    {
      auto quantized_info = desc->info;
      quantized_info.typeInfo(model_info);
      qtensorpool.emplace_back(
        std::make_unique<EdgeTensor>(quantized_info, entryExecutor()->inputLayout(i)));
      qtensorpool.back()->allocate_buffer();
 
      input_tensors.push_back(tensorpool.back().get());
      input_qtensors.push_back(qtensorpool.back().get());
      inputs[i] = qtensorpool.back().get();
      if (desc->layout == ir::Layout::NCHW)
        input_permute_types.push_back(ir::PermuteType::NCHW_TO_NHWC);
      else
        input_permute_types.push_back(ir::PermuteType::COPY);
    }
    else
      inputs[i] = tensorpool.back().get();
  }
 
  // Prepare UserTensor and EdgeTensor for output dequantization
  for (uint32_t i = 0; i < outputs.size(); i++)
  {
    auto &desc = ctx.desc.outputs[i];
 
    // Output is optional if buffer is nullptr, and optional output's size is 0
    if (desc->buffer == nullptr && (desc->size != 0 || desc->info.total_size() != 0))
      throw std::runtime_error{"Output " + std::to_string(i) + "'s buffer is not set."};
 
    tensorpool.emplace_back(std::make_unique<backend::builtin::UserTensor>(
      desc->info, desc->layout, static_cast<uint8_t *>(desc->buffer), desc->size));
 
    auto user_type = desc->info.typeInfo().type();
    auto &model_info = entryExecutor()->outputInfo(i).typeInfo();
    auto model_type = model_info.type();
    if ((user_type != model_type && user_type == ir::DataType::FLOAT32) ||
        (desc->layout == ir::Layout::NCHW))
    {
      auto quantized_info = desc->info;
      quantized_info.typeInfo(model_info);
      qtensorpool.emplace_back(
        std::make_unique<EdgeTensor>(quantized_info, entryExecutor()->outputLayout(i)));
      qtensorpool.back()->allocate_buffer();
 
      output_qtensors.push_back(qtensorpool.back().get());
      output_tensors.push_back(tensorpool.back().get());
      outputs[i] = qtensorpool.back().get();
      if (desc->layout == ir::Layout::NCHW)
        output_permute_types.push_back(ir::PermuteType::NHWC_TO_NCHW);
      else
        output_permute_types.push_back(ir::PermuteType::COPY);
    }
    else
      outputs[i] = tensorpool.back().get();
  }
 
  // Run quantization
  if (input_tensors.size() > 0)
  {
    auto input_quantize_layer = PermuteLayer(input_tensors, input_qtensors, input_permute_types);
    input_quantize_layer.prepare();
    input_quantize_layer.run();
  }
 
  // Executor
  entryExecutor()->execute(inputs, outputs, ctx.options);
 
  // Run dequantization
  if (output_tensors.size() != 0)
  {
    auto output_dequantize_layer =
      PermuteLayer(output_qtensors, output_tensors, output_permute_types);
    output_dequantize_layer.prepare();
    output_dequantize_layer.run();
  }
 
  // Get dynamic shape inference result
  for (uint32_t i = 0; i < outputs.size(); i++)
  {
    if (ctx.desc.outputs[i]->buffer == nullptr)
    {
      // Output is optional if buffer is nullptr
      continue;
    }
 
    ctx.desc.outputs[i]->info.shape(outputs[i]->getShape());
  }
}

References onert::ir::COPY, onert::exec::ExecutionContext::desc, onert::exec::IExecutors::entryExecutor(), onert::exec::IExecutor::execute(), onert::exec::IExecutor::inputInfo(), onert::exec::IODescription::inputs, onert::ir::NCHW, onert::ir::NCHW_TO_NHWC, onert::ir::NHWC_TO_NCHW, onert::exec::ExecutionContext::options, onert::exec::IExecutor::outputInfo(), onert::exec::IODescription::outputs, onert::ir::TypeInfo::type(), and onert::ir::OperandInfo::typeInfo().

inputInfo()

const ir::OperandInfo & onert::exec::SingleModelExecutors::inputInfo ( const ir::IOIndex &  index ) const

overridevirtual

Return NN package input tensor info.

Parameters

[in]

index

Input index

Returns

Tensor info

Implements onert::exec::IExecutors.

Definition at line 42 of file SingleModelExecutors.cc.

{
  return entryExecutor()->inputInfo(index.value());
}

References onert::exec::IExecutors::entryExecutor(), and onert::exec::IExecutor::inputInfo().

inputSize()

uint32_t onert::exec::SingleModelExecutors::inputSize ( ) const

overridevirtual

Return executor set's number of input.

Returns

Number of input

Implements onert::exec::IExecutors.

Definition at line 38 of file SingleModelExecutors.cc.

{ return entryExecutor()->inputSize(); }

References onert::exec::IExecutors::entryExecutor(), and onert::exec::IExecutor::inputSize().

outputInfo()

const ir::OperandInfo & onert::exec::SingleModelExecutors::outputInfo ( const ir::IOIndex &  index ) const

overridevirtual

Return NN package output tensor info.

Parameters

[in]

index

Output index

Returns

Tensor info

Implements onert::exec::IExecutors.

Definition at line 47 of file SingleModelExecutors.cc.

{
  return entryExecutor()->outputInfo(index.value());
}

References onert::exec::IExecutors::entryExecutor(), and onert::exec::IExecutor::outputInfo().

outputSize()

uint32_t onert::exec::SingleModelExecutors::outputSize ( ) const

overridevirtual

Return executor set's number of output.

Returns

Number of output

Implements onert::exec::IExecutors.

Definition at line 40 of file SingleModelExecutors.cc.

{ return entryExecutor()->outputSize(); }

References onert::exec::IExecutors::entryExecutor(), and onert::exec::IExecutor::outputSize().

---

The documentation for this class was generated from the following files:

• runtime/onert/core/src/exec/SingleModelExecutors.h
• runtime/onert/core/src/exec/SingleModelExecutors.cc