onert::exec::MultiModelExecutors Class Reference

Class to gather executors. More...

#include <MultiModelExecutors.h>

Collaboration diagram for onert::exec::MultiModelExecutors:

Public Member Functions
	MultiModelExecutors (void)=delete
	MultiModelExecutors (std::unique_ptr< ir::ModelEdges > model_edges)
	MultiModelExecutors (const MultiModelExecutors &)=delete
	MultiModelExecutors (MultiModelExecutors &&)=default
	~MultiModelExecutors ()=default
void	emplace (const ir::ModelIndex &model_index, const ir::SubgraphIndex &subg_index, std::unique_ptr< IExecutor > exec) override
	Insert executor in executor set.
IExecutor *	at (const ir::ModelIndex &model_index, const ir::SubgraphIndex &subg_index) const override
	Return executor of index.
uint32_t	inputSize () const override
	Return executor set's number of input.
uint32_t	outputSize () const override
	Return executor set's number of output.
const ir::OperandInfo &	inputInfo (const ir::IOIndex &index) const override
	Return NN package input tensor info.
const ir::OperandInfo &	outputInfo (const ir::IOIndex &index) const override
	Return NN package output tensor info.
const void *	outputBuffer (const ir::IOIndex &index) const final
	Return NN package output buffer.
const backend::IPortableTensor *	outputTensor (const ir::IOIndex &index) const final
	Return NN package output tensor.
void	execute (ExecutionContext &ctx) override
	Execute NN package executor set.
Public Member Functions inherited from onert::exec::IExecutors
virtual	~IExecutors ()=default
	Virtual IExecutors destructor.
virtual IExecutor *	entryExecutor () const

Detailed Description

Class to gather executors.

Definition at line 47 of file MultiModelExecutors.h.

Constructor & Destructor Documentation

MultiModelExecutors() [1/4]

onert::exec::MultiModelExecutors::MultiModelExecutors ( void  )

delete

MultiModelExecutors() [2/4]

onert::exec::MultiModelExecutors::MultiModelExecutors ( std::unique_ptr< ir::ModelEdges >  model_edges )

inline

Definition at line 51 of file MultiModelExecutors.h.

    : _executors{}, _model_edges{std::move(model_edges)}, _edge_tensors{},
      _is_created_edge_tensors{false}, _pkg_input_tensors{}, _pkg_output_tensors{}
  {
    for (const auto &edge : _model_edges->edges)
    {
      _edge_map[edge.from].emplace_back(edge.to);
    }
  }

MultiModelExecutors() [3/4]

onert::exec::MultiModelExecutors::MultiModelExecutors ( const MultiModelExecutors &  )

delete

MultiModelExecutors() [4/4]

onert::exec::MultiModelExecutors::MultiModelExecutors ( MultiModelExecutors &&  )

default

~MultiModelExecutors()

onert::exec::MultiModelExecutors::~MultiModelExecutors ( )

default

Member Function Documentation

at()

IExecutor * onert::exec::MultiModelExecutors::at ( const ir::ModelIndex &  model_index, const ir::SubgraphIndex &  subg_index  ) const

overridevirtual

Return executor of index.

Parameters

[in]	model_index	Model index
[in]	subg_index	Subgraph index

Returns

Executor

Implements onert::exec::IExecutors.

Definition at line 64 of file MultiModelExecutors.cc.

{
  return _executors.at(std::make_pair(model_index, subg_index)).get();
}

Referenced by execute(), inputInfo(), outputBuffer(), outputInfo(), and outputTensor().

emplace()

void onert::exec::MultiModelExecutors::emplace ( const ir::ModelIndex &  model_index, const ir::SubgraphIndex &  subg_index, std::unique_ptr< IExecutor >  exec  )

overridevirtual

Insert executor in executor set.

Parameters

[in]	model_index	Model index
[in]	subg_index	Subgraph index
[in]	exec	Executor to insert

Implements onert::exec::IExecutors.

Definition at line 57 of file MultiModelExecutors.cc.

{
  _executors.emplace(std::make_pair(model_index, subg_index), std::move(exec));
}

execute()

void onert::exec::MultiModelExecutors::execute ( ExecutionContext &  ctx )

overridevirtual

Execute NN package executor set.

Parameters

[in,out]

ctx

Execution context. It reflects execution result (ex. output shape inference)

Implements onert::exec::IExecutors.

Definition at line 229 of file MultiModelExecutors.cc.

{
  auto &desc = ctx.desc;
 
  // Check supported multi model package
  checkSupportedMultimodel();
 
  // TODO Move creating edge tensors in compilation stage
  createEdgeTensors();
 
  // TODO Create IOTensors only once and recreate them only if nnpkg info changes
  CreatePkgIOTensors(desc);
 
  // TODO Find better way to schedule order of executors
  auto const model_count = modelCount();
 
  auto find_from = [&](const ir::ModelIndex &model_index, const ir::SubgraphIndex &subg_index,
                       const ir::IOIndex &io_index) {
    for (const auto &edge : _model_edges->edges)
    {
      if ((std::get<ir::ModelIndex>(edge.to) == model_index) &&
          (std::get<ir::SubgraphIndex>(edge.to) == subg_index) &&
          (std::get<ir::IOIndex>(edge.to) == io_index))
        return edge.from;
    }
 
    throw std::runtime_error{"Cannot find edge for model input"};
  };
 
  // Execute each model
  // NOTE May be better to use vector instead of unordered_map for _executors
  for (auto model_index = ir::ModelIndex{0}; model_index.value() < model_count; model_index++)
  {
    // Find executor
    auto executor = at(model_index, ir::SubgraphIndex{0});
 
    // Set IOTensors
    // TODO Set internal IOTensors only once
    std::vector<backend::IPortableTensor *> inputs_inter;
    std::vector<backend::IPortableTensor *> outputs_inter;
    auto const input_size = executor->inputSize();
    auto const output_size = executor->outputSize();
    inputs_inter.resize(input_size);
    outputs_inter.resize(output_size);
 
    // Set inputs of executor
    // TODO Create layer to allocate/deallocate buffers of EdgeTensor for each executor
    for (uint32_t i = 0; i < input_size; i++)
    {
      const auto input_pkg_index = find_input_index(_model_edges->pkg_inputs, model_index,
                                                    ir::SubgraphIndex{0}, ir::IOIndex{i});
      const auto input_io_desc = ir::IODesc{model_index, ir::SubgraphIndex{0}, ir::IOIndex{i}};
      if (input_pkg_index != -1)
      {
        inputs_inter[i] = _pkg_input_tensors[input_io_desc].get();
      }
      else
      {
        auto from_iodesc = find_from(model_index, ir::SubgraphIndex{0}, ir::IOIndex{i});
 
        // Supported only sequantial execution of models
        assert(std::get<ir::ModelIndex>(from_iodesc).value() < model_index.value());
        assert(std::get<ir::SubgraphIndex>(from_iodesc).value() == 0);
        inputs_inter[i] = _edge_tensors.at(from_iodesc).get();
        assert(inputs_inter[i]->buffer() != nullptr);
      }
    }
 
    // Set outputs of executor
    for (uint32_t i = 0; i < output_size; i++)
    {
      const auto output_pkg_index = find_output_index(_model_edges->pkg_outputs, model_index,
                                                      ir::SubgraphIndex{0}, ir::IOIndex{i});
      const auto output_io_desc = ir::IODesc{model_index, ir::SubgraphIndex{0}, ir::IOIndex{i}};
      if (output_pkg_index != -1)
      {
        outputs_inter[i] = _pkg_output_tensors[output_io_desc].get();
      }
      else
      {
        // Allocate buffer of `from` tensors
        const auto from_iodesc = ir::IODesc{model_index, ir::SubgraphIndex{0}, ir::IOIndex{i}};
        _edge_tensors[from_iodesc]->allocate_buffer();
        outputs_inter[i] = _edge_tensors[from_iodesc].get();
 
        // Increase reference count of `from` tensors for edges
        for (uint32_t i = 0; i < _edge_map[from_iodesc].size(); i++)
          _edge_tensors[from_iodesc]->increase_ref();
      }
    }
 
    executor->execute(inputs_inter, outputs_inter, ctx.options);
 
    // Release input buffers that are no longer needed
    for (uint32_t i = 0; i < input_size; i++)
    {
      const auto input_pkg_index = find_input_index(_model_edges->pkg_inputs, model_index,
                                                    ir::SubgraphIndex{0}, ir::IOIndex{i});
      if (input_pkg_index == -1)
      {
        // Decrease reference count of `from` tensor if input tensor is the `from` tensor
        const auto from_iodesc = find_from(model_index, ir::SubgraphIndex{0}, ir::IOIndex{i});
        _edge_tensors[from_iodesc]->decrease_ref();
      }
    }
 
    // Get dynamic shape inference result
    for (uint32_t i = 0; i < output_size; i++)
    {
      const auto output_pkg_index = find_output_index(_model_edges->pkg_outputs, model_index,
                                                      ir::SubgraphIndex{0}, ir::IOIndex{i});
 
      if (output_pkg_index != -1)
      {
        const auto output_io_tensor =
          dynamic_cast<const backend::builtin::IOTensor *>(outputTensor(ir::IOIndex{i}));
        if (!output_io_tensor)
          throw std::runtime_error{"Output tensor must be IOTensor"};
 
        ctx.desc.outputs[output_pkg_index].info.shape(output_io_tensor->get_info().shape());
      }
    }
  }
}

References at(), onert::exec::ExecutionContext::desc, onert::exec::IExecutor::inputSize(), onert::exec::ExecutionContext::options, onert::exec::IODescription::outputs, outputTensor(), and onert::util::Index< T, DummyTag >::value().

inputInfo()

const ir::OperandInfo & onert::exec::MultiModelExecutors::inputInfo ( const ir::IOIndex &  index ) const

overridevirtual

Return NN package input tensor info.

Parameters

[in]

index

Input index

Returns

Tensor info

Implements onert::exec::IExecutors.

Definition at line 74 of file MultiModelExecutors.cc.

{
  auto const [model_index, subg_index, io_index] = _model_edges->pkg_inputs[index.value()];
  auto const executor = at(model_index, subg_index);
  return executor->inputInfo(io_index.value());
}

References at(), and onert::util::Index< T, DummyTag >::value().

inputSize()

uint32_t onert::exec::MultiModelExecutors::inputSize ( ) const

overridevirtual

Return executor set's number of input.

Returns

Number of input

Implements onert::exec::IExecutors.

Definition at line 70 of file MultiModelExecutors.cc.

{ return _model_edges->pkg_inputs.size(); }

outputBuffer()

const void * onert::exec::MultiModelExecutors::outputBuffer ( const ir::IOIndex &  index ) const

finalvirtual

Return NN package output buffer.

Parameters

[in]

index

Output index

Returns

Buffer of output

Implements onert::exec::IExecutors.

Definition at line 88 of file MultiModelExecutors.cc.

{
  auto const [model_index, subg_index, io_index] = _model_edges->pkg_outputs[index.value()];
  auto const executor = at(model_index, subg_index);
  return static_cast<const void *>(executor->outputBuffer(io_index.value()));
}

References at(), and onert::util::Index< T, DummyTag >::value().

outputInfo()

const ir::OperandInfo & onert::exec::MultiModelExecutors::outputInfo ( const ir::IOIndex &  index ) const

overridevirtual

Return NN package output tensor info.

Parameters

[in]

index

Output index

Returns

Tensor info

Implements onert::exec::IExecutors.

Definition at line 81 of file MultiModelExecutors.cc.

{
  auto const [model_index, subg_index, io_index] = _model_edges->pkg_outputs[index.value()];
  auto const executor = at(model_index, subg_index);
  return executor->outputInfo(io_index.value());
}

References at(), and onert::util::Index< T, DummyTag >::value().

outputSize()

uint32_t onert::exec::MultiModelExecutors::outputSize ( ) const

overridevirtual

Return executor set's number of output.

Returns

Number of output

Implements onert::exec::IExecutors.

Definition at line 72 of file MultiModelExecutors.cc.

{ return _model_edges->pkg_outputs.size(); }

outputTensor()

const backend::IPortableTensor * onert::exec::MultiModelExecutors::outputTensor ( const ir::IOIndex &  index ) const

finalvirtual

Return NN package output tensor.

Parameters

[in]

index

Output index

Returns

Tensor of output

Implements onert::exec::IExecutors.

Definition at line 95 of file MultiModelExecutors.cc.

{
  auto const [model_index, subg_index, io_index] = _model_edges->pkg_outputs[index.value()];
  auto const executor = at(model_index, subg_index);
  return executor->outputTensor(io_index.value());
}

References at(), and onert::util::Index< T, DummyTag >::value().

Referenced by execute().

---

The documentation for this class was generated from the following files:

• runtime/onert/core/src/exec/MultiModelExecutors.h
• runtime/onert/core/src/exec/MultiModelExecutors.cc