Class to gather executors. More...

#include <MultiModelExecutors.h>

Collaboration diagram for onert::exec::MultiModelExecutors:

Public Member Functions
	MultiModelExecutors (void)=delete

	MultiModelExecutors (std::unique_ptr< ir::ModelEdges > model_edges)

	MultiModelExecutors (const MultiModelExecutors &)=delete

	MultiModelExecutors (MultiModelExecutors &&)=default

	~MultiModelExecutors ()=default

void	emplace (const ir::ModelIndex &model_index, const ir::SubgraphIndex &subg_index, std::unique_ptr< IExecutor > exec) override
	Insert executor in executor set.

IExecutor *	at (const ir::ModelIndex &model_index, const ir::SubgraphIndex &subg_index) const override
	Return executor of index.

uint32_t	inputSize () const override
	Return executor set's number of input.

uint32_t	outputSize () const override
	Return executor set's number of output.

const ir::OperandInfo &	inputInfo (const ir::IOIndex &index) const override
	Return NN package input tensor info.

const ir::OperandInfo &	outputInfo (const ir::IOIndex &index) const override
	Return NN package output tensor info.

void	execute (const ExecutionContext &ctx) override
	Execute NN package executor set.

Public Member Functions inherited from onert::exec::IExecutors
virtual	~IExecutors ()=default
	Virtual IExecutors destructor.

IExecutor *	entryExecutor () const

Detailed Description

Class to gather executors.

Definition at line 47 of file MultiModelExecutors.h.

Constructor & Destructor Documentation

◆ MultiModelExecutors() [1/4]

onert::exec::MultiModelExecutors::MultiModelExecutors ( void )

delete

◆ MultiModelExecutors() [2/4]

onert::exec::MultiModelExecutors::MultiModelExecutors ( std::unique_ptr< ir::ModelEdges > model_edges )

inline

Definition at line 51 of file MultiModelExecutors.h.

    : _executors{}, _model_edges{std::move(model_edges)}, _edge_quant_layers{},
      _edge_quant_tensors{}, _edge_tensors{}, _is_created_edge_quant_layers{false},
      _pkg_input_quant_layers{}, _pkg_output_quant_layers{}, _pkg_input_quant_tensors{},
      _pkg_output_quant_tensors{}, _pkg_input_tensors{}, _pkg_output_tensors{}
  {
    for (const auto &edge : _model_edges->edges)
    {
      _edge_map[edge.from].emplace_back(edge.to);
    }
  }

◆ MultiModelExecutors() [3/4]

onert::exec::MultiModelExecutors::MultiModelExecutors ( const MultiModelExecutors & )

delete

◆ MultiModelExecutors() [4/4]

onert::exec::MultiModelExecutors::MultiModelExecutors ( MultiModelExecutors && )

default

◆ ~MultiModelExecutors()

onert::exec::MultiModelExecutors::~MultiModelExecutors ( )

default

Member Function Documentation

◆ at()

IExecutor * onert::exec::MultiModelExecutors::at	(	const ir::ModelIndex &	model_index,
		const ir::SubgraphIndex &	subg_index
	)		const

overridevirtual

Return executor of index.

Parameters

[in]	model_index	Model index
[in]	subg_index	Subgraph index

Returns: Executor

Implements onert::exec::IExecutors.

Definition at line 62 of file MultiModelExecutors.cc.

{
  return _executors.at(std::make_pair(model_index, subg_index)).get();
}

Referenced by execute(), inputInfo(), and outputInfo().

◆ emplace()

void onert::exec::MultiModelExecutors::emplace	(	const ir::ModelIndex &	model_index,
		const ir::SubgraphIndex &	subg_index,
		std::unique_ptr< IExecutor >	exec
	)

overridevirtual

Insert executor in executor set.

Parameters

[in]	model_index	Model index
[in]	subg_index	Subgraph index
[in]	exec	Executor to insert

Implements onert::exec::IExecutors.

Definition at line 55 of file MultiModelExecutors.cc.

{
  _executors.emplace(std::make_pair(model_index, subg_index), std::move(exec));
}

◆ execute()

void onert::exec::MultiModelExecutors::execute ( const ExecutionContext & ctx )

overridevirtual

Execute NN package executor set.

Parameters

[in] ctx Execution context

Implements onert::exec::IExecutors.

Definition at line 353 of file MultiModelExecutors.cc.

{
  auto &desc = ctx.desc;
 
  // Check supported multi model package
  checkSupportedMultimodel();
 
  // TODO Move creating type-aware quantization layers for edges in compilation stage
  createEdgeQuantLayers();
 
  // TODO Create IOTensors only once and recreate them only if nnpkg info changes
  CreatePkgIOTensors(desc);
 
  // TODO Create type-aware quantization layers only once and recreate them only if type changes
  createPkgIOQuantLayers(desc);
 
  // TODO Find better way to schedule order of executors
  auto const model_count = modelCount();
 
  auto find_from = [&](const ir::ModelIndex &model_index, const ir::SubgraphIndex &subg_index,
                       const ir::IOIndex &io_index) {
    for (const auto &edge : _model_edges->edges)
    {
      if ((std::get<ir::ModelIndex>(edge.to) == model_index) &&
          (std::get<ir::SubgraphIndex>(edge.to) == subg_index) &&
          (std::get<ir::IOIndex>(edge.to) == io_index))
        return edge.from;
    }
 
    throw std::runtime_error{"Cannot find edge for model input"};
  };
 
  // Execute each model
  // NOTE May be better to use vector instead of unordered_map for _executors
  for (auto model_index = ir::ModelIndex{0}; model_index.value() < model_count; model_index++)
  {
    // Find executor
    auto executor = at(model_index, ir::SubgraphIndex{0});
 
    // Set IOTensors
    // TODO Set internal IOTensors only once
    std::vector<backend::IPortableTensor *> inputs_inter;
    std::vector<backend::IPortableTensor *> outputs_inter;
    auto const input_size = executor->inputSize();
    auto const output_size = executor->outputSize();
    inputs_inter.resize(input_size);
    outputs_inter.resize(output_size);
 
    // Set inputs of executor
    // TODO Create layer to allocate/deallocate buffers of EdgeTensor for each executor
    for (uint32_t i = 0; i < input_size; i++)
    {
      const auto input_pkg_index = find_input_index(_model_edges->pkg_inputs, model_index,
                                                    ir::SubgraphIndex{0}, ir::IOIndex{i});
      const auto input_io_desc = ir::IODesc{model_index, ir::SubgraphIndex{0}, ir::IOIndex{i}};
      if (input_pkg_index != -1)
      {
        // Allocate type-aware quantization tensors for nnpkg inputs and set internal tensors
        if (_pkg_input_quant_tensors.find(input_io_desc) != _pkg_input_quant_tensors.end())
        {
          _pkg_input_quant_tensors[input_io_desc]->allocate_buffer();
 
          inputs_inter[i] = _pkg_input_quant_tensors[input_io_desc].get();
        }
        else
        {
          inputs_inter[i] = _pkg_input_tensors[input_io_desc].get();
        }
      }
      else
      {
        auto from_iodesc = find_from(model_index, ir::SubgraphIndex{0}, ir::IOIndex{i});
 
        // Supported only sequantial execution of models
        assert(std::get<ir::ModelIndex>(from_iodesc).value() < model_index.value());
        assert(std::get<ir::SubgraphIndex>(from_iodesc).value() == 0);
        const auto to_iodesc = ir::IODesc{model_index, ir::SubgraphIndex{0}, ir::IOIndex{i}};
        if (_edge_quant_tensors.find(to_iodesc) == _edge_quant_tensors.end())
        {
          inputs_inter[i] = _edge_tensors.at(from_iodesc).get();
        }
        else
        {
          inputs_inter[i] = _edge_quant_tensors.at(to_iodesc).get();
        }
        assert(inputs_inter[i]->buffer() != nullptr);
      }
    }
 
    // Set outputs of executor
    for (uint32_t i = 0; i < output_size; i++)
    {
      const auto output_pkg_index = find_output_index(_model_edges->pkg_outputs, model_index,
                                                      ir::SubgraphIndex{0}, ir::IOIndex{i});
      const auto output_io_desc = ir::IODesc{model_index, ir::SubgraphIndex{0}, ir::IOIndex{i}};
      if (output_pkg_index != -1)
      {
        // Allocate type-aware quantization tensors for nnpkg outputs and set internal tensors
        if (_pkg_output_quant_tensors.find(output_io_desc) != _pkg_output_quant_tensors.end())
        {
          _pkg_output_quant_tensors[output_io_desc]->allocate_buffer();
 
          outputs_inter[i] = _pkg_output_quant_tensors[output_io_desc].get();
        }
        else
        {
          outputs_inter[i] = _pkg_output_tensors[output_io_desc].get();
        }
      }
      else
      {
        // Allocate buffer of `from` tensors
        const auto from_iodesc = ir::IODesc{model_index, ir::SubgraphIndex{0}, ir::IOIndex{i}};
        _edge_tensors[from_iodesc]->allocate_buffer();
        outputs_inter[i] = _edge_tensors[from_iodesc].get();
 
        // Allocate buffer of tensors for type-aware quantization
        for (const auto &to_iodesc : _edge_map[from_iodesc])
        {
          _edge_tensors[from_iodesc]->increase_ref();
          if (_edge_quant_tensors.find(to_iodesc) != _edge_quant_tensors.end())
          {
            auto type_aware_quant_tensor = _edge_quant_tensors.at(to_iodesc).get();
            type_aware_quant_tensor->allocate_buffer();
 
            _edge_tensors[from_iodesc]->decrease_ref();
          }
        }
      }
    }
 
    _pkg_input_quant_layers[{model_index, ir::SubgraphIndex{0}}]->run();
 
    executor->execute(inputs_inter, outputs_inter, ctx.options);
 
    _edge_quant_layers[{model_index, ir::SubgraphIndex{0}}]->run();
    _pkg_output_quant_layers[{model_index, ir::SubgraphIndex{0}}]->run();
 
    // Release input buffers that are no longer needed
    for (uint32_t i = 0; i < input_size; i++)
    {
      const auto input_pkg_index = find_input_index(_model_edges->pkg_inputs, model_index,
                                                    ir::SubgraphIndex{0}, ir::IOIndex{i});
 
      const auto to_iodesc = ir::IODesc{model_index, ir::SubgraphIndex{0}, ir::IOIndex{i}};
      if (input_pkg_index == -1)
      {
        if (_edge_quant_tensors.find(to_iodesc) != _edge_quant_tensors.end())
        {
          // Decrease reference count of tensor for type-aware quantization if input tensor is the
          // tensor
          const auto to_iodesc = ir::IODesc{model_index, ir::SubgraphIndex{0}, ir::IOIndex{i}};
          if (_edge_quant_tensors.find(to_iodesc) != _edge_quant_tensors.end())
          {
            _edge_quant_tensors[to_iodesc]->decrease_ref();
          }
        }
        else
        {
          // Decrease reference count of `from` tensor if input tensor is the `from` tensor
          const auto from_iodesc = find_from(model_index, ir::SubgraphIndex{0}, ir::IOIndex{i});
          _edge_tensors[from_iodesc]->decrease_ref();
 
          // Decrease reference count of nnpkg inputs
          if (_pkg_input_quant_tensors.find(to_iodesc) != _pkg_input_quant_tensors.end())
          {
            _pkg_input_quant_tensors[to_iodesc]->decrease_ref();
          }
        }
      }
    }
 
    // Release output buffers if those buffers are no longer used other executors because of
    // type-aware quantization
    // FIXME if tensors for type-aware quantization unified for the same `from` tensor and same type
    for (uint32_t i = 0; i < output_size; i++)
    {
      auto from_iodesc = ir::IODesc{model_index, ir::SubgraphIndex{0}, ir::IOIndex{i}};
 
      // Check if other executors will use the buffer of edge tensor
      const auto &to_list = _edge_map[from_iodesc];
      if (to_list.size() == 0)
      {
        // This condition means `from_iodesc` tensor is an output of nnpkg
        continue;
      }
 
      bool to_be_release =
        !std::any_of(to_list.begin(), to_list.end(), [&](const ir::IODesc &to_iodesc) {
          // This condition means another executor uses the buffer of edge tensor
          return _edge_quant_tensors.find(to_iodesc) == _edge_quant_tensors.end();
        });
 
      if (to_be_release)
      {
        // This edge tensor's buffer won't be used in other executors
        // Tensors for type-aware quantization take over the role of this edge tensor instead
        _edge_tensors[from_iodesc]->decrease_ref();
      }
 
      // Decrease reference count of nnpkg outputs
      if (_pkg_output_quant_tensors.find(from_iodesc) != _pkg_output_quant_tensors.end())
      {
        _pkg_output_quant_tensors[from_iodesc]->decrease_ref();
      }
    }
  }
}

References at(), onert::exec::ExecutionContext::desc, onert::exec::IExecutor::inputSize(), onert::exec::ExecutionContext::options, and onert::util::Index< T, DummyTag >::value().

◆ inputInfo()

const ir::OperandInfo & onert::exec::MultiModelExecutors::inputInfo ( const ir::IOIndex & index ) const

overridevirtual

Return NN package input tensor info.

Parameters

[in] index Input index

Returns: Tensor info

Implements onert::exec::IExecutors.

Definition at line 72 of file MultiModelExecutors.cc.

{
  auto const [model_index, subg_index, io_index] = _model_edges->pkg_inputs[index.value()];
  auto const executor = at(model_index, subg_index);
  return executor->inputInfo(io_index.value());
}

References at(), and onert::util::Index< T, DummyTag >::value().

◆ inputSize()

uint32_t onert::exec::MultiModelExecutors::inputSize ( ) const

overridevirtual

Return executor set's number of input.

Returns: Number of input

Implements onert::exec::IExecutors.

Definition at line 68 of file MultiModelExecutors.cc.

68{ return _model_edges->pkg_inputs.size(); }

◆ outputInfo()

const ir::OperandInfo & onert::exec::MultiModelExecutors::outputInfo ( const ir::IOIndex & index ) const

overridevirtual

Return NN package output tensor info.

Parameters

[in] index Output index

Returns: Tensor info

Implements onert::exec::IExecutors.

Definition at line 79 of file MultiModelExecutors.cc.

{
  auto const [model_index, subg_index, io_index] = _model_edges->pkg_outputs[index.value()];
  auto const executor = at(model_index, subg_index);
  return executor->outputInfo(io_index.value());
}

References at(), and onert::util::Index< T, DummyTag >::value().

◆ outputSize()

uint32_t onert::exec::MultiModelExecutors::outputSize ( ) const

overridevirtual

Return executor set's number of output.

Returns: Number of output

Implements onert::exec::IExecutors.

Definition at line 70 of file MultiModelExecutors.cc.

70{ return _model_edges->pkg_outputs.size(); }

The documentation for this class was generated from the following files:

runtime/onert/core/src/exec/MultiModelExecutors.h
runtime/onert/core/src/exec/MultiModelExecutors.cc

Public Member Functions

Detailed Description

Constructor & Destructor Documentation

◆ MultiModelExecutors() [1/4]

◆ MultiModelExecutors() [2/4]

◆ MultiModelExecutors() [3/4]

◆ MultiModelExecutors() [4/4]

◆ ~MultiModelExecutors()

Member Function Documentation

◆ at()

◆ emplace()

◆ execute()

◆ inputInfo()

◆ inputSize()

◆ outputInfo()

◆ outputSize()