onert::backend::trix::ops::BulkPipelineModel Class Reference

#include <BulkPipelineModel.h>

Public Types
enum class	BufferOwnership { OWNER , SHARED }

Public Member Functions
	BulkPipelineModel (const std::string &model_path, int device_id, BufferOwnership ownership=BufferOwnership::OWNER)
	~BulkPipelineModel ()
	BulkPipelineModel (const BulkPipelineModel &)=delete
BulkPipelineModel &	operator= (const BulkPipelineModel &)=delete
bool	initialize ()
bool	prepare ()
void	release ()
bool	isPrepared () const
void	run (const std::vector< const IPortableTensor * > &inputs, std::vector< IPortableTensor * > &outputs)
void	shareBuffersFrom (const BulkPipelineModel &owner)
void	setNextModel (std::shared_ptr< BulkPipelineModel > next)
std::shared_ptr< BulkPipelineModel >	getNextModel ()
void	setBufferOwnership (BufferOwnership ownership)
void	waitForBufferReady ()
void	markBufferReady ()
void	startAsyncBufferFill ()
const npubin_meta *	metadata () const
uint64_t	programSize () const
uint64_t	weightSize () const
uint32_t	modelId () const
npudev_h	device () const
const std::string &	modelPath () const
BufferOwnership	ownership () const

Detailed Description

Definition at line 35 of file BulkPipelineModel.h.

Member Enumeration Documentation

BufferOwnership

enum class onert::backend::trix::ops::BulkPipelineModel::BufferOwnership

strong

Enumerator
OWNER
SHARED

Definition at line 38 of file BulkPipelineModel.h.

  {
    OWNER,
    SHARED
  };

Constructor & Destructor Documentation

BulkPipelineModel() [1/2]

onert::backend::trix::ops::BulkPipelineModel::BulkPipelineModel	(	const std::string &	model_path,
		int	device_id,
		BufferOwnership	ownership = BufferOwnership::OWNER
	)

Definition at line 26 of file BulkPipelineModel.cc.

  : _model_path(model_path), _device_id(device_id), _ownership(ownership)
{
  // DO NOTHING
}

~BulkPipelineModel()

onert::backend::trix::ops::BulkPipelineModel::~BulkPipelineModel

(

)

Definition at line 33 of file BulkPipelineModel.cc.

{ release(); }

References release().

BulkPipelineModel() [2/2]

onert::backend::trix::ops::BulkPipelineModel::BulkPipelineModel ( const BulkPipelineModel &  )

delete

Member Function Documentation

device()

npudev_h onert::backend::trix::ops::BulkPipelineModel::device ( ) const

inline

Definition at line 74 of file BulkPipelineModel.h.

{ return _dev; }

Referenced by shareBuffersFrom().

getNextModel()

std::shared_ptr< BulkPipelineModel > onert::backend::trix::ops::BulkPipelineModel::getNextModel ( )

inline

Definition at line 63 of file BulkPipelineModel.h.

{ return _next_model; };

initialize()

bool onert::backend::trix::ops::BulkPipelineModel::initialize

(

)

Definition at line 35 of file BulkPipelineModel.cc.

{
  if (_initialized.load())
  {
    return true;
  }
 
  if (!loadMetadata())
  {
    return false;
  }
 
  _initialized = true;
  return true;
}

isPrepared()

bool onert::backend::trix::ops::BulkPipelineModel::isPrepared ( ) const

inline

Definition at line 56 of file BulkPipelineModel.h.

{ return _prepared; }

Referenced by shareBuffersFrom().

markBufferReady()

void onert::backend::trix::ops::BulkPipelineModel::markBufferReady

(

)

Definition at line 190 of file BulkPipelineModel.cc.

{
  {
    std::lock_guard<std::mutex> lock(_buffer_mutex);
    _buffer_ready = true;
  }
  _buffer_cv.notify_all();
}

Referenced by prepare(), and startAsyncBufferFill().

metadata()

const npubin_meta * onert::backend::trix::ops::BulkPipelineModel::metadata ( ) const

inline

Definition at line 70 of file BulkPipelineModel.h.

{ return _meta.get(); }

modelId()

uint32_t onert::backend::trix::ops::BulkPipelineModel::modelId ( ) const

inline

Definition at line 73 of file BulkPipelineModel.h.

{ return _model_id; }

Referenced by shareBuffersFrom().

modelPath()

const std::string & onert::backend::trix::ops::BulkPipelineModel::modelPath ( ) const

inline

Definition at line 75 of file BulkPipelineModel.h.

{ return _model_path; }

Referenced by shareBuffersFrom().

operator=()

BulkPipelineModel & onert::backend::trix::ops::BulkPipelineModel::operator= ( const BulkPipelineModel &  )

delete

ownership()

BufferOwnership onert::backend::trix::ops::BulkPipelineModel::ownership ( ) const

inline

Definition at line 76 of file BulkPipelineModel.h.

{ return _ownership; }

Referenced by setBufferOwnership().

prepare()

bool onert::backend::trix::ops::BulkPipelineModel::prepare

(

)

Definition at line 51 of file BulkPipelineModel.cc.

{
  if (_prepared.load())
  {
    return true;
  }
 
  try
  {
    if (_ownership == BufferOwnership::OWNER)
    {
      openDevice();
      allocateBuffers();
      fillBuffers();
      markBufferReady();
      registerModel();
    }
 
    _prepared = true;
    return true;
  }
  catch (const std::exception &e)
  {
    std::cerr << "Failed to prepare model " << _model_path << ": " << e.what() << std::endl;
    release();
    return false;
  }
}

References markBufferReady(), OWNER, and release().

programSize()

uint64_t onert::backend::trix::ops::BulkPipelineModel::programSize ( ) const

inline

Definition at line 71 of file BulkPipelineModel.h.

{ return _meta->program_size; }

release()

void onert::backend::trix::ops::BulkPipelineModel::release

(

)

Definition at line 80 of file BulkPipelineModel.cc.

{
  // Cancel a asynchronous job
  if (_async_fill_future.valid())
  {
    _async_fill_future.wait();
  }
 
  if (_ownership == BufferOwnership::OWNER)
  {
    unregisterModel();
    closeDevice();
  }
 
  if (_fp)
  {
    fclose(_fp);
    _fp = nullptr;
  }
 
  _program_buffer.reset();
  _weight_buffer.reset();
  _meta.reset();
  _meta_size = 0;
  _model_id = 0;
 
  _prepared = false;
}

References OWNER.

Referenced by prepare(), and ~BulkPipelineModel().

run()

void onert::backend::trix::ops::BulkPipelineModel::run	(	const std::vector< const IPortableTensor * > &	inputs,
		std::vector< IPortableTensor * > &	outputs
	)

Definition at line 109 of file BulkPipelineModel.cc.

{
  if (!_prepared.load())
  {
    throw std::runtime_error("Model is not prepared: " + _model_path);
  }
 
  if (!_meta)
  {
    throw std::runtime_error("Model metadata is not loaded: " + _model_path);
  }
 
  // Prepare input buffers
  input_buffers input;
  input.num_buffers = _meta->input_seg_num;
  for (uint32_t i = 0; i < input.num_buffers; i++)
  {
    uint32_t idx = _meta->input_seg_idx[i];
    input.bufs[i].addr = inputs[i]->buffer();
    input.bufs[i].type = BUFFER_MAPPED;
    input.bufs[i].size = _meta->segment_size[idx];
  }
 
  // Prepare output buffers
  output_buffers output;
  output.num_buffers = _meta->output_seg_num;
  for (uint32_t i = 0; i < output.num_buffers; i++)
  {
    uint32_t idx = _meta->output_seg_idx[i];
    output.bufs[i].addr = outputs[i]->buffer();
    output.bufs[i].type = BUFFER_MAPPED;
    output.bufs[i].size = _meta->segment_size[idx];
  }
 
  // Execute the model
  int ret = runNPU_model(_dev, _model_id, NPU_INFER_BLOCKING, &input, &output, nullptr, nullptr);
  if (ret < 0)
  {
    throw std::runtime_error("runNPU_model() failed for " + _model_path +
                             ", ret: " + std::to_string(ret));
  }
}

setBufferOwnership()

void onert::backend::trix::ops::BulkPipelineModel::setBufferOwnership ( BufferOwnership  ownership )

inline

Definition at line 64 of file BulkPipelineModel.h.

{ _ownership = ownership; }

References ownership().

setNextModel()

void onert::backend::trix::ops::BulkPipelineModel::setNextModel

(

std::shared_ptr< BulkPipelineModel >

next

)

Definition at line 174 of file BulkPipelineModel.cc.

{
  _next_model = next;
}

shareBuffersFrom()

void onert::backend::trix::ops::BulkPipelineModel::shareBuffersFrom

(

const BulkPipelineModel &

owner

)

Definition at line 153 of file BulkPipelineModel.cc.

{
  if (_ownership == BufferOwnership::OWNER)
  {
    throw std::runtime_error("Cannot share buffers with owner model: " + _model_path);
  }
 
  if (!owner.isPrepared())
  {
    throw std::runtime_error("Owner model is not prepared: " + owner.modelPath());
  }
 
  // Sharing the buffers
  _program_buffer = owner._program_buffer;
  _weight_buffer = owner._weight_buffer;
 
  // Sharing the device and model id
  _dev = owner.device();
  _model_id = owner.modelId();
}

References device(), isPrepared(), modelId(), modelPath(), and OWNER.

startAsyncBufferFill()

void onert::backend::trix::ops::BulkPipelineModel::startAsyncBufferFill

(

)

Definition at line 199 of file BulkPipelineModel.cc.

{
  _buffer_ready = false;
  _buffer_error = false;
  _async_fill_future = std::async(std::launch::async, [this] {
    try
    {
      fillBuffers();
      markBufferReady();
    }
    catch (const std::exception &e)
    {
      {
        std::lock_guard<std::mutex> lock(_buffer_mutex);
        _buffer_ready = false;
        _buffer_error = true;
      }
      _buffer_cv.notify_all();
      std::cerr << "Failed to fill buffers asynchronously: " << e.what() << std::endl;
    }
  });
}

References markBufferReady().

waitForBufferReady()

void onert::backend::trix::ops::BulkPipelineModel::waitForBufferReady

(

)

Definition at line 179 of file BulkPipelineModel.cc.

{
  std::unique_lock<std::mutex> lock(_buffer_mutex);
  _buffer_cv.wait(lock, [this] { return _buffer_ready.load() || _buffer_error.load(); });
 
  if (_buffer_error.load())
  {
    throw std::runtime_error("Buffer fill failed");
  }
}

weightSize()

uint64_t onert::backend::trix::ops::BulkPipelineModel::weightSize ( ) const

inline

Definition at line 72 of file BulkPipelineModel.h.

{ return _meta->weight_size; }

---

The documentation for this class was generated from the following files:

• runtime/onert/backend/trix/ops/BulkPipelineModel.h
• runtime/onert/backend/trix/ops/BulkPipelineModel.cc