onert_micro::train::optimizers::Adam Class Reference

#include <Adam.h>

Public Member Functions
	Adam ()=default
	Adam (const Adam &)=delete
	Adam (Adam &&)=delete
Adam &	operator= (const Adam &)=delete
Adam &&	operator= (const Adam &&)=delete
	~Adam ()
void	fullReset ()
void	reset ()
bool	isReset ()
uint8_t *	getExponentAvgDataByTensorIndex (uint16_t tensor_index)
uint8_t *	getExponentAvgSquaresDataByTensorIndex (uint16_t tensor_index)
void	setExponentAvgDataByTensorIndex (uint16_t tensor_index, uint8_t *data)
void	setExponentAvgSquaresDataByTensorIndex (uint16_t tensor_index, uint8_t *data)
OMStatus	handle (core::OMRuntimeStorage &backward_storage, core::OMRuntimeContext &context, core::OMRuntimeStorage &storage)
OMStatus	updateWeights (const OMTrainingContext &training_config, core::OMRuntimeContext &context, core::OMRuntimeStorage &storage, std::unordered_map< uint16_t, core::OpTrainableRankType > &)

Detailed Description

Definition at line 38 of file Adam.h.

Constructor & Destructor Documentation

Adam() [1/3]

onert_micro::train::optimizers::Adam::Adam ( )

default

Adam() [2/3]

onert_micro::train::optimizers::Adam::Adam ( const Adam &  )

delete

Adam() [3/3]

onert_micro::train::optimizers::Adam::Adam ( Adam &&  )

delete

~Adam()

onert_micro::train::optimizers::Adam::~Adam ( )

inline

Definition at line 55 of file Adam.h.

{ fullReset(); }

References fullReset().

Member Function Documentation

fullReset()

void Adam::fullReset

(

)

Definition at line 147 of file Adam.cpp.

{
  for (auto &cur_tensor_index_data : _tensor_to_exponent_avg)
  {
    uint8_t *allocated_data = cur_tensor_index_data.second;
 
    core::memory::OMMemoryManager::deallocateMemory(allocated_data);
  }
  _tensor_to_exponent_avg.clear();
 
  for (auto &cur_tensor_index_data : _tensor_to_exponent_avg_squares)
  {
    uint8_t *allocated_data = cur_tensor_index_data.second;
 
    core::memory::OMMemoryManager::deallocateMemory(allocated_data);
  }
  _tensor_to_exponent_avg_squares.clear();
 
  for (auto &cur_tensor_index_data : _tensor_index_to_gradient)
  {
    uint8_t *allocated_data = cur_tensor_index_data.second;
 
    core::memory::OMMemoryManager::deallocateMemory(allocated_data);
  }
  _tensor_index_to_gradient.clear();
}

References onert_micro::core::memory::OMMemoryManager::deallocateMemory().

Referenced by ~Adam().

getExponentAvgDataByTensorIndex()

uint8_t * Adam::getExponentAvgDataByTensorIndex

(

uint16_t

tensor_index

)

Definition at line 185 of file Adam.cpp.

{
  auto it = _tensor_to_exponent_avg.find(tensor_index);
  if (it == _tensor_to_exponent_avg.end())
    return nullptr;
 
  return it->second;
}

getExponentAvgSquaresDataByTensorIndex()

uint8_t * Adam::getExponentAvgSquaresDataByTensorIndex

(

uint16_t

tensor_index

)

Definition at line 194 of file Adam.cpp.

{
  auto it = _tensor_to_exponent_avg_squares.find(tensor_index);
  if (it == _tensor_to_exponent_avg_squares.end())
    return nullptr;
 
  return it->second;
}

handle()

OMStatus Adam::handle	(	core::OMRuntimeStorage &	backward_storage,
		core::OMRuntimeContext &	context,
		core::OMRuntimeStorage &	storage
	)

Definition at line 224 of file Adam.cpp.

{
  auto &backward_tensor_to_data = backward_storage.getTensorIndexToData();
 
  // Check is allocated or not helper buffers
  if (_tensor_to_exponent_avg_squares.empty())
  {
    // If not - let's allocate it
    assert(_tensor_to_exponent_avg.empty() == true);
    // Goes over all calculated gradients
    // Warning: assume that backward storage at this moment contains only weighs gradients -
    // This should be done due to execution plan work
    for (auto &tensor_to_data : backward_tensor_to_data)
    {
      auto tensor_index = tensor_to_data.first;
      auto tensor = context.getTensorByIndex(tensor_index);
      auto num_elements = core::OMRuntimeShape(tensor).flatSize();
 
#ifndef DIS_DYN_SHAPES
      int32_t dynamic_tensor_size = storage.getDynamicRuntimeShape(tensor_index).flatSize();
      if (dynamic_tensor_size != 0)
        num_elements = dynamic_tensor_size;
#endif // DIS_DYN_SHAPES
 
      auto tensor_size = num_elements * sizeof(core::OMDataType(tensor->type()));
 
      // Allocate data for exponent calculation
      uint8_t *exponent_data = nullptr;
      OMStatus status = core::memory::OMMemoryManager::allocateMemory(tensor_size, &exponent_data);
      assert(status == Ok);
      if (status != Ok)
        return UnknownError;
      // Set to zeros
      std::memset(exponent_data, 0, tensor_size);
      _tensor_to_exponent_avg[tensor_to_data.first] = exponent_data;
 
      // Allocate data for exponent square calculation
      uint8_t *exponent_square_data = nullptr;
      status = core::memory::OMMemoryManager::allocateMemory(tensor_size, &exponent_square_data);
      assert(status == Ok);
      if (status != Ok)
        return UnknownError;
      // Set to zeros
      std::memset(exponent_square_data, 0, tensor_size);
      _tensor_to_exponent_avg_squares[tensor_to_data.first] = exponent_square_data;
    }
  }
 
  // Check is allocated or not helper buffer
  if (_tensor_index_to_gradient.empty())
  {
    // If not - let's just move it with calculations
    // Goes over all calculated gradients
    // Warning: assume that backward storage at this moment contains only weights gradients -
    // This should be done due to execution plan work
    for (auto &tensor_to_data : backward_tensor_to_data)
    {
      // Move data
      _tensor_index_to_gradient[tensor_to_data.first] = tensor_to_data.second;
      tensor_to_data.second = nullptr;
    }
    backward_tensor_to_data.clear();
  }
  else
  {
    // Goes over all calculated gradients
    // Warning: assume that backward storage at this moment contains only weighs gradients -
    // This should be done due to execution plan work
    for (auto &tensor_to_data : backward_tensor_to_data)
    {
      auto tensor = context.getTensorByIndex(tensor_to_data.first);
      auto num_elements = core::OMRuntimeShape(tensor).flatSize();
 
#ifndef DIS_DYN_SHAPES
      int32_t dynamic_tensor_size = storage.getDynamicRuntimeShape(tensor_to_data.first).flatSize();
      if (dynamic_tensor_size != 0)
        num_elements = dynamic_tensor_size;
#endif // DIS_DYN_SHAPES
 
      auto *grad_data = reinterpret_cast<float *>(_tensor_index_to_gradient[tensor_to_data.first]);
      auto *calculated_data = reinterpret_cast<float *>(tensor_to_data.second);
 
      for (uint32_t i = 0; i < num_elements; ++i)
      {
        grad_data[i] += calculated_data[i];
      }
    }
  }
 
  return Ok;
}

References onert_micro::core::memory::OMMemoryManager::allocateMemory(), onert_micro::core::OMRuntimeShape::flatSize(), onert_micro::core::OMRuntimeStorage::getDynamicRuntimeShape(), onert_micro::core::OMRuntimeContext::getTensorByIndex(), onert_micro::core::OMRuntimeStorage::getTensorIndexToData(), onert_micro::Ok, and onert_micro::UnknownError.

isReset()

bool onert_micro::train::optimizers::Adam::isReset ( )

inline

Definition at line 73 of file Adam.h.

  {
    return _tensor_to_exponent_avg_squares.empty() or _tensor_to_exponent_avg.empty();
  }

operator=() [1/2]

Adam && onert_micro::train::optimizers::Adam::operator= ( const Adam &&  )

delete

operator=() [2/2]

Adam & onert_micro::train::optimizers::Adam::operator= ( const Adam &  )

delete

reset()

void Adam::reset

(

)

Definition at line 174 of file Adam.cpp.

{
  for (auto &cur_tensor_index_data : _tensor_index_to_gradient)
  {
    uint8_t *allocated_data = cur_tensor_index_data.second;
 
    core::memory::OMMemoryManager::deallocateMemory(allocated_data);
  }
  _tensor_index_to_gradient.clear();
}

References onert_micro::core::memory::OMMemoryManager::deallocateMemory().

setExponentAvgDataByTensorIndex()

void Adam::setExponentAvgDataByTensorIndex	(	uint16_t	tensor_index,
		uint8_t *	data
	)

Definition at line 203 of file Adam.cpp.

{
  assert(_tensor_to_exponent_avg.find(tensor_index) == _tensor_to_exponent_avg.end());
  assert(data != nullptr);
 
  _tensor_to_exponent_avg[tensor_index] = data;
}

setExponentAvgSquaresDataByTensorIndex()

void Adam::setExponentAvgSquaresDataByTensorIndex	(	uint16_t	tensor_index,
		uint8_t *	data
	)

Definition at line 211 of file Adam.cpp.

{
  assert(_tensor_to_exponent_avg_squares.find(tensor_index) ==
         _tensor_to_exponent_avg_squares.end());
  assert(data != nullptr);
 
  _tensor_to_exponent_avg_squares[tensor_index] = data;
}

updateWeights()

OMStatus Adam::updateWeights	(	const OMTrainingContext &	training_config,
		core::OMRuntimeContext &	context,
		core::OMRuntimeStorage &	storage,
		std::unordered_map< uint16_t, core::OpTrainableRankType > &	tensor_index_to_rank_type_map
	)

Definition at line 328 of file Adam.cpp.

{
  assert(!_tensor_index_to_gradient.empty());
  for (auto &tensor_to_data : _tensor_index_to_gradient)
  {
    auto exponent_squares_it = _tensor_to_exponent_avg_squares.find(tensor_to_data.first);
    if (exponent_squares_it == _tensor_to_exponent_avg_squares.end())
      return UnknownError;
 
    auto exponent_it = _tensor_to_exponent_avg.find(tensor_to_data.first);
    if (exponent_it == _tensor_to_exponent_avg.end())
      return UnknownError;
 
    auto tensor = context.getTensorByIndex(tensor_to_data.first);
    core::OMRuntimeShape shape(tensor);
 
    auto original_d = shape.dims(0);
 
    auto num_elements = core::OMRuntimeShape(tensor).flatSize();
 
#ifndef DIS_DYN_SHAPES
    int32_t dynamic_tensor_size = storage.getDynamicRuntimeShape(tensor_to_data.first).flatSize();
    if (dynamic_tensor_size != 0)
      num_elements = dynamic_tensor_size;
#endif // DIS_DYN_SHAPES
 
    auto *exponent_data = reinterpret_cast<float *>(exponent_it->second);
    auto *exponent_square_data = reinterpret_cast<float *>(exponent_squares_it->second);
    auto *calculated_data = reinterpret_cast<float *>(tensor_to_data.second);
    float beta = training_config.beta;
    float beta_squares = training_config.beta_squares;
    auto batches = static_cast<float>(training_config.batch_size);
    for (uint32_t i = 0; i < num_elements; ++i)
    {
      const auto cur_val = calculated_data[i];
      exponent_data[i] = beta * exponent_data[i] + (1 - beta) * cur_val;
      exponent_square_data[i] =
        beta_squares * exponent_square_data[i] + (1 - beta_squares) * cur_val * cur_val;
    }
 
    uint8_t *weight_data = nullptr;
    if (context.getConstDataByTensorIndex(&weight_data, tensor_to_data.first) != Ok)
      return UnknownError;
 
    assert(weight_data != nullptr);
    if (weight_data == nullptr)
      return UnknownError;
 
    auto *f_weight_data = reinterpret_cast<float *>(weight_data);
    float lambda = training_config.learning_rate;
    auto num_step = static_cast<float>(training_config.num_step);
    float beta_in_pow_batch = std::pow(beta, num_step);
    float beta_square_in_pow_batch = std::pow(beta_squares, num_step);
    float epsilon = training_config.epsilon;
 
    assert((1.f - beta_in_pow_batch) != 0);
    assert((1.f - beta_square_in_pow_batch) != 0);
    auto train_it = tensor_index_to_rank_type_map.find(tensor_to_data.first);
    core::OpTrainableRankType rank = train_it == tensor_index_to_rank_type_map.end()
                                       ? core::OpTrainableRankType::ALL
                                       : core::OpTrainableRankType(train_it->second);
    auto depth_bounds = getUpLowerWeightTensorDepth(rank, original_d);
 
    for (uint32_t i = 0; i < num_elements; ++i)
    {
      float exponent_corrected = exponent_data[i] / (1.f - beta_in_pow_batch);
      float exponent_square_corrected = exponent_square_data[i] / (1.f - beta_square_in_pow_batch);
      f_weight_data[i + depth_bounds.first] -=
        lambda * (exponent_corrected / (std::sqrt(exponent_square_corrected + epsilon)));
    }
  }
 
  return Ok;
}

References onert_micro::core::ALL, onert_micro::OMTrainingContext::batch_size, onert_micro::OMTrainingContext::beta, onert_micro::OMTrainingContext::beta_squares, onert_micro::core::OMRuntimeShape::dims(), onert_micro::OMTrainingContext::epsilon, onert_micro::core::OMRuntimeShape::flatSize(), onert_micro::core::OMRuntimeContext::getConstDataByTensorIndex(), onert_micro::core::OMRuntimeStorage::getDynamicRuntimeShape(), onert_micro::core::OMRuntimeContext::getTensorByIndex(), onert_micro::OMTrainingContext::learning_rate, onert_micro::OMTrainingContext::num_step, onert_micro::Ok, and onert_micro::UnknownError.

---

The documentation for this class was generated from the following files:

• onert-micro/onert-micro/include/train/train_optimizers/Adam.h
• onert-micro/onert-micro/src/train/train_optimizers/Adam.cpp