onert::backend::train::optimizer::Adam Class Reference

Adam optimizer. More...

#include <Adam.h>

Collaboration diagram for onert::backend::train::optimizer::Adam:

Data Structures
struct	Property

Public Types
using	UpdateFactors = exec::train::optimizer::UpdateFactors

Public Member Functions
	Adam ()
	Adam (const Property &props)
	Adam (double lr)
	Adam (const Property &props, double lr)
std::string	name () const override
	Get the name of optimizer.
double	getLearningRate (uint32_t training_step) const override
	Get the Learning Rate.
virtual uint32_t	getVarCount () const override
	Get the number of optimizer variables s.
void	applyGradient (const UpdateFactors &factors) const override
	Apply gradient to a trainable tensor.
Public Member Functions inherited from onert::exec::train::optimizer::Optimizer
virtual	~Optimizer ()=default

Detailed Description

Adam optimizer.

Definition at line 32 of file Adam.h.

Member Typedef Documentation

UpdateFactors

using onert::backend::train::optimizer::Adam::UpdateFactors = exec::train::optimizer::UpdateFactors

Definition at line 35 of file Adam.h.

Constructor & Destructor Documentation

Adam() [1/4]

onert::backend::train::optimizer::Adam::Adam ( )

inlineexplicit

Definition at line 46 of file Adam.h.

: _props{}, _learning_rate{0.001} {}

Adam() [2/4]

onert::backend::train::optimizer::Adam::Adam ( const Property &  props )

inlineexplicit

Definition at line 47 of file Adam.h.

: _props{props}, _learning_rate{0.001} {}

Adam() [3/4]

onert::backend::train::optimizer::Adam::Adam ( double  lr )

inlineexplicit

Definition at line 48 of file Adam.h.

: _props{}, _learning_rate{lr} {}

Adam() [4/4]

onert::backend::train::optimizer::Adam::Adam ( const Property &  props, double  lr  )

inlineexplicit

Definition at line 49 of file Adam.h.

: _props{props}, _learning_rate{lr} {}

Member Function Documentation

applyGradient()

void onert::backend::train::optimizer::Adam::applyGradient ( const UpdateFactors &  factors ) const

overridevirtual

Apply gradient to a trainable tensor.

Parameters

factors

UpdateFactors to be used for applying gradient to a trainable tensor

Implements onert::exec::train::optimizer::Optimizer.

Definition at line 33 of file Adam.cc.

{
  auto [grad_tensor, trainable_tensor, training_step] = factors;
  assert(trainable_tensor.data_type() == grad_tensor.data_type());
 
  const auto opt_vars = trainable_tensor.optVars();
  assert(opt_vars.size() == 2);
  // Get the variable for exponential moving average of the gradient
  auto m_tensor = nnfw::misc::polymorphic_downcast<IPortableTensor *>(opt_vars.at(0));
  // Get the variable for exponential moving average of the squared_gradient
  auto v_tensor = nnfw::misc::polymorphic_downcast<IPortableTensor *>(opt_vars.at(1));
 
  const auto beta1_power = std::pow(_props.beta1, training_step + 1);
  const auto beta2_power = std::pow(_props.beta2, training_step + 1);
  // TODO Support nesterov
  const bool use_nesterov = false;
 
  if (trainable_tensor.getShape() != grad_tensor.getShape())
  {
    throw std::runtime_error("Adam: Invalid gradient tensor");
  }
 
  switch (grad_tensor.data_type())
  {
    case ir::DataType::FLOAT32:
      nnfw::cker::train::Adam(
        ops::getShape(&trainable_tensor), ops::getBuffer<float>(&trainable_tensor),
        ops::getShape(&grad_tensor), ops::getBuffer<float>(&grad_tensor), ops::getShape(m_tensor),
        ops::getBuffer<float>(m_tensor), ops::getShape(v_tensor), ops::getBuffer<float>(v_tensor),
        beta1_power, beta2_power, _learning_rate, _props.beta1, _props.beta2, _props.epsilon,
        use_nesterov);
      break;
    default:
      throw std::runtime_error("Adam: Not supported data type");
  }
}

References nnfw::cker::train::Adam(), onert::backend::train::optimizer::Adam::Property::beta1, onert::backend::train::optimizer::Adam::Property::beta2, and onert::backend::train::optimizer::Adam::Property::epsilon.

getLearningRate()

double onert::backend::train::optimizer::Adam::getLearningRate ( uint32_t  training_step ) const

overridevirtual

Get the Learning Rate.

Parameters

training_step

The number of training steps

Returns

Learning rate

Implements onert::exec::train::optimizer::Optimizer.

Definition at line 27 of file Adam.cc.

{
  auto biasCorrection = [&](double f) { return 1.0f - std::pow(f, training_step + 1); };
  return _learning_rate * (std::sqrt(biasCorrection(_props.beta2)) / biasCorrection(_props.beta1));
}

References onert::backend::train::optimizer::Adam::Property::beta1, and onert::backend::train::optimizer::Adam::Property::beta2.

getVarCount()

virtual uint32_t onert::backend::train::optimizer::Adam::getVarCount ( ) const

inlineoverridevirtual

Get the number of optimizer variables s.

Returns

The number of optimizer variables

Implements onert::exec::train::optimizer::Optimizer.

Definition at line 72 of file Adam.h.

{ return 2; };

name()

std::string onert::backend::train::optimizer::Adam::name ( ) const

inlineoverridevirtual

Get the name of optimizer.

Returns

The name of optimizer

Reimplemented from onert::exec::train::optimizer::Optimizer.

Definition at line 57 of file Adam.h.

{ return std::string{"Adam"}; }

---

The documentation for this class was generated from the following files:

• runtime/onert/backend/train/optimizer/Adam.h
• runtime/onert/backend/train/optimizer/Adam.cc