ONE - On-device Neural Engine
Loading...
Searching...
No Matches
onert::backend::train::optimizer::Adam Class Reference

Adam optimizer. More...

#include <Adam.h>

Collaboration diagram for onert::backend::train::optimizer::Adam:

Data Structures

struct  Property
 

Public Types

using UpdateFactors = exec::train::optimizer::UpdateFactors
 

Public Member Functions

 Adam ()
 
 Adam (const Property &props)
 
 Adam (double lr)
 
 Adam (const Property &props, double lr)
 
std::string name () const override
 Get the name of optimizer.
 
double getLearningRate (uint32_t training_step) const override
 Get the Learning Rate.
 
virtual uint32_t getVarCount () const override
 Get the number of optimizer variables s.
 
void applyGradient (const UpdateFactors &factors) const override
 Apply gradient to a trainable tensor.
 
- Public Member Functions inherited from onert::exec::train::optimizer::Optimizer
virtual ~Optimizer ()=default
 

Detailed Description

Adam optimizer.

Definition at line 38 of file Adam.h.

Member Typedef Documentation

◆ UpdateFactors

Constructor & Destructor Documentation

◆ Adam() [1/4]

onert::backend::train::optimizer::Adam::Adam ( )
inlineexplicit

Definition at line 52 of file Adam.h.

52: _props{}, _learning_rate{0.001} {}

◆ Adam() [2/4]

onert::backend::train::optimizer::Adam::Adam ( const Property props)
inlineexplicit

Definition at line 53 of file Adam.h.

53: _props{props}, _learning_rate{0.001} {}

◆ Adam() [3/4]

onert::backend::train::optimizer::Adam::Adam ( double  lr)
inlineexplicit

Definition at line 54 of file Adam.h.

54: _props{}, _learning_rate{lr} {}

◆ Adam() [4/4]

onert::backend::train::optimizer::Adam::Adam ( const Property props,
double  lr 
)
inlineexplicit

Definition at line 55 of file Adam.h.

55: _props{props}, _learning_rate{lr} {}

Member Function Documentation

◆ applyGradient()

void onert::backend::train::optimizer::Adam::applyGradient ( const UpdateFactors factors) const
overridevirtual

Apply gradient to a trainable tensor.

Parameters
factorsUpdateFactors to be used for applying gradient to a trainable tensor

Implements onert::exec::train::optimizer::Optimizer.

Definition at line 39 of file Adam.cc.

40{
41 auto [grad_tensor, trainable_tensor, training_step] = factors;
42 assert(trainable_tensor.data_type() == grad_tensor.data_type());
43
44 const auto opt_vars = trainable_tensor.optVars();
45 assert(opt_vars.size() == 2);
46 // Get the variable for exponential moving average of the gradient
47 auto m_tensor = nnfw::misc::polymorphic_downcast<IPortableTensor *>(opt_vars.at(0));
48 // Get the variable for exponential moving average of the squared_gradient
49 auto v_tensor = nnfw::misc::polymorphic_downcast<IPortableTensor *>(opt_vars.at(1));
50
51 const auto beta1_power = std::pow(_props.beta1, training_step + 1);
52 const auto beta2_power = std::pow(_props.beta2, training_step + 1);
53 // TODO Support nesterov
54 const bool use_nesterov = false;
55
56 if (trainable_tensor.getShape() != grad_tensor.getShape())
57 {
58 throw std::runtime_error("Adam: Invalid gradient tensor");
59 }
60
61 switch (grad_tensor.data_type())
62 {
63 case ir::DataType::FLOAT32:
65 ops::getShape(&trainable_tensor), ops::getBuffer<float>(&trainable_tensor),
66 ops::getShape(&grad_tensor), ops::getBuffer<float>(&grad_tensor), ops::getShape(m_tensor),
67 ops::getBuffer<float>(m_tensor), ops::getShape(v_tensor), ops::getBuffer<float>(v_tensor),
68 beta1_power, beta2_power, _learning_rate, _props.beta1, _props.beta2, _props.epsilon,
69 use_nesterov);
70 break;
71 default:
72 throw std::runtime_error("Adam: Not supported data type");
73 }
74}
void Adam(const Shape &trainable_shape, float *trainable_data, const Shape &grad_shape, const float *grad_data, const Shape &m_shape, float *m_data, const Shape &v_shape, float *v_data, float beta1_power, float beta2_power, float learning_rate, float beta1, float beta2, float epsilon, bool use_nesterov)
Definition Adam.h:33

References nnfw::cker::train::Adam(), onert::backend::train::optimizer::Adam::Property::beta1, onert::backend::train::optimizer::Adam::Property::beta2, and onert::backend::train::optimizer::Adam::Property::epsilon.

◆ getLearningRate()

double onert::backend::train::optimizer::Adam::getLearningRate ( uint32_t  training_step) const
overridevirtual

Get the Learning Rate.

Parameters
training_stepThe number of training steps
Returns
Learning rate

Implements onert::exec::train::optimizer::Optimizer.

Definition at line 33 of file Adam.cc.

34{
35 auto biasCorrection = [&](double f) { return 1.0f - std::pow(f, training_step + 1); };
36 return _learning_rate * (std::sqrt(biasCorrection(_props.beta2)) / biasCorrection(_props.beta1));
37}

References onert::backend::train::optimizer::Adam::Property::beta1, and onert::backend::train::optimizer::Adam::Property::beta2.

◆ getVarCount()

virtual uint32_t onert::backend::train::optimizer::Adam::getVarCount ( ) const
inlineoverridevirtual

Get the number of optimizer variables s.

Returns
The number of optimizer variables

Implements onert::exec::train::optimizer::Optimizer.

Definition at line 78 of file Adam.h.

78{ return 2; };

◆ name()

std::string onert::backend::train::optimizer::Adam::name ( ) const
inlineoverridevirtual

Get the name of optimizer.

Returns
The name of optimizer

Reimplemented from onert::exec::train::optimizer::Optimizer.

Definition at line 63 of file Adam.h.

63{ return std::string{"Adam"}; }

The documentation for this class was generated from the following files: