12#ifndef MLPACK_METHODS_ANN_LAYER_LAYERNORM_HPP
13#define MLPACK_METHODS_ANN_LAYER_LAYERNORM_HPP
62 typename InputDataType = arma::mat,
63 typename OutputDataType = arma::mat
77 LayerNorm(
const size_t size,
const double eps = 1e-8);
93 void Forward(
const arma::Mat<eT>& input, arma::Mat<eT>& output);
102 template<
typename eT>
104 const arma::Mat<eT>& gy,
114 template<
typename eT>
116 const arma::Mat<eT>& error,
117 arma::Mat<eT>& gradient);
130 OutputDataType
const&
Delta()
const {
return delta; }
132 OutputDataType&
Delta() {
return delta; }
135 OutputDataType
const&
Gradient()
const {
return gradient; }
140 OutputDataType
Mean() {
return mean; }
154 template<
typename Archive>
168 OutputDataType gamma;
174 OutputDataType weights;
180 OutputDataType variance;
183 OutputDataType gradient;
186 OutputDataType delta;
189 OutputDataType outputParameter;
192 OutputDataType normalized;
195 OutputDataType inputMean;
202#include "layer_norm_impl.hpp"
Declaration of the Layer Normalization class.
LayerNorm(const size_t size, const double eps=1e-8)
Create the LayerNorm object for a specified number of input units.
OutputDataType const & Delta() const
Get the delta.
OutputDataType const & Parameters() const
Get the parameters.
void Reset()
Reset the layer parameters.
void Forward(const arma::Mat< eT > &input, arma::Mat< eT > &output)
Forward pass of Layer Normalization.
OutputDataType const & OutputParameter() const
Get the output parameter.
void Backward(const arma::Mat< eT > &input, const arma::Mat< eT > &gy, arma::Mat< eT > &g)
Backward pass through the layer.
OutputDataType Variance()
Get the variance across single training data.
LayerNorm()
Create the LayerNorm object.
void Gradient(const arma::Mat< eT > &input, const arma::Mat< eT > &error, arma::Mat< eT > &gradient)
Calculate the gradient using the output delta and the input activations.
OutputDataType const & Gradient() const
Get the gradient.
OutputDataType Mean()
Get the mean across single training data.
OutputDataType & Gradient()
Modify the gradient.
OutputDataType & OutputParameter()
Modify the output parameter.
size_t InSize() const
Get the number of input units.
void serialize(Archive &ar, const unsigned int)
Serialize the layer.
OutputDataType & Parameters()
Modify the parameters.
double Epsilon() const
Get the value of epsilon.
OutputDataType & Delta()
Modify the delta.
Linear algebra utility functions, generally performed on matrices or vectors.
The core includes that mlpack expects; standard C++ includes and Armadillo.