mlpack 3.4.2
ann_test_tools.hpp
Go to the documentation of this file.
1
12#ifndef MLPACK_TESTS_ANN_TEST_TOOLS_HPP
13#define MLPACK_TESTS_ANN_TEST_TOOLS_HPP
14
15#include <mlpack/core.hpp>
16
17using namespace mlpack;
18using namespace mlpack::ann;
19
20// Helper function which calls the Reset function of the given module.
21template<class T>
23 T& layer,
24 typename std::enable_if<HasResetCheck<T, void(T::*)()>::value>::type* = 0)
25{
26 layer.Reset();
27}
28
29template<class T>
31 T& /* layer */,
32 typename std::enable_if<!HasResetCheck<T, void(T::*)()>::value>::type* = 0)
33{
34 /* Nothing to do here */
35}
36
37// Approximate Jacobian and supposedly-true Jacobian, then compare them
38// similarly to before.
39template<typename ModuleType>
40double JacobianTest(ModuleType& module,
41 arma::mat& input,
42 const double minValue = -2,
43 const double maxValue = -1,
44 const double perturbation = 1e-6)
45{
46 arma::mat output, outputA, outputB, jacobianA, jacobianB;
47
48 // Initialize the input matrix.
49 RandomInitialization init(minValue, maxValue);
50 init.Initialize(input, input.n_rows, input.n_cols);
51
52 // Initialize the module parameters.
53 ResetFunction(module);
54
55 // Initialize the jacobian matrix.
56 module.Forward(input, output);
57 jacobianA = arma::zeros(input.n_elem, output.n_elem);
58
59 // Share the input paramter matrix.
60 arma::mat sin = arma::mat(input.memptr(), input.n_rows, input.n_cols,
61 false, false);
62
63 for (size_t i = 0; i < input.n_elem; ++i)
64 {
65 double original = sin(i);
66 sin(i) = original - perturbation;
67 module.Forward(input, outputA);
68 sin(i) = original + perturbation;
69 module.Forward(input, outputB);
70 sin(i) = original;
71
72 outputB -= outputA;
73 outputB /= 2 * perturbation;
74 jacobianA.row(i) = outputB.t();
75 }
76
77 // Initialize the derivative parameter.
78 arma::mat deriv = arma::zeros(output.n_rows, output.n_cols);
79
80 // Share the derivative parameter.
81 arma::mat derivTemp = arma::mat(deriv.memptr(), deriv.n_rows, deriv.n_cols,
82 false, false);
83
84 // Initialize the jacobian matrix.
85 jacobianB = arma::zeros(input.n_elem, output.n_elem);
86
87 for (size_t i = 0; i < derivTemp.n_elem; ++i)
88 {
89 deriv.zeros();
90 derivTemp(i) = 1;
91
92 arma::mat delta;
93 module.Backward(input, deriv, delta);
94
95 jacobianB.col(i) = delta;
96 }
97
98 return arma::max(arma::max(arma::abs(jacobianA - jacobianB)));
99}
100
101// Custom Jacobian Test where we get the input from outside of this function
102// unlike the original Jacobian Test where input is generated inside that
103// funcion.
104template <typename ModuleType>
105double CustomJacobianTest(ModuleType& module,
106 arma::mat& input,
107 const double perturbation = 1e-6)
108{
109 arma::mat output, outputA, outputB, jacobianA, jacobianB;
110
111 // Initialize the module parameters.
112 ResetFunction(module);
113
114 // Initialize the jacobian matrix.
115 module.Forward(input, output);
116 jacobianA = arma::zeros(input.n_elem, output.n_elem);
117
118 for (size_t i = 0; i < input.n_elem; ++i)
119 {
120 double original = input(i);
121 input(i) = original - perturbation;
122 module.Forward(input, outputA);
123 input(i) = original + perturbation;
124 module.Forward(input, outputB);
125 input(i) = original;
126
127 outputB -= outputA;
128 outputB /= 2 * perturbation;
129 jacobianA.row(i) = outputB.t();
130 }
131
132 // Initialize the derivative parameter.
133 arma::mat deriv = arma::zeros(output.n_rows, output.n_cols);
134
135 // Initialize the jacobian matrix.
136 jacobianB = arma::zeros(input.n_elem, output.n_elem);
137
138 for (size_t i = 0; i < deriv.n_elem; ++i)
139 {
140 deriv.zeros();
141 deriv(i) = 1;
142
143 arma::mat delta;
144 module.Backward(input, deriv, delta);
145
146 jacobianB.col(i) = delta;
147 }
148
149 return arma::max(arma::max(arma::abs(jacobianA - jacobianB)));
150}
151
152// Approximate Jacobian and supposedly-true Jacobian, then compare them
153// similarly to before.
154template<typename ModuleType>
155double JacobianPerformanceTest(ModuleType& module,
156 arma::mat& input,
157 arma::mat& target,
158 const double eps = 1e-6)
159{
160 module.Forward(input, target);
161
162 arma::mat delta;
163 module.Backward(input, target, delta);
164
165 arma::mat centralDifference = arma::zeros(delta.n_rows, delta.n_cols);
166 arma::mat inputTemp = arma::mat(input.memptr(), input.n_rows, input.n_cols,
167 false, false);
168
169 arma::mat centralDifferenceTemp = arma::mat(centralDifference.memptr(),
170 centralDifference.n_rows, centralDifference.n_cols, false, false);
171
172 for (size_t i = 0; i < input.n_elem; ++i)
173 {
174 inputTemp(i) = inputTemp(i) + eps;
175 double outputA = module.Forward(input, target);
176 inputTemp(i) = inputTemp(i) - (2 * eps);
177 double outputB = module.Forward(input, target);
178
179 centralDifferenceTemp(i) = (outputA - outputB) / (2 * eps);
180 inputTemp(i) = inputTemp(i) + eps;
181 }
182
183 return arma::max(arma::max(arma::abs(centralDifference - delta)));
184}
185
186// Simple numerical gradient checker.
187template<class FunctionType>
188double CheckGradient(FunctionType& function, const double eps = 1e-7)
189{
190 // Get gradients for the current parameters.
191 arma::mat orgGradient, gradient, estGradient;
192 function.Gradient(orgGradient);
193
194 estGradient = arma::zeros(orgGradient.n_rows, orgGradient.n_cols);
195
196 // Compute numeric approximations to gradient.
197 for (size_t i = 0; i < orgGradient.n_elem; ++i)
198 {
199 double tmp = function.Parameters()(i);
200
201 // Perturb parameter with a positive constant and get costs.
202 function.Parameters()(i) += eps;
203 double costPlus = function.Gradient(gradient);
204
205 // Perturb parameter with a negative constant and get costs.
206 function.Parameters()(i) -= (2 * eps);
207 double costMinus = function.Gradient(gradient);
208
209 // Restore the parameter value.
210 function.Parameters()(i) = tmp;
211
212 // Compute numerical gradients using the costs calculated above.
213 estGradient(i) = (costPlus - costMinus) / (2 * eps);
214 }
215
216 // Estimate error of gradient.
217 return arma::norm(orgGradient - estGradient) /
218 arma::norm(orgGradient + estGradient);
219}
220
221// Simple numerical gradient checker for regularizers.
222template<class FunctionType>
223double CheckRegularizerGradient(FunctionType& function, const double eps = 1e-7)
224{
225 // Get gradients for the current parameters.
226 arma::mat weight = arma::randu(10, 10);
227 arma::mat orgGradient = arma::zeros(10 * 10, 1);
228 function.Gradient(weight, orgGradient);
229
230 arma::mat estGradient = arma::zeros(weight.n_rows, weight.n_cols);
231
232 // Compute numeric approximations to gradient.
233 for (size_t i = 0; i < weight.n_rows; ++i)
234 {
235 for (size_t j = 0; j < weight.n_cols; ++j)
236 {
237 double tmp = weight(i, j);
238
239 weight(i, j) += eps;
240 double costPlus = function.Output(weight, i, j);
241 weight(i, j) -= (2 * eps);
242 double costMinus = function.Output(weight, i, j);
243
244 // Restore the weight value.
245 weight(i, j) = tmp;
246 estGradient(i, j) = (costPlus - costMinus) / (2 * eps);
247 }
248 }
249
250 estGradient = arma::vectorise(estGradient);
251 // Estimate error of gradient.
252 return arma::norm(orgGradient - estGradient) /
253 arma::norm(orgGradient + estGradient);
254}
255
256#endif
double CustomJacobianTest(ModuleType &module, arma::mat &input, const double perturbation=1e-6)
double JacobianPerformanceTest(ModuleType &module, arma::mat &input, arma::mat &target, const double eps=1e-6)
double JacobianTest(ModuleType &module, arma::mat &input, const double minValue=-2, const double maxValue=-1, const double perturbation=1e-6)
void ResetFunction(T &layer, typename std::enable_if< HasResetCheck< T, void(T::*)()>::value >::type *=0)
double CheckRegularizerGradient(FunctionType &function, const double eps=1e-7)
double CheckGradient(FunctionType &function, const double eps=1e-7)
This class is used to initialize randomly the weight matrix.
Definition: random_init.hpp:25
void Initialize(arma::Mat< eT > &W, const size_t rows, const size_t cols)
Initialize randomly the elements of the specified weight matrix.
Definition: random_init.hpp:56
Include all of the base components required to write mlpack methods, and the main mlpack Doxygen docu...
Artificial Neural Network.
Linear algebra utility functions, generally performed on matrices or vectors.
Definition: cv.hpp:1