Electroneum
performance_tests.h
Go to the documentation of this file.
1 // Copyrights(c) 2017-2021, The Electroneum Project
2 // Copyrights(c) 2014-2019, The Monero Project
3 //
4 // All rights reserved.
5 //
6 // Redistribution and use in source and binary forms, with or without modification, are
7 // permitted provided that the following conditions are met:
8 //
9 // 1. Redistributions of source code must retain the above copyright notice, this list of
10 // conditions and the following disclaimer.
11 //
12 // 2. Redistributions in binary form must reproduce the above copyright notice, this list
13 // of conditions and the following disclaimer in the documentation and/or other
14 // materials provided with the distribution.
15 //
16 // 3. Neither the name of the copyright holder nor the names of its contributors may be
17 // used to endorse or promote products derived from this software without specific
18 // prior written permission.
19 //
20 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
21 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
22 // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
23 // THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
25 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
27 // STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
28 // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 //
30 // Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
31 
32 #pragma once
33 
34 #include <iostream>
35 #include <stdint.h>
36 
37 #include <boost/chrono.hpp>
38 #include <boost/regex.hpp>
39 
40 #include "misc_language.h"
41 #include "stats.h"
42 #include "common/perf_timer.h"
43 #include "common/timings.h"
44 
46 {
47 public:
48  typedef boost::chrono::high_resolution_clock clock;
49 
51  {
52  m_base = clock::now();
53  }
54 
55  void start()
56  {
57  m_start = clock::now();
58  }
59 
60  int elapsed_ms()
61  {
62  clock::duration elapsed = clock::now() - m_start;
63  return static_cast<int>(boost::chrono::duration_cast<boost::chrono::milliseconds>(elapsed).count());
64  }
65 
66 private:
67  clock::time_point m_base;
68  clock::time_point m_start;
69 };
70 
71 struct Params
72 {
74  bool verbose;
75  bool stats;
76  unsigned loop_multiplier;
77 };
78 
79 template <typename T>
81 {
82 public:
83  test_runner(const Params &params)
84  : m_elapsed(0)
85  , m_params(params)
86  , m_per_call_timers(T::loop_count * params.loop_multiplier, {true})
87  {
88  }
89 
90  bool run()
91  {
92  static_assert(0 < T::loop_count, "T::loop_count must be greater than 0");
93 
94  T test;
95  if (!test.init())
96  return false;
97 
98  performance_timer timer;
99  timer.start();
100  warm_up();
101  if (m_params.verbose)
102  std::cout << "Warm up: " << timer.elapsed_ms() << " ms" << std::endl;
103 
104  timer.start();
105  for (size_t i = 0; i < T::loop_count * m_params.loop_multiplier; ++i)
106  {
107  if (m_params.stats)
108  m_per_call_timers[i].resume();
109  if (!test.test())
110  return false;
111  if (m_params.stats)
112  m_per_call_timers[i].pause();
113  }
114  m_elapsed = timer.elapsed_ms();
115  m_stats.reset(new Stats<tools::PerformanceTimer, uint64_t>(m_per_call_timers));
116 
117  return true;
118  }
119 
120  int elapsed_time() const { return m_elapsed; }
121  size_t get_size() const { return m_stats->get_size(); }
122 
123  int time_per_call(int scale = 1) const
124  {
125  static_assert(0 < T::loop_count, "T::loop_count must be greater than 0");
126  return m_elapsed * scale / (T::loop_count * m_params.loop_multiplier);
127  }
128 
129  uint64_t get_min() const { return m_stats->get_min(); }
130  uint64_t get_max() const { return m_stats->get_max(); }
131  double get_mean() const { return m_stats->get_mean(); }
132  uint64_t get_median() const { return m_stats->get_median(); }
133  double get_stddev() const { return m_stats->get_standard_deviation(); }
134  double get_non_parametric_skew() const { return m_stats->get_non_parametric_skew(); }
135  std::vector<uint64_t> get_quantiles(size_t n) const { return m_stats->get_quantiles(n); }
136 
137  bool is_same_distribution(size_t npoints, double mean, double stddev) const
138  {
139  return m_stats->is_same_distribution_99(npoints, mean, stddev);
140  }
141 
142 private:
146  uint64_t warm_up()
147  {
148  const size_t warm_up_rounds = 1000 * 1000 * 1000;
149  m_warm_up = 0;
150  for (size_t i = 0; i < warm_up_rounds; ++i)
151  {
152  ++m_warm_up;
153  }
154  return m_warm_up;
155  }
156 
157 private:
158  volatile uint64_t m_warm_up;
159  int m_elapsed;
160  Params m_params;
161  std::vector<tools::PerformanceTimer> m_per_call_timers;
162  std::unique_ptr<Stats<tools::PerformanceTimer, uint64_t>> m_stats;
163 };
164 
165 template <typename T>
166 void run_test(const std::string &filter, Params &params, const char* test_name)
167 {
168  boost::smatch match;
169  if (!filter.empty() && !boost::regex_match(std::string(test_name), match, boost::regex(filter)))
170  return;
171 
172  test_runner<T> runner(params);
173  if (runner.run())
174  {
175  if (params.verbose)
176  {
177  std::cout << test_name << " - OK:\n";
178  std::cout << " loop count: " << T::loop_count * params.loop_multiplier << '\n';
179  std::cout << " elapsed: " << runner.elapsed_time() << " ms\n";
180  if (params.stats)
181  {
182  std::cout << " min: " << runner.get_min() << " ns\n";
183  std::cout << " max: " << runner.get_max() << " ns\n";
184  std::cout << " median: " << runner.get_median() << " ns\n";
185  std::cout << " std dev: " << runner.get_stddev() << " ns\n";
186  }
187  }
188  else
189  {
190  std::cout << test_name << " (" << T::loop_count * params.loop_multiplier << " calls) - OK:";
191  }
192  const char *unit = "ms";
193  double scale = 1000000;
194  uint64_t time_per_call = runner.time_per_call();
195  if (time_per_call < 100) {
196  scale = 1000;
197  time_per_call = runner.time_per_call(1000);
198 #ifdef _WIN32
199  unit = "\xb5s";
200 #else
201  unit = "µs";
202 #endif
203  }
204  const auto quantiles = runner.get_quantiles(10);
205  double min = runner.get_min();
206  double max = runner.get_max();
207  double med = runner.get_median();
208  double mean = runner.get_mean();
209  double stddev = runner.get_stddev();
210  double npskew = runner.get_non_parametric_skew();
211 
212  std::vector<TimingsDatabase::instance> prev_instances = params.td.get(test_name);
213  params.td.add(test_name, {time(NULL), runner.get_size(), min, max, mean, med, stddev, npskew, quantiles});
214 
215  std::cout << (params.verbose ? " time per call: " : " ") << time_per_call << " " << unit << "/call" << (params.verbose ? "\n" : "");
216  if (params.stats)
217  {
218  uint64_t mins = min / scale;
219  uint64_t maxs = max / scale;
220  uint64_t meds = med / scale;
221  uint64_t p95s = quantiles[9] / scale;
222  uint64_t stddevs = stddev / scale;
223  std::string cmp;
224  if (!prev_instances.empty())
225  {
226  const TimingsDatabase::instance &prev_instance = prev_instances.back();
227  if (!runner.is_same_distribution(prev_instance.npoints, prev_instance.mean, prev_instance.stddev))
228  {
229  double pc = fabs(100. * (prev_instance.mean - runner.get_mean()) / prev_instance.mean);
230  cmp = ", " + std::to_string(pc) + "% " + (mean > prev_instance.mean ? "slower" : "faster");
231  }
232 cmp += " -- " + std::to_string(prev_instance.mean);
233  }
234  std::cout << " (min " << mins << " " << unit << ", 90th " << p95s << " " << unit << ", median " << meds << " " << unit << ", std dev " << stddevs << " " << unit << ")" << cmp;
235  }
236  std::cout << std::endl;
237  }
238  else
239  {
240  std::cout << test_name << " - FAILED" << std::endl;
241  }
242 }
243 
244 #define QUOTEME(x) #x
245 #define TEST_PERFORMANCE0(filter, params, test_class) run_test< test_class >(filter, params, QUOTEME(test_class))
246 #define TEST_PERFORMANCE1(filter, params, test_class, a0) run_test< test_class<a0> >(filter, params, QUOTEME(test_class<a0>))
247 #define TEST_PERFORMANCE2(filter, params, test_class, a0, a1) run_test< test_class<a0, a1> >(filter, params, QUOTEME(test_class) "<" QUOTEME(a0) ", " QUOTEME(a1) ">")
248 #define TEST_PERFORMANCE3(filter, params, test_class, a0, a1, a2) run_test< test_class<a0, a1, a2> >(filter, params, QUOTEME(test_class) "<" QUOTEME(a0) ", " QUOTEME(a1) ", " QUOTEME(a2) ">")
249 #define TEST_PERFORMANCE4(filter, params, test_class, a0, a1, a2, a3) run_test< test_class<a0, a1, a2, a3> >(filter, params, QUOTEME(test_class) "<" QUOTEME(a0) ", " QUOTEME(a1) ", " QUOTEME(a2) ", " QUOTEME(a3) ">")
250 #define TEST_PERFORMANCE5(filter, params, test_class, a0, a1, a2, a3, a4) run_test< test_class<a0, a1, a2, a3, a4> >(filter, params, QUOTEME(test_class) "<" QUOTEME(a0) ", " QUOTEME(a1) ", " QUOTEME(a2) ", " QUOTEME(a3) ", " QUOTEME(a4) ">")
251 #define TEST_PERFORMANCE6(filter, params, test_class, a0, a1, a2, a3, a4, a5) run_test< test_class<a0, a1, a2, a3, a4, a5> >(filter, params, QUOTEME(test_class) "<" QUOTEME(a0) ", " QUOTEME(a1) ", " QUOTEME(a2) ", " QUOTEME(a3) ", " QUOTEME(a4) ", " QUOTEME(a5) ">")
const uint32_t T[512]
double get_non_parametric_skew() const
uint64_t get_max() const
::std::string string
Definition: gtest-port.h:1097
size_t get_size() const
test_runner(const Params &params)
std::vector< instance > get(const char *name) const
int time_per_call(int scale=1) const
TimingsDatabase td
uint64_t get_min() const
mdb_size_t count(MDB_cursor *cur)
time_t time
Definition: blockchain.cpp:93
void run_test(const std::string &filter, Params &params, const char *test_name)
double get_mean() const
double get_stddev() const
std::vector< uint64_t > get_quantiles(size_t n) const
unsigned __int64 uint64_t
Definition: stdint.h:136
boost::chrono::high_resolution_clock clock
const base::type::char_t * unit
bool is_same_distribution(size_t npoints, double mean, double stddev) const
unsigned loop_multiplier
std::string to_string(t_connection_type type)
int elapsed_time() const
uint64_t get_median() const
void add(const char *name, const instance &data)