performance_tests: better stats, and keep track of timing history
This commit is contained in:
moneromooo-monero
parent
4a0e4c7d70
commit
1eef056588
|
|
@ -0,0 +1,58 @@
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
template<typename T, typename Tpod = T>
|
||||||
|
class Stats
|
||||||
|
{
|
||||||
|
public:
|
||||||
|
Stats(const std::vector<T> &v): values(v), cached(0) {}
|
||||||
|
~Stats() {}
|
||||||
|
|
||||||
|
size_t get_size() const;
|
||||||
|
Tpod get_min() const;
|
||||||
|
Tpod get_max() const;
|
||||||
|
Tpod get_median() const;
|
||||||
|
double get_mean() const;
|
||||||
|
double get_confidence_interval_95() const;
|
||||||
|
double get_confidence_interval_99() const;
|
||||||
|
double get_standard_deviation() const;
|
||||||
|
double get_standard_error() const;
|
||||||
|
double get_variance() const;
|
||||||
|
double get_kurtosis() const;
|
||||||
|
double get_non_parametric_skew() const;
|
||||||
|
double get_t_test(T t) const;
|
||||||
|
double get_t_test(size_t npoints, double mean, double stddev) const;
|
||||||
|
double get_t_test(const Stats<T> &other) const;
|
||||||
|
double get_z_test(const Stats<T> &other) const;
|
||||||
|
double get_test(const Stats<T> &other) const;
|
||||||
|
std::vector<Tpod> get_quantiles(unsigned int quantiles) const;
|
||||||
|
std::vector<size_t> get_bins(unsigned int bins) const;
|
||||||
|
bool is_same_distribution_95(size_t npoints, double mean, double stddev) const;
|
||||||
|
bool is_same_distribution_95(const Stats<T> &other) const;
|
||||||
|
bool is_same_distribution_99(size_t npoints, double mean, double stddev) const;
|
||||||
|
bool is_same_distribution_99(const Stats<T> &other) const;
|
||||||
|
|
||||||
|
double get_cdf95(size_t df) const;
|
||||||
|
double get_cdf95(const Stats<T> &other) const;
|
||||||
|
double get_cdf99(size_t df) const;
|
||||||
|
double get_cdf99(const Stats<T> &other) const;
|
||||||
|
|
||||||
|
private:
|
||||||
|
inline bool is_cached(int bit) const;
|
||||||
|
inline void set_cached(int bit) const;
|
||||||
|
|
||||||
|
const std::vector<T> &values;
|
||||||
|
|
||||||
|
mutable uint64_t cached;
|
||||||
|
mutable Tpod min;
|
||||||
|
mutable Tpod max;
|
||||||
|
mutable Tpod median;
|
||||||
|
mutable double mean;
|
||||||
|
mutable double standard_deviation;
|
||||||
|
mutable double standard_error;
|
||||||
|
mutable double variance;
|
||||||
|
mutable double kurtosis;
|
||||||
|
};
|
||||||
|
|
||||||
|
#include "stats.inl"
|
||||||
|
|
@ -0,0 +1,359 @@
|
||||||
|
#include <math.h>
|
||||||
|
#include <limits>
|
||||||
|
#include <algorithm>
|
||||||
|
#include "stats.h"
|
||||||
|
|
||||||
|
enum
|
||||||
|
{
|
||||||
|
bit_min = 0,
|
||||||
|
bit_max,
|
||||||
|
bit_median,
|
||||||
|
bit_mean,
|
||||||
|
bit_standard_deviation,
|
||||||
|
bit_standard_error,
|
||||||
|
bit_variance,
|
||||||
|
bit_kurtosis,
|
||||||
|
};
|
||||||
|
|
||||||
|
static inline double square(double x)
|
||||||
|
{
|
||||||
|
return x * x;
|
||||||
|
}
|
||||||
|
|
||||||
|
template<typename T>
|
||||||
|
static inline double interpolate(T v, T v0, double i0, T v1, double i1)
|
||||||
|
{
|
||||||
|
return i0 + (i1 - i0) * (v - v0) / (v1 - v0);
|
||||||
|
}
|
||||||
|
|
||||||
|
template<typename T, typename Tpod>
|
||||||
|
inline bool Stats<T, Tpod>::is_cached(int bit) const
|
||||||
|
{
|
||||||
|
return cached & (1<<bit);
|
||||||
|
}
|
||||||
|
|
||||||
|
template<typename T, typename Tpod>
|
||||||
|
inline void Stats<T, Tpod>::set_cached(int bit) const
|
||||||
|
{
|
||||||
|
cached |= 1<<bit;
|
||||||
|
}
|
||||||
|
|
||||||
|
template<typename T, typename Tpod>
|
||||||
|
size_t Stats<T, Tpod>::get_size() const
|
||||||
|
{
|
||||||
|
return values.size();
|
||||||
|
}
|
||||||
|
|
||||||
|
template<typename T, typename Tpod>
|
||||||
|
Tpod Stats<T, Tpod>::get_min() const
|
||||||
|
{
|
||||||
|
if (!is_cached(bit_min))
|
||||||
|
{
|
||||||
|
min = std::numeric_limits<Tpod>::max();
|
||||||
|
for (const T &v: values)
|
||||||
|
min = std::min<Tpod>(min, v);
|
||||||
|
set_cached(bit_min);
|
||||||
|
}
|
||||||
|
return min;
|
||||||
|
}
|
||||||
|
|
||||||
|
template<typename T, typename Tpod>
|
||||||
|
Tpod Stats<T, Tpod>::get_max() const
|
||||||
|
{
|
||||||
|
if (!is_cached(bit_max))
|
||||||
|
{
|
||||||
|
max = std::numeric_limits<Tpod>::min();
|
||||||
|
for (const T &v: values)
|
||||||
|
max = std::max<Tpod>(max, v);
|
||||||
|
set_cached(bit_max);
|
||||||
|
}
|
||||||
|
return max;
|
||||||
|
}
|
||||||
|
|
||||||
|
template<typename T, typename Tpod>
|
||||||
|
Tpod Stats<T, Tpod>::get_median() const
|
||||||
|
{
|
||||||
|
if (!is_cached(bit_median))
|
||||||
|
{
|
||||||
|
std::vector<Tpod> sorted;
|
||||||
|
sorted.reserve(values.size());
|
||||||
|
for (const T &v: values)
|
||||||
|
sorted.push_back(v);
|
||||||
|
std::sort(sorted.begin(), sorted.end());
|
||||||
|
if (sorted.size() & 1)
|
||||||
|
{
|
||||||
|
median = sorted[sorted.size() / 2];
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
median = (sorted[(sorted.size() - 1) / 2] + sorted[sorted.size() / 2]) / 2;
|
||||||
|
}
|
||||||
|
set_cached(bit_median);
|
||||||
|
}
|
||||||
|
return median;
|
||||||
|
}
|
||||||
|
|
||||||
|
template<typename T, typename Tpod>
|
||||||
|
double Stats<T, Tpod>::get_mean() const
|
||||||
|
{
|
||||||
|
if (values.empty())
|
||||||
|
return 0.0;
|
||||||
|
if (!is_cached(bit_mean))
|
||||||
|
{
|
||||||
|
mean = 0.0;
|
||||||
|
for (const T &v: values)
|
||||||
|
mean += v;
|
||||||
|
mean /= values.size();
|
||||||
|
set_cached(bit_mean);
|
||||||
|
}
|
||||||
|
return mean;
|
||||||
|
}
|
||||||
|
|
||||||
|
template<typename T, typename Tpod>
|
||||||
|
double Stats<T, Tpod>::get_cdf95(size_t df) const
|
||||||
|
{
|
||||||
|
static const double p[101] = {
|
||||||
|
-1, 12.706, 4.3027, 3.1824, 2.7765, 2.5706, 2.4469, 2.3646, 2.3060, 2.2622, 2.2281, 2.2010, 2.1788, 2.1604, 2.1448, 2.1315,
|
||||||
|
2.1199, 2.1098, 2.1009, 2.0930, 2.0860, 2.0796, 2.0739, 2.0687, 2.0639, 2.0595, 2.0555, 2.0518, 2.0484, 2.0452, 2.0423, 2.0395,
|
||||||
|
2.0369, 2.0345, 2.0322, 2.0301, 2.0281, 2.0262, 2.0244, 2.0227, 2.0211, 2.0195, 2.0181, 2.0167, 2.0154, 2.0141, 2.0129, 2.0117,
|
||||||
|
2.0106, 2.0096, 2.0086, 2.0076, 2.0066, 2.0057, 2.0049, 2.0040, 2.0032, 2.0025, 2.0017, 2.0010, 2.0003, 1.9996, 1.9990, 1.9983,
|
||||||
|
1.9977, 1.9971, 1.9966, 1.9960, 1.9955, 1.9949, 1.9944, 1.9939, 1.9935, 1.9930, 1.9925, 1.9921, 1.9917, 1.9913, 1.9908, 1.9905,
|
||||||
|
1.9901, 1.9897, 1.9893, 1.9890, 1.9886, 1.9883, 1.9879, 1.9876, 1.9873, 1.9870, 1.9867, 1.9864, 1.9861, 1.9858, 1.9855, 1.9852,
|
||||||
|
1.9850, 1.9847, 1.9845, 1.9842, 1.9840,
|
||||||
|
};
|
||||||
|
if (df <= 100)
|
||||||
|
return p[df];
|
||||||
|
if (df <= 120)
|
||||||
|
return interpolate<size_t>(df, 100, 1.9840, 120, 1.98);
|
||||||
|
return 1.96;
|
||||||
|
}
|
||||||
|
|
||||||
|
template<typename T, typename Tpod>
|
||||||
|
double Stats<T, Tpod>::get_cdf95(const Stats<T> &other) const
|
||||||
|
{
|
||||||
|
return get_cdf95(get_size() + other.get_size() - 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
template<typename T, typename Tpod>
|
||||||
|
double Stats<T, Tpod>::get_cdf99(size_t df) const
|
||||||
|
{
|
||||||
|
static const double p[101] = {
|
||||||
|
-1, 9.9250, 5.8408, 4.6041, 4.0321, 3.7074, 3.4995, 3.3554, 3.2498, 3.1693, 3.1058, 3.0545, 3.0123, 2.9768, 2.9467, 2.9208, 2.8982,
|
||||||
|
2.8784, 2.8609, 2.8453, 2.8314, 2.8188, 2.8073, 2.7970, 2.7874, 2.7787, 2.7707, 2.7633, 2.7564, 2.7500, 2.7440, 2.7385, 2.7333,
|
||||||
|
2.7284, 2.7238, 2.7195, 2.7154, 2.7116, 2.7079, 2.7045, 2.7012, 2.6981, 2.6951, 2.6923, 2.6896, 2.6870, 2.6846, 2.6822, 2.6800,
|
||||||
|
2.6778, 2.6757, 2.6737, 2.6718, 2.6700, 2.6682, 2.6665, 2.6649, 2.6633, 2.6618, 2.6603, 2.6589, 2.6575, 2.6561, 2.6549, 2.6536,
|
||||||
|
2.6524, 2.6512, 2.6501, 2.6490, 2.6479, 2.6469, 2.6458, 2.6449, 2.6439, 2.6430, 2.6421, 2.6412, 2.6403, 2.6395, 2.6387, 2.6379,
|
||||||
|
2.6371, 2.6364, 2.6356, 2.6349, 2.6342, 2.6335, 2.6329, 2.6322, 2.6316, 2.6309, 2.6303, 2.6297, 2.6291, 2.6286, 2.6280, 2.6275,
|
||||||
|
2.6269, 2.6264, 2.6259,
|
||||||
|
};
|
||||||
|
if (df <= 100)
|
||||||
|
return p[df];
|
||||||
|
if (df <= 120)
|
||||||
|
return interpolate<size_t>(df, 100, 2.6529, 120, 2.617);
|
||||||
|
return 2.576;
|
||||||
|
}
|
||||||
|
|
||||||
|
template<typename T, typename Tpod>
|
||||||
|
double Stats<T, Tpod>::get_cdf99(const Stats<T> &other) const
|
||||||
|
{
|
||||||
|
return get_cdf99(get_size() + other.get_size() - 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
template<typename T, typename Tpod>
|
||||||
|
double Stats<T, Tpod>::get_confidence_interval_95() const
|
||||||
|
{
|
||||||
|
const size_t df = get_size() - 1;
|
||||||
|
return get_standard_error() * get_cdf95(df);
|
||||||
|
}
|
||||||
|
|
||||||
|
template<typename T, typename Tpod>
|
||||||
|
double Stats<T, Tpod>::get_confidence_interval_99() const
|
||||||
|
{
|
||||||
|
const size_t df = get_size() - 1;
|
||||||
|
return get_standard_error() * get_cdf99(df);
|
||||||
|
}
|
||||||
|
|
||||||
|
template<typename T, typename Tpod>
|
||||||
|
bool Stats<T, Tpod>::is_same_distribution_95(size_t npoints, double mean, double stddev) const
|
||||||
|
{
|
||||||
|
return fabs(get_t_test(npoints, mean, stddev)) < get_cdf95(get_size() + npoints - 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
template<typename T, typename Tpod>
|
||||||
|
bool Stats<T, Tpod>::is_same_distribution_95(const Stats<T> &other) const
|
||||||
|
{
|
||||||
|
return fabs(get_t_test(other)) < get_cdf95(other);
|
||||||
|
}
|
||||||
|
|
||||||
|
template<typename T, typename Tpod>
|
||||||
|
bool Stats<T, Tpod>::is_same_distribution_99(size_t npoints, double mean, double stddev) const
|
||||||
|
{
|
||||||
|
return fabs(get_t_test(npoints, mean, stddev)) < get_cdf99(get_size() + npoints - 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
template<typename T, typename Tpod>
|
||||||
|
bool Stats<T, Tpod>::is_same_distribution_99(const Stats<T> &other) const
|
||||||
|
{
|
||||||
|
return fabs(get_t_test(other)) < get_cdf99(other);
|
||||||
|
}
|
||||||
|
|
||||||
|
template<typename T, typename Tpod>
|
||||||
|
double Stats<T, Tpod>::get_standard_deviation() const
|
||||||
|
{
|
||||||
|
if (values.size() <= 1)
|
||||||
|
return 0.0;
|
||||||
|
if (!is_cached(bit_standard_deviation))
|
||||||
|
{
|
||||||
|
Tpod m = get_mean(), t = 0;
|
||||||
|
for (const T &v: values)
|
||||||
|
t += ((T)v - m) * ((T)v - m);
|
||||||
|
standard_deviation = sqrt(t / ((double)values.size() - 1));
|
||||||
|
set_cached(bit_standard_deviation);
|
||||||
|
}
|
||||||
|
return standard_deviation;
|
||||||
|
}
|
||||||
|
|
||||||
|
template<typename T, typename Tpod>
|
||||||
|
double Stats<T, Tpod>::get_standard_error() const
|
||||||
|
{
|
||||||
|
if (!is_cached(bit_standard_error))
|
||||||
|
{
|
||||||
|
standard_error = get_standard_deviation() / sqrt(get_size());
|
||||||
|
set_cached(bit_standard_error);
|
||||||
|
}
|
||||||
|
return standard_error;
|
||||||
|
}
|
||||||
|
|
||||||
|
template<typename T, typename Tpod>
|
||||||
|
double Stats<T, Tpod>::get_variance() const
|
||||||
|
{
|
||||||
|
if (!is_cached(bit_variance))
|
||||||
|
{
|
||||||
|
double stddev = get_standard_deviation();
|
||||||
|
variance = stddev * stddev;
|
||||||
|
set_cached(bit_variance);
|
||||||
|
}
|
||||||
|
return variance;
|
||||||
|
}
|
||||||
|
|
||||||
|
template<typename T, typename Tpod>
|
||||||
|
double Stats<T, Tpod>::get_kurtosis() const
|
||||||
|
{
|
||||||
|
if (values.empty())
|
||||||
|
return 0.0;
|
||||||
|
if (!is_cached(bit_kurtosis))
|
||||||
|
{
|
||||||
|
double m = get_mean();
|
||||||
|
double n = 0, d = 0;
|
||||||
|
for (const T &v: values)
|
||||||
|
{
|
||||||
|
T p2 = (v - m) * (v - m);
|
||||||
|
T p4 = p2 * p2;
|
||||||
|
n += p4;
|
||||||
|
d += p2;
|
||||||
|
}
|
||||||
|
n /= values.size();
|
||||||
|
d /= values.size();
|
||||||
|
d *= d;
|
||||||
|
kurtosis = n / d;
|
||||||
|
set_cached(bit_kurtosis);
|
||||||
|
}
|
||||||
|
return kurtosis;
|
||||||
|
}
|
||||||
|
|
||||||
|
template<typename T, typename Tpod>
|
||||||
|
double Stats<T, Tpod>::get_non_parametric_skew() const
|
||||||
|
{
|
||||||
|
return (get_mean() - get_median()) / get_standard_deviation();
|
||||||
|
}
|
||||||
|
|
||||||
|
template<typename T, typename Tpod>
|
||||||
|
double Stats<T, Tpod>::get_t_test(T t) const
|
||||||
|
{
|
||||||
|
const double n = get_mean() - t;
|
||||||
|
const double d = get_standard_deviation() / sqrt(get_size());
|
||||||
|
return n / d;
|
||||||
|
}
|
||||||
|
|
||||||
|
template<typename T, typename Tpod>
|
||||||
|
double Stats<T, Tpod>::get_t_test(size_t npoints, double mean, double stddev) const
|
||||||
|
{
|
||||||
|
const double n = get_mean() - mean;
|
||||||
|
const double d = sqrt(get_variance() / get_size() + square(stddev) / npoints);
|
||||||
|
return n / d;
|
||||||
|
}
|
||||||
|
|
||||||
|
template<typename T, typename Tpod>
|
||||||
|
double Stats<T, Tpod>::get_t_test(const Stats<T> &other) const
|
||||||
|
{
|
||||||
|
const double n = get_mean() - other.get_mean();
|
||||||
|
const double d = sqrt(get_variance() / get_size() + other.get_variance() / other.get_size());
|
||||||
|
return n / d;
|
||||||
|
}
|
||||||
|
|
||||||
|
template<typename T, typename Tpod>
|
||||||
|
double Stats<T, Tpod>::get_z_test(const Stats<T> &other) const
|
||||||
|
{
|
||||||
|
const double m0 = get_mean();
|
||||||
|
const double m1 = other.get_mean();
|
||||||
|
const double sd0 = get_standard_deviation();
|
||||||
|
const double sd1 = other.get_standard_deviation();
|
||||||
|
const size_t s0 = get_size();
|
||||||
|
const size_t s1 = other.get_size();
|
||||||
|
|
||||||
|
const double n = m0 - m1;
|
||||||
|
const double d = sqrt(square(sd0 / sqrt(s0)) + square(sd1 / sqrt(s1)));
|
||||||
|
|
||||||
|
return n / d;
|
||||||
|
}
|
||||||
|
|
||||||
|
template<typename T, typename Tpod>
|
||||||
|
double Stats<T, Tpod>::get_test(const Stats<T> &other) const
|
||||||
|
{
|
||||||
|
if (get_size() >= 30 && other.get_size() >= 30)
|
||||||
|
return get_z_test(other);
|
||||||
|
else
|
||||||
|
return get_t_test(other);
|
||||||
|
}
|
||||||
|
|
||||||
|
template<typename T, typename Tpod>
|
||||||
|
std::vector<Tpod> Stats<T, Tpod>::get_quantiles(unsigned int quantiles) const
|
||||||
|
{
|
||||||
|
std::vector<Tpod> sorted;
|
||||||
|
sorted.reserve(values.size());
|
||||||
|
for (const T &v: values)
|
||||||
|
sorted.push_back(v);
|
||||||
|
std::sort(sorted.begin(), sorted.end());
|
||||||
|
std::vector<Tpod> q(quantiles + 1, 0);
|
||||||
|
for (unsigned int i = 1; i <= quantiles; ++i)
|
||||||
|
{
|
||||||
|
unsigned idx = (unsigned)ceil(values.size() * i / (double)quantiles);
|
||||||
|
q[i] = sorted[idx - 1];
|
||||||
|
}
|
||||||
|
if (!is_cached(bit_min))
|
||||||
|
{
|
||||||
|
min = sorted.front();
|
||||||
|
set_cached(bit_min);
|
||||||
|
}
|
||||||
|
q[0] = min;
|
||||||
|
if (!is_cached(bit_max))
|
||||||
|
{
|
||||||
|
max = sorted.back();
|
||||||
|
set_cached(bit_max);
|
||||||
|
}
|
||||||
|
return q;
|
||||||
|
}
|
||||||
|
|
||||||
|
template<typename T, typename Tpod>
|
||||||
|
std::vector<size_t> Stats<T, Tpod>::get_bins(unsigned int bins) const
|
||||||
|
{
|
||||||
|
std::vector<size_t> b(bins, 0);
|
||||||
|
const double scale = 1.0 / (get_max() - get_min());
|
||||||
|
const T base = get_min();
|
||||||
|
for (const T &v: values)
|
||||||
|
{
|
||||||
|
unsigned int idx = (v - base) * scale;
|
||||||
|
++b[idx];
|
||||||
|
}
|
||||||
|
return b;
|
||||||
|
}
|
||||||
|
|
@ -45,6 +45,7 @@ set(common_sources
|
||||||
threadpool.cpp
|
threadpool.cpp
|
||||||
updates.cpp
|
updates.cpp
|
||||||
aligned.c
|
aligned.c
|
||||||
|
timings.cc
|
||||||
combinator.cpp)
|
combinator.cpp)
|
||||||
|
|
||||||
if (STACK_TRACE)
|
if (STACK_TRACE)
|
||||||
|
|
@ -84,6 +85,7 @@ set(common_private_headers
|
||||||
threadpool.h
|
threadpool.h
|
||||||
updates.h
|
updates.h
|
||||||
aligned.h
|
aligned.h
|
||||||
|
timings.h
|
||||||
combinator.h)
|
combinator.h)
|
||||||
|
|
||||||
monero_private_headers(common
|
monero_private_headers(common
|
||||||
|
|
|
||||||
|
|
@ -53,6 +53,7 @@ public:
|
||||||
void resume();
|
void resume();
|
||||||
void reset();
|
void reset();
|
||||||
uint64_t value() const;
|
uint64_t value() const;
|
||||||
|
operator uint64_t() const { return value(); }
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
uint64_t ticks;
|
uint64_t ticks;
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,125 @@
|
||||||
|
#include <string.h>
|
||||||
|
#include <error.h>
|
||||||
|
#include <time.h>
|
||||||
|
#include <algorithm>
|
||||||
|
#include <boost/algorithm/string.hpp>
|
||||||
|
#include "misc_log_ex.h"
|
||||||
|
#include "timings.h"
|
||||||
|
|
||||||
|
#define N_EXPECTED_FIELDS (8+11)
|
||||||
|
|
||||||
|
TimingsDatabase::TimingsDatabase()
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
TimingsDatabase::TimingsDatabase(const std::string &filename):
|
||||||
|
filename(filename)
|
||||||
|
{
|
||||||
|
load();
|
||||||
|
}
|
||||||
|
|
||||||
|
TimingsDatabase::~TimingsDatabase()
|
||||||
|
{
|
||||||
|
save();
|
||||||
|
}
|
||||||
|
|
||||||
|
bool TimingsDatabase::load()
|
||||||
|
{
|
||||||
|
instances.clear();
|
||||||
|
|
||||||
|
if (filename.empty())
|
||||||
|
return true;
|
||||||
|
|
||||||
|
FILE *f = fopen(filename.c_str(), "r");
|
||||||
|
if (!f)
|
||||||
|
{
|
||||||
|
MDEBUG("Failed to load timings file " << filename << ": " << strerror(errno));
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
while (1)
|
||||||
|
{
|
||||||
|
char s[4096];
|
||||||
|
if (!fgets(s, sizeof(s), f))
|
||||||
|
break;
|
||||||
|
char *tab = strchr(s, '\t');
|
||||||
|
if (!tab)
|
||||||
|
{
|
||||||
|
MWARNING("Bad format: no tab found");
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const std::string name = std::string(s, tab - s);
|
||||||
|
std::vector<std::string> fields;
|
||||||
|
char *ptr = tab + 1;
|
||||||
|
boost::split(fields, ptr, boost::is_any_of(" "));
|
||||||
|
if (fields.size() != N_EXPECTED_FIELDS)
|
||||||
|
{
|
||||||
|
MERROR("Bad format: wrong number of fields: got " << fields.size() << " expected " << N_EXPECTED_FIELDS);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
instance i;
|
||||||
|
|
||||||
|
unsigned int idx = 0;
|
||||||
|
i.t = atoi(fields[idx++].c_str());
|
||||||
|
i.npoints = atoi(fields[idx++].c_str());
|
||||||
|
i.min = atof(fields[idx++].c_str());
|
||||||
|
i.max = atof(fields[idx++].c_str());
|
||||||
|
i.mean = atof(fields[idx++].c_str());
|
||||||
|
i.median = atof(fields[idx++].c_str());
|
||||||
|
i.stddev = atof(fields[idx++].c_str());
|
||||||
|
i.npskew = atof(fields[idx++].c_str());
|
||||||
|
i.deciles.reserve(11);
|
||||||
|
for (int n = 0; n < 11; ++n)
|
||||||
|
{
|
||||||
|
i.deciles.push_back(atoi(fields[idx++].c_str()));
|
||||||
|
}
|
||||||
|
instances.insert(std::make_pair(name, i));
|
||||||
|
}
|
||||||
|
fclose(f);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool TimingsDatabase::save()
|
||||||
|
{
|
||||||
|
if (filename.empty())
|
||||||
|
return true;
|
||||||
|
|
||||||
|
FILE *f = fopen(filename.c_str(), "w");
|
||||||
|
if (!f)
|
||||||
|
{
|
||||||
|
MERROR("Failed to write to file " << filename << ": " << strerror(errno));
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
for (const auto &i: instances)
|
||||||
|
{
|
||||||
|
fprintf(f, "%s", i.first.c_str());
|
||||||
|
fprintf(f, "\t%lu", (unsigned long)i.second.t);
|
||||||
|
fprintf(f, " %zu", i.second.npoints);
|
||||||
|
fprintf(f, " %f", i.second.min);
|
||||||
|
fprintf(f, " %f", i.second.max);
|
||||||
|
fprintf(f, " %f", i.second.mean);
|
||||||
|
fprintf(f, " %f", i.second.median);
|
||||||
|
fprintf(f, " %f", i.second.stddev);
|
||||||
|
fprintf(f, " %f", i.second.npskew);
|
||||||
|
for (uint64_t v: i.second.deciles)
|
||||||
|
fprintf(f, " %lu", (unsigned long)v);
|
||||||
|
fputc('\n', f);
|
||||||
|
}
|
||||||
|
fclose(f);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::vector<TimingsDatabase::instance> TimingsDatabase::get(const char *name) const
|
||||||
|
{
|
||||||
|
std::vector<instance> ret;
|
||||||
|
auto range = instances.equal_range(name);
|
||||||
|
for (auto i = range.first; i != range.second; ++i)
|
||||||
|
ret.push_back(i->second);
|
||||||
|
std::sort(ret.begin(), ret.end(), [](const instance &e0, const instance &e1){ return e0.t < e1.t; });
|
||||||
|
return ret;
|
||||||
|
}
|
||||||
|
|
||||||
|
void TimingsDatabase::add(const char *name, const instance &i)
|
||||||
|
{
|
||||||
|
instances.insert(std::make_pair(name, i));
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,34 @@
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include <stdint.h>
|
||||||
|
#include <string>
|
||||||
|
#include <vector>
|
||||||
|
#include <map>
|
||||||
|
|
||||||
|
class TimingsDatabase
|
||||||
|
{
|
||||||
|
public:
|
||||||
|
struct instance
|
||||||
|
{
|
||||||
|
time_t t;
|
||||||
|
size_t npoints;
|
||||||
|
double min, max, mean, median, stddev, npskew;
|
||||||
|
std::vector<uint64_t> deciles;
|
||||||
|
};
|
||||||
|
|
||||||
|
public:
|
||||||
|
TimingsDatabase();
|
||||||
|
TimingsDatabase(const std::string &filename);
|
||||||
|
~TimingsDatabase();
|
||||||
|
|
||||||
|
std::vector<instance> get(const char *name) const;
|
||||||
|
void add(const char *name, const instance &data);
|
||||||
|
|
||||||
|
private:
|
||||||
|
bool load();
|
||||||
|
bool save();
|
||||||
|
|
||||||
|
private:
|
||||||
|
std::string filename;
|
||||||
|
std::multimap<std::string, instance> instances;
|
||||||
|
};
|
||||||
|
|
@ -77,10 +77,12 @@ int main(int argc, char** argv)
|
||||||
const command_line::arg_descriptor<bool> arg_verbose = { "verbose", "Verbose output", false };
|
const command_line::arg_descriptor<bool> arg_verbose = { "verbose", "Verbose output", false };
|
||||||
const command_line::arg_descriptor<bool> arg_stats = { "stats", "Including statistics (min/median)", false };
|
const command_line::arg_descriptor<bool> arg_stats = { "stats", "Including statistics (min/median)", false };
|
||||||
const command_line::arg_descriptor<unsigned> arg_loop_multiplier = { "loop-multiplier", "Run for that many times more loops", 1 };
|
const command_line::arg_descriptor<unsigned> arg_loop_multiplier = { "loop-multiplier", "Run for that many times more loops", 1 };
|
||||||
|
const command_line::arg_descriptor<std::string> arg_timings_database = { "timings-database", "Keep timings history in a file" };
|
||||||
command_line::add_arg(desc_options, arg_filter);
|
command_line::add_arg(desc_options, arg_filter);
|
||||||
command_line::add_arg(desc_options, arg_verbose);
|
command_line::add_arg(desc_options, arg_verbose);
|
||||||
command_line::add_arg(desc_options, arg_stats);
|
command_line::add_arg(desc_options, arg_stats);
|
||||||
command_line::add_arg(desc_options, arg_loop_multiplier);
|
command_line::add_arg(desc_options, arg_loop_multiplier);
|
||||||
|
command_line::add_arg(desc_options, arg_timings_database);
|
||||||
|
|
||||||
po::variables_map vm;
|
po::variables_map vm;
|
||||||
bool r = command_line::handle_error_helper(desc_options, [&]()
|
bool r = command_line::handle_error_helper(desc_options, [&]()
|
||||||
|
|
@ -93,7 +95,10 @@ int main(int argc, char** argv)
|
||||||
return 1;
|
return 1;
|
||||||
|
|
||||||
const std::string filter = tools::glob_to_regex(command_line::get_arg(vm, arg_filter));
|
const std::string filter = tools::glob_to_regex(command_line::get_arg(vm, arg_filter));
|
||||||
|
const std::string timings_database = command_line::get_arg(vm, arg_timings_database);
|
||||||
Params p;
|
Params p;
|
||||||
|
if (!timings_database.empty())
|
||||||
|
p.td = TimingsDatabase(timings_database);
|
||||||
p.verbose = command_line::get_arg(vm, arg_verbose);
|
p.verbose = command_line::get_arg(vm, arg_verbose);
|
||||||
p.stats = command_line::get_arg(vm, arg_stats);
|
p.stats = command_line::get_arg(vm, arg_stats);
|
||||||
p.loop_multiplier = command_line::get_arg(vm, arg_loop_multiplier);
|
p.loop_multiplier = command_line::get_arg(vm, arg_loop_multiplier);
|
||||||
|
|
|
||||||
|
|
@ -37,7 +37,9 @@
|
||||||
#include <boost/regex.hpp>
|
#include <boost/regex.hpp>
|
||||||
|
|
||||||
#include "misc_language.h"
|
#include "misc_language.h"
|
||||||
|
#include "stats.h"
|
||||||
#include "common/perf_timer.h"
|
#include "common/perf_timer.h"
|
||||||
|
#include "common/timings.h"
|
||||||
|
|
||||||
class performance_timer
|
class performance_timer
|
||||||
{
|
{
|
||||||
|
|
@ -67,6 +69,7 @@ private:
|
||||||
|
|
||||||
struct Params
|
struct Params
|
||||||
{
|
{
|
||||||
|
TimingsDatabase td;
|
||||||
bool verbose;
|
bool verbose;
|
||||||
bool stats;
|
bool stats;
|
||||||
unsigned loop_multiplier;
|
unsigned loop_multiplier;
|
||||||
|
|
@ -85,6 +88,8 @@ public:
|
||||||
|
|
||||||
bool run()
|
bool run()
|
||||||
{
|
{
|
||||||
|
static_assert(0 < T::loop_count, "T::loop_count must be greater than 0");
|
||||||
|
|
||||||
T test;
|
T test;
|
||||||
if (!test.init())
|
if (!test.init())
|
||||||
return false;
|
return false;
|
||||||
|
|
@ -106,11 +111,13 @@ public:
|
||||||
m_per_call_timers[i].pause();
|
m_per_call_timers[i].pause();
|
||||||
}
|
}
|
||||||
m_elapsed = timer.elapsed_ms();
|
m_elapsed = timer.elapsed_ms();
|
||||||
|
m_stats.reset(new Stats<tools::PerformanceTimer, uint64_t>(m_per_call_timers));
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
int elapsed_time() const { return m_elapsed; }
|
int elapsed_time() const { return m_elapsed; }
|
||||||
|
size_t get_size() const { return m_stats->get_size(); }
|
||||||
|
|
||||||
int time_per_call(int scale = 1) const
|
int time_per_call(int scale = 1) const
|
||||||
{
|
{
|
||||||
|
|
@ -118,59 +125,19 @@ public:
|
||||||
return m_elapsed * scale / (T::loop_count * m_params.loop_multiplier);
|
return m_elapsed * scale / (T::loop_count * m_params.loop_multiplier);
|
||||||
}
|
}
|
||||||
|
|
||||||
uint64_t per_call_min() const
|
uint64_t get_min() const { return m_stats->get_min(); }
|
||||||
{
|
uint64_t get_max() const { return m_stats->get_max(); }
|
||||||
uint64_t v = std::numeric_limits<uint64_t>::max();
|
double get_mean() const { return m_stats->get_mean(); }
|
||||||
for (const auto &pt: m_per_call_timers)
|
uint64_t get_median() const { return m_stats->get_median(); }
|
||||||
v = std::min(v, pt.value());
|
double get_stddev() const { return m_stats->get_standard_deviation(); }
|
||||||
return v;
|
double get_non_parametric_skew() const { return m_stats->get_non_parametric_skew(); }
|
||||||
}
|
std::vector<uint64_t> get_quantiles(size_t n) const { return m_stats->get_quantiles(n); }
|
||||||
|
|
||||||
uint64_t per_call_max() const
|
bool is_same_distribution(size_t npoints, double mean, double stddev) const
|
||||||
{
|
{
|
||||||
uint64_t v = std::numeric_limits<uint64_t>::min();
|
return m_stats->is_same_distribution_99(npoints, mean, stddev);
|
||||||
for (const auto &pt: m_per_call_timers)
|
|
||||||
v = std::max(v, pt.value());
|
|
||||||
return v;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
uint64_t per_call_mean() const
|
|
||||||
{
|
|
||||||
uint64_t v = 0;
|
|
||||||
for (const auto &pt: m_per_call_timers)
|
|
||||||
v += pt.value();
|
|
||||||
return v / m_per_call_timers.size();
|
|
||||||
}
|
|
||||||
|
|
||||||
uint64_t per_call_median() const
|
|
||||||
{
|
|
||||||
std::vector<uint64_t> values;
|
|
||||||
values.reserve(m_per_call_timers.size());
|
|
||||||
for (const auto &pt: m_per_call_timers)
|
|
||||||
values.push_back(pt.value());
|
|
||||||
return epee::misc_utils::median(values);
|
|
||||||
}
|
|
||||||
|
|
||||||
uint64_t per_call_stddev() const
|
|
||||||
{
|
|
||||||
if (m_per_call_timers.size() <= 1)
|
|
||||||
return 0;
|
|
||||||
const uint64_t mean = per_call_mean();
|
|
||||||
uint64_t acc = 0;
|
|
||||||
for (const auto &pt: m_per_call_timers)
|
|
||||||
{
|
|
||||||
int64_t dv = pt.value() - mean;
|
|
||||||
acc += dv * dv;
|
|
||||||
}
|
|
||||||
acc /= m_per_call_timers.size () - 1;
|
|
||||||
return sqrt(acc);
|
|
||||||
}
|
|
||||||
|
|
||||||
uint64_t min_time_ns() const { return tools::ticks_to_ns(per_call_min()); }
|
|
||||||
uint64_t max_time_ns() const { return tools::ticks_to_ns(per_call_max()); }
|
|
||||||
uint64_t median_time_ns() const { return tools::ticks_to_ns(per_call_median()); }
|
|
||||||
uint64_t standard_deviation_time_ns() const { return tools::ticks_to_ns(per_call_stddev()); }
|
|
||||||
|
|
||||||
private:
|
private:
|
||||||
/**
|
/**
|
||||||
* Warm up processor core, enabling turbo boost, etc.
|
* Warm up processor core, enabling turbo boost, etc.
|
||||||
|
|
@ -191,10 +158,11 @@ private:
|
||||||
int m_elapsed;
|
int m_elapsed;
|
||||||
Params m_params;
|
Params m_params;
|
||||||
std::vector<tools::PerformanceTimer> m_per_call_timers;
|
std::vector<tools::PerformanceTimer> m_per_call_timers;
|
||||||
|
std::unique_ptr<Stats<tools::PerformanceTimer, uint64_t>> m_stats;
|
||||||
};
|
};
|
||||||
|
|
||||||
template <typename T>
|
template <typename T>
|
||||||
void run_test(const std::string &filter, const Params ¶ms, const char* test_name)
|
void run_test(const std::string &filter, Params ¶ms, const char* test_name)
|
||||||
{
|
{
|
||||||
boost::smatch match;
|
boost::smatch match;
|
||||||
if (!filter.empty() && !boost::regex_match(std::string(test_name), match, boost::regex(filter)))
|
if (!filter.empty() && !boost::regex_match(std::string(test_name), match, boost::regex(filter)))
|
||||||
|
|
@ -210,10 +178,10 @@ void run_test(const std::string &filter, const Params ¶ms, const char* test_
|
||||||
std::cout << " elapsed: " << runner.elapsed_time() << " ms\n";
|
std::cout << " elapsed: " << runner.elapsed_time() << " ms\n";
|
||||||
if (params.stats)
|
if (params.stats)
|
||||||
{
|
{
|
||||||
std::cout << " min: " << runner.min_time_ns() << " ns\n";
|
std::cout << " min: " << runner.get_min() << " ns\n";
|
||||||
std::cout << " max: " << runner.max_time_ns() << " ns\n";
|
std::cout << " max: " << runner.get_max() << " ns\n";
|
||||||
std::cout << " median: " << runner.median_time_ns() << " ns\n";
|
std::cout << " median: " << runner.get_median() << " ns\n";
|
||||||
std::cout << " std dev: " << runner.standard_deviation_time_ns() << " ns\n";
|
std::cout << " std dev: " << runner.get_stddev() << " ns\n";
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
|
|
@ -221,24 +189,48 @@ void run_test(const std::string &filter, const Params ¶ms, const char* test_
|
||||||
std::cout << test_name << " (" << T::loop_count * params.loop_multiplier << " calls) - OK:";
|
std::cout << test_name << " (" << T::loop_count * params.loop_multiplier << " calls) - OK:";
|
||||||
}
|
}
|
||||||
const char *unit = "ms";
|
const char *unit = "ms";
|
||||||
uint64_t scale = 1000000;
|
double scale = 1000000;
|
||||||
int time_per_call = runner.time_per_call();
|
uint64_t time_per_call = runner.time_per_call();
|
||||||
if (time_per_call < 30000) {
|
if (time_per_call < 100) {
|
||||||
|
scale = 1000;
|
||||||
time_per_call = runner.time_per_call(1000);
|
time_per_call = runner.time_per_call(1000);
|
||||||
#ifdef _WIN32
|
#ifdef _WIN32
|
||||||
unit = "\xb5s";
|
unit = "\xb5s";
|
||||||
#else
|
#else
|
||||||
unit = "µs";
|
unit = "µs";
|
||||||
#endif
|
#endif
|
||||||
scale = 1000;
|
|
||||||
}
|
}
|
||||||
|
const auto quantiles = runner.get_quantiles(10);
|
||||||
|
double min = runner.get_min();
|
||||||
|
double max = runner.get_max();
|
||||||
|
double med = runner.get_median();
|
||||||
|
double mean = runner.get_mean();
|
||||||
|
double stddev = runner.get_stddev();
|
||||||
|
double npskew = runner.get_non_parametric_skew();
|
||||||
|
|
||||||
|
std::vector<TimingsDatabase::instance> prev_instances = params.td.get(test_name);
|
||||||
|
params.td.add(test_name, {time(NULL), runner.get_size(), min, max, mean, med, stddev, npskew, quantiles});
|
||||||
|
|
||||||
std::cout << (params.verbose ? " time per call: " : " ") << time_per_call << " " << unit << "/call" << (params.verbose ? "\n" : "");
|
std::cout << (params.verbose ? " time per call: " : " ") << time_per_call << " " << unit << "/call" << (params.verbose ? "\n" : "");
|
||||||
if (params.stats)
|
if (params.stats)
|
||||||
{
|
{
|
||||||
uint64_t min_ns = runner.min_time_ns() / scale;
|
uint64_t mins = min / scale;
|
||||||
uint64_t med_ns = runner.median_time_ns() / scale;
|
uint64_t maxs = max / scale;
|
||||||
uint64_t stddev_ns = runner.standard_deviation_time_ns() / scale;
|
uint64_t meds = med / scale;
|
||||||
std::cout << " (min " << min_ns << " " << unit << ", median " << med_ns << " " << unit << ", std dev " << stddev_ns << " " << unit << ")";
|
uint64_t p95s = quantiles[9] / scale;
|
||||||
|
uint64_t stddevs = stddev / scale;
|
||||||
|
std::string cmp;
|
||||||
|
if (!prev_instances.empty())
|
||||||
|
{
|
||||||
|
const TimingsDatabase::instance &prev_instance = prev_instances.back();
|
||||||
|
if (!runner.is_same_distribution(prev_instance.npoints, prev_instance.mean, prev_instance.stddev))
|
||||||
|
{
|
||||||
|
double pc = fabs(100. * (prev_instance.mean - runner.get_mean()) / prev_instance.mean);
|
||||||
|
cmp = ", " + std::to_string(pc) + "% " + (mean > prev_instance.mean ? "slower" : "faster");
|
||||||
|
}
|
||||||
|
cmp += " -- " + std::to_string(prev_instance.mean);
|
||||||
|
}
|
||||||
|
std::cout << " (min " << mins << " " << unit << ", 90th " << p95s << " " << unit << ", median " << meds << " " << unit << ", std dev " << stddevs << " " << unit << ")" << cmp;
|
||||||
}
|
}
|
||||||
std::cout << std::endl;
|
std::cout << std::endl;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue