univariate_statistics.hpp 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466
  1. // (C) Copyright Nick Thompson 2018.
  2. // Use, modification and distribution are subject to the
  3. // Boost Software License, Version 1.0. (See accompanying file
  4. // LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
  5. #ifndef BOOST_MATH_STATISTICS_UNIVARIATE_STATISTICS_HPP
  6. #define BOOST_MATH_STATISTICS_UNIVARIATE_STATISTICS_HPP
  7. #include <algorithm>
  8. #include <iterator>
  9. #include <boost/assert.hpp>
  10. namespace boost::math::statistics {
  11. template<class ForwardIterator>
  12. auto mean(ForwardIterator first, ForwardIterator last)
  13. {
  14. using Real = typename std::iterator_traits<ForwardIterator>::value_type;
  15. BOOST_ASSERT_MSG(first != last, "At least one sample is required to compute the mean.");
  16. if constexpr (std::is_integral<Real>::value)
  17. {
  18. double mu = 0;
  19. double i = 1;
  20. for(auto it = first; it != last; ++it) {
  21. mu = mu + (*it - mu)/i;
  22. i += 1;
  23. }
  24. return mu;
  25. }
  26. else if constexpr (std::is_same_v<typename std::iterator_traits<ForwardIterator>::iterator_category, std::random_access_iterator_tag>)
  27. {
  28. size_t elements = std::distance(first, last);
  29. Real mu0 = 0;
  30. Real mu1 = 0;
  31. Real mu2 = 0;
  32. Real mu3 = 0;
  33. Real i = 1;
  34. auto end = last - (elements % 4);
  35. for(auto it = first; it != end; it += 4) {
  36. Real inv = Real(1)/i;
  37. Real tmp0 = (*it - mu0);
  38. Real tmp1 = (*(it+1) - mu1);
  39. Real tmp2 = (*(it+2) - mu2);
  40. Real tmp3 = (*(it+3) - mu3);
  41. // please generate a vectorized fma here
  42. mu0 += tmp0*inv;
  43. mu1 += tmp1*inv;
  44. mu2 += tmp2*inv;
  45. mu3 += tmp3*inv;
  46. i += 1;
  47. }
  48. Real num1 = Real(elements - (elements %4))/Real(4);
  49. Real num2 = num1 + Real(elements % 4);
  50. for (auto it = end; it != last; ++it)
  51. {
  52. mu3 += (*it-mu3)/i;
  53. i += 1;
  54. }
  55. return (num1*(mu0+mu1+mu2) + num2*mu3)/Real(elements);
  56. }
  57. else
  58. {
  59. auto it = first;
  60. Real mu = *it;
  61. Real i = 2;
  62. while(++it != last)
  63. {
  64. mu += (*it - mu)/i;
  65. i += 1;
  66. }
  67. return mu;
  68. }
  69. }
  70. template<class Container>
  71. inline auto mean(Container const & v)
  72. {
  73. return mean(v.cbegin(), v.cend());
  74. }
  75. template<class ForwardIterator>
  76. auto variance(ForwardIterator first, ForwardIterator last)
  77. {
  78. using Real = typename std::iterator_traits<ForwardIterator>::value_type;
  79. BOOST_ASSERT_MSG(first != last, "At least one sample is required to compute mean and variance.");
  80. // Higham, Accuracy and Stability, equation 1.6a and 1.6b:
  81. if constexpr (std::is_integral<Real>::value)
  82. {
  83. double M = *first;
  84. double Q = 0;
  85. double k = 2;
  86. for (auto it = std::next(first); it != last; ++it)
  87. {
  88. double tmp = *it - M;
  89. Q = Q + ((k-1)*tmp*tmp)/k;
  90. M = M + tmp/k;
  91. k += 1;
  92. }
  93. return Q/(k-1);
  94. }
  95. else
  96. {
  97. Real M = *first;
  98. Real Q = 0;
  99. Real k = 2;
  100. for (auto it = std::next(first); it != last; ++it)
  101. {
  102. Real tmp = (*it - M)/k;
  103. Q += k*(k-1)*tmp*tmp;
  104. M += tmp;
  105. k += 1;
  106. }
  107. return Q/(k-1);
  108. }
  109. }
  110. template<class Container>
  111. inline auto variance(Container const & v)
  112. {
  113. return variance(v.cbegin(), v.cend());
  114. }
  115. template<class ForwardIterator>
  116. auto sample_variance(ForwardIterator first, ForwardIterator last)
  117. {
  118. size_t n = std::distance(first, last);
  119. BOOST_ASSERT_MSG(n > 1, "At least two samples are required to compute the sample variance.");
  120. return n*variance(first, last)/(n-1);
  121. }
  122. template<class Container>
  123. inline auto sample_variance(Container const & v)
  124. {
  125. return sample_variance(v.cbegin(), v.cend());
  126. }
  127. template<class ForwardIterator>
  128. auto mean_and_sample_variance(ForwardIterator first, ForwardIterator last)
  129. {
  130. using Real = typename std::iterator_traits<ForwardIterator>::value_type;
  131. BOOST_ASSERT_MSG(first != last, "At least one sample is required to compute mean and variance.");
  132. // Higham, Accuracy and Stability, equation 1.6a and 1.6b:
  133. if constexpr (std::is_integral<Real>::value)
  134. {
  135. double M = *first;
  136. double Q = 0;
  137. double k = 2;
  138. for (auto it = std::next(first); it != last; ++it)
  139. {
  140. double tmp = *it - M;
  141. Q = Q + ((k-1)*tmp*tmp)/k;
  142. M = M + tmp/k;
  143. k += 1;
  144. }
  145. return std::pair<double, double>{M, Q/(k-2)};
  146. }
  147. else
  148. {
  149. Real M = *first;
  150. Real Q = 0;
  151. Real k = 2;
  152. for (auto it = std::next(first); it != last; ++it)
  153. {
  154. Real tmp = (*it - M)/k;
  155. Q += k*(k-1)*tmp*tmp;
  156. M += tmp;
  157. k += 1;
  158. }
  159. return std::pair<Real, Real>{M, Q/(k-2)};
  160. }
  161. }
  162. template<class Container>
  163. auto mean_and_sample_variance(Container const & v)
  164. {
  165. return mean_and_sample_variance(v.begin(), v.end());
  166. }
  167. // Follows equation 1.5 of:
  168. // https://prod.sandia.gov/techlib-noauth/access-control.cgi/2008/086212.pdf
  169. template<class ForwardIterator>
  170. auto skewness(ForwardIterator first, ForwardIterator last)
  171. {
  172. using Real = typename std::iterator_traits<ForwardIterator>::value_type;
  173. BOOST_ASSERT_MSG(first != last, "At least one sample is required to compute skewness.");
  174. if constexpr (std::is_integral<Real>::value)
  175. {
  176. double M1 = *first;
  177. double M2 = 0;
  178. double M3 = 0;
  179. double n = 2;
  180. for (auto it = std::next(first); it != last; ++it)
  181. {
  182. double delta21 = *it - M1;
  183. double tmp = delta21/n;
  184. M3 = M3 + tmp*((n-1)*(n-2)*delta21*tmp - 3*M2);
  185. M2 = M2 + tmp*(n-1)*delta21;
  186. M1 = M1 + tmp;
  187. n += 1;
  188. }
  189. double var = M2/(n-1);
  190. if (var == 0)
  191. {
  192. // The limit is technically undefined, but the interpretation here is clear:
  193. // A constant dataset has no skewness.
  194. return double(0);
  195. }
  196. double skew = M3/(M2*sqrt(var));
  197. return skew;
  198. }
  199. else
  200. {
  201. Real M1 = *first;
  202. Real M2 = 0;
  203. Real M3 = 0;
  204. Real n = 2;
  205. for (auto it = std::next(first); it != last; ++it)
  206. {
  207. Real delta21 = *it - M1;
  208. Real tmp = delta21/n;
  209. M3 += tmp*((n-1)*(n-2)*delta21*tmp - 3*M2);
  210. M2 += tmp*(n-1)*delta21;
  211. M1 += tmp;
  212. n += 1;
  213. }
  214. Real var = M2/(n-1);
  215. if (var == 0)
  216. {
  217. // The limit is technically undefined, but the interpretation here is clear:
  218. // A constant dataset has no skewness.
  219. return Real(0);
  220. }
  221. Real skew = M3/(M2*sqrt(var));
  222. return skew;
  223. }
  224. }
  225. template<class Container>
  226. inline auto skewness(Container const & v)
  227. {
  228. return skewness(v.cbegin(), v.cend());
  229. }
  230. // Follows equation 1.5/1.6 of:
  231. // https://prod.sandia.gov/techlib-noauth/access-control.cgi/2008/086212.pdf
  232. template<class ForwardIterator>
  233. auto first_four_moments(ForwardIterator first, ForwardIterator last)
  234. {
  235. using Real = typename std::iterator_traits<ForwardIterator>::value_type;
  236. BOOST_ASSERT_MSG(first != last, "At least one sample is required to compute the first four moments.");
  237. if constexpr (std::is_integral<Real>::value)
  238. {
  239. double M1 = *first;
  240. double M2 = 0;
  241. double M3 = 0;
  242. double M4 = 0;
  243. double n = 2;
  244. for (auto it = std::next(first); it != last; ++it)
  245. {
  246. double delta21 = *it - M1;
  247. double tmp = delta21/n;
  248. M4 = M4 + tmp*(tmp*tmp*delta21*((n-1)*(n*n-3*n+3)) + 6*tmp*M2 - 4*M3);
  249. M3 = M3 + tmp*((n-1)*(n-2)*delta21*tmp - 3*M2);
  250. M2 = M2 + tmp*(n-1)*delta21;
  251. M1 = M1 + tmp;
  252. n += 1;
  253. }
  254. return std::make_tuple(M1, M2/(n-1), M3/(n-1), M4/(n-1));
  255. }
  256. else
  257. {
  258. Real M1 = *first;
  259. Real M2 = 0;
  260. Real M3 = 0;
  261. Real M4 = 0;
  262. Real n = 2;
  263. for (auto it = std::next(first); it != last; ++it)
  264. {
  265. Real delta21 = *it - M1;
  266. Real tmp = delta21/n;
  267. M4 = M4 + tmp*(tmp*tmp*delta21*((n-1)*(n*n-3*n+3)) + 6*tmp*M2 - 4*M3);
  268. M3 = M3 + tmp*((n-1)*(n-2)*delta21*tmp - 3*M2);
  269. M2 = M2 + tmp*(n-1)*delta21;
  270. M1 = M1 + tmp;
  271. n += 1;
  272. }
  273. return std::make_tuple(M1, M2/(n-1), M3/(n-1), M4/(n-1));
  274. }
  275. }
  276. template<class Container>
  277. inline auto first_four_moments(Container const & v)
  278. {
  279. return first_four_moments(v.cbegin(), v.cend());
  280. }
  281. // Follows equation 1.6 of:
  282. // https://prod.sandia.gov/techlib-noauth/access-control.cgi/2008/086212.pdf
  283. template<class ForwardIterator>
  284. auto kurtosis(ForwardIterator first, ForwardIterator last)
  285. {
  286. auto [M1, M2, M3, M4] = first_four_moments(first, last);
  287. if (M2 == 0)
  288. {
  289. return M2;
  290. }
  291. return M4/(M2*M2);
  292. }
  293. template<class Container>
  294. inline auto kurtosis(Container const & v)
  295. {
  296. return kurtosis(v.cbegin(), v.cend());
  297. }
  298. template<class ForwardIterator>
  299. auto excess_kurtosis(ForwardIterator first, ForwardIterator last)
  300. {
  301. return kurtosis(first, last) - 3;
  302. }
  303. template<class Container>
  304. inline auto excess_kurtosis(Container const & v)
  305. {
  306. return excess_kurtosis(v.cbegin(), v.cend());
  307. }
  308. template<class RandomAccessIterator>
  309. auto median(RandomAccessIterator first, RandomAccessIterator last)
  310. {
  311. size_t num_elems = std::distance(first, last);
  312. BOOST_ASSERT_MSG(num_elems > 0, "The median of a zero length vector is undefined.");
  313. if (num_elems & 1)
  314. {
  315. auto middle = first + (num_elems - 1)/2;
  316. std::nth_element(first, middle, last);
  317. return *middle;
  318. }
  319. else
  320. {
  321. auto middle = first + num_elems/2 - 1;
  322. std::nth_element(first, middle, last);
  323. std::nth_element(middle, middle+1, last);
  324. return (*middle + *(middle+1))/2;
  325. }
  326. }
  327. template<class RandomAccessContainer>
  328. inline auto median(RandomAccessContainer & v)
  329. {
  330. return median(v.begin(), v.end());
  331. }
  332. template<class RandomAccessIterator>
  333. auto gini_coefficient(RandomAccessIterator first, RandomAccessIterator last)
  334. {
  335. using Real = typename std::iterator_traits<RandomAccessIterator>::value_type;
  336. BOOST_ASSERT_MSG(first != last && std::next(first) != last, "Computation of the Gini coefficient requires at least two samples.");
  337. std::sort(first, last);
  338. if constexpr (std::is_integral<Real>::value)
  339. {
  340. double i = 1;
  341. double num = 0;
  342. double denom = 0;
  343. for (auto it = first; it != last; ++it)
  344. {
  345. num += *it*i;
  346. denom += *it;
  347. ++i;
  348. }
  349. // If the l1 norm is zero, all elements are zero, so every element is the same.
  350. if (denom == 0)
  351. {
  352. return double(0);
  353. }
  354. return ((2*num)/denom - i)/(i-1);
  355. }
  356. else
  357. {
  358. Real i = 1;
  359. Real num = 0;
  360. Real denom = 0;
  361. for (auto it = first; it != last; ++it)
  362. {
  363. num += *it*i;
  364. denom += *it;
  365. ++i;
  366. }
  367. // If the l1 norm is zero, all elements are zero, so every element is the same.
  368. if (denom == 0)
  369. {
  370. return Real(0);
  371. }
  372. return ((2*num)/denom - i)/(i-1);
  373. }
  374. }
  375. template<class RandomAccessContainer>
  376. inline auto gini_coefficient(RandomAccessContainer & v)
  377. {
  378. return gini_coefficient(v.begin(), v.end());
  379. }
  380. template<class RandomAccessIterator>
  381. inline auto sample_gini_coefficient(RandomAccessIterator first, RandomAccessIterator last)
  382. {
  383. size_t n = std::distance(first, last);
  384. return n*gini_coefficient(first, last)/(n-1);
  385. }
  386. template<class RandomAccessContainer>
  387. inline auto sample_gini_coefficient(RandomAccessContainer & v)
  388. {
  389. return sample_gini_coefficient(v.begin(), v.end());
  390. }
  391. template<class RandomAccessIterator>
  392. auto median_absolute_deviation(RandomAccessIterator first, RandomAccessIterator last, typename std::iterator_traits<RandomAccessIterator>::value_type center=std::numeric_limits<typename std::iterator_traits<RandomAccessIterator>::value_type>::quiet_NaN())
  393. {
  394. using std::abs;
  395. using Real = typename std::iterator_traits<RandomAccessIterator>::value_type;
  396. using std::isnan;
  397. if (isnan(center))
  398. {
  399. center = boost::math::statistics::median(first, last);
  400. }
  401. size_t num_elems = std::distance(first, last);
  402. BOOST_ASSERT_MSG(num_elems > 0, "The median of a zero-length vector is undefined.");
  403. auto comparator = [&center](Real a, Real b) { return abs(a-center) < abs(b-center);};
  404. if (num_elems & 1)
  405. {
  406. auto middle = first + (num_elems - 1)/2;
  407. std::nth_element(first, middle, last, comparator);
  408. return abs(*middle);
  409. }
  410. else
  411. {
  412. auto middle = first + num_elems/2 - 1;
  413. std::nth_element(first, middle, last, comparator);
  414. std::nth_element(middle, middle+1, last, comparator);
  415. return (abs(*middle) + abs(*(middle+1)))/abs(static_cast<Real>(2));
  416. }
  417. }
  418. template<class RandomAccessContainer>
  419. inline auto median_absolute_deviation(RandomAccessContainer & v, typename RandomAccessContainer::value_type center=std::numeric_limits<typename RandomAccessContainer::value_type>::quiet_NaN())
  420. {
  421. return median_absolute_deviation(v.begin(), v.end(), center);
  422. }
  423. }
  424. #endif