weighted_p_square_quantile.hpp 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266
  1. ///////////////////////////////////////////////////////////////////////////////
  2. // weighted_p_square_quantile.hpp
  3. //
  4. // Copyright 2005 Daniel Egloff. Distributed under the Boost
  5. // Software License, Version 1.0. (See accompanying file
  6. // LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
  7. #ifndef BOOST_ACCUMULATORS_STATISTICS_WEIGHTED_P_SQUARE_QUANTILE_HPP_DE_01_01_2006
  8. #define BOOST_ACCUMULATORS_STATISTICS_WEIGHTED_P_SQUARE_QUANTILE_HPP_DE_01_01_2006
  9. #include <cmath>
  10. #include <functional>
  11. #include <boost/array.hpp>
  12. #include <boost/parameter/keyword.hpp>
  13. #include <boost/mpl/placeholders.hpp>
  14. #include <boost/type_traits/is_same.hpp>
  15. #include <boost/accumulators/framework/accumulator_base.hpp>
  16. #include <boost/accumulators/framework/extractor.hpp>
  17. #include <boost/accumulators/numeric/functional.hpp>
  18. #include <boost/accumulators/framework/parameters/sample.hpp>
  19. #include <boost/accumulators/statistics_fwd.hpp>
  20. #include <boost/accumulators/statistics/count.hpp>
  21. #include <boost/accumulators/statistics/sum.hpp>
  22. #include <boost/accumulators/statistics/parameters/quantile_probability.hpp>
  23. namespace boost { namespace accumulators
  24. {
  25. namespace impl {
  26. ///////////////////////////////////////////////////////////////////////////////
  27. // weighted_p_square_quantile_impl
  28. // single quantile estimation with weighted samples
  29. /**
  30. @brief Single quantile estimation with the \f$P^2\f$ algorithm for weighted samples
  31. This version of the \f$P^2\f$ algorithm extends the \f$P^2\f$ algorithm to support weighted samples.
  32. The \f$P^2\f$ algorithm estimates a quantile dynamically without storing samples. Instead of
  33. storing the whole sample cumulative distribution, only five points (markers) are stored. The heights
  34. of these markers are the minimum and the maximum of the samples and the current estimates of the
  35. \f$(p/2)\f$-, \f$p\f$ - and \f$(1+p)/2\f$ -quantiles. Their positions are equal to the number
  36. of samples that are smaller or equal to the markers. Each time a new sample is added, the
  37. positions of the markers are updated and if necessary their heights are adjusted using a piecewise-
  38. parabolic formula.
  39. For further details, see
  40. R. Jain and I. Chlamtac, The P^2 algorithm for dynamic calculation of quantiles and
  41. histograms without storing observations, Communications of the ACM,
  42. Volume 28 (October), Number 10, 1985, p. 1076-1085.
  43. @param quantile_probability
  44. */
  45. template<typename Sample, typename Weight, typename Impl>
  46. struct weighted_p_square_quantile_impl
  47. : accumulator_base
  48. {
  49. typedef typename numeric::functional::multiplies<Sample, Weight>::result_type weighted_sample;
  50. typedef typename numeric::functional::fdiv<weighted_sample, std::size_t>::result_type float_type;
  51. typedef array<float_type, 5> array_type;
  52. // for boost::result_of
  53. typedef float_type result_type;
  54. template<typename Args>
  55. weighted_p_square_quantile_impl(Args const &args)
  56. : p(is_same<Impl, for_median>::value ? 0.5 : args[quantile_probability | 0.5])
  57. , heights()
  58. , actual_positions()
  59. , desired_positions()
  60. {
  61. }
  62. template<typename Args>
  63. void operator ()(Args const &args)
  64. {
  65. std::size_t cnt = count(args);
  66. // accumulate 5 first samples
  67. if (cnt <= 5)
  68. {
  69. this->heights[cnt - 1] = args[sample];
  70. // In this initialization phase, actual_positions stores the weights of the
  71. // initial samples that are needed at the end of the initialization phase to
  72. // compute the correct initial positions of the markers.
  73. this->actual_positions[cnt - 1] = args[weight];
  74. // complete the initialization of heights and actual_positions by sorting
  75. if (cnt == 5)
  76. {
  77. // TODO: we need to sort the initial samples (in heights) in ascending order and
  78. // sort their weights (in actual_positions) the same way. The following lines do
  79. // it, but there must be a better and more efficient way of doing this.
  80. typename array_type::iterator it_begin, it_end, it_min;
  81. it_begin = this->heights.begin();
  82. it_end = this->heights.end();
  83. std::size_t pos = 0;
  84. while (it_begin != it_end)
  85. {
  86. it_min = std::min_element(it_begin, it_end);
  87. std::size_t d = std::distance(it_begin, it_min);
  88. std::swap(*it_begin, *it_min);
  89. std::swap(this->actual_positions[pos], this->actual_positions[pos + d]);
  90. ++it_begin;
  91. ++pos;
  92. }
  93. // calculate correct initial actual positions
  94. for (std::size_t i = 1; i < 5; ++i)
  95. {
  96. this->actual_positions[i] += this->actual_positions[i - 1];
  97. }
  98. }
  99. }
  100. else
  101. {
  102. std::size_t sample_cell = 1; // k
  103. // find cell k such that heights[k-1] <= args[sample] < heights[k] and adjust extreme values
  104. if (args[sample] < this->heights[0])
  105. {
  106. this->heights[0] = args[sample];
  107. this->actual_positions[0] = args[weight];
  108. sample_cell = 1;
  109. }
  110. else if (this->heights[4] <= args[sample])
  111. {
  112. this->heights[4] = args[sample];
  113. sample_cell = 4;
  114. }
  115. else
  116. {
  117. typedef typename array_type::iterator iterator;
  118. iterator it = std::upper_bound(
  119. this->heights.begin()
  120. , this->heights.end()
  121. , args[sample]
  122. );
  123. sample_cell = std::distance(this->heights.begin(), it);
  124. }
  125. // increment positions of markers above sample_cell
  126. for (std::size_t i = sample_cell; i < 5; ++i)
  127. {
  128. this->actual_positions[i] += args[weight];
  129. }
  130. // update desired positions for all markers
  131. this->desired_positions[0] = this->actual_positions[0];
  132. this->desired_positions[1] = (sum_of_weights(args) - this->actual_positions[0])
  133. * this->p/2. + this->actual_positions[0];
  134. this->desired_positions[2] = (sum_of_weights(args) - this->actual_positions[0])
  135. * this->p + this->actual_positions[0];
  136. this->desired_positions[3] = (sum_of_weights(args) - this->actual_positions[0])
  137. * (1. + this->p)/2. + this->actual_positions[0];
  138. this->desired_positions[4] = sum_of_weights(args);
  139. // adjust height and actual positions of markers 1 to 3 if necessary
  140. for (std::size_t i = 1; i <= 3; ++i)
  141. {
  142. // offset to desired positions
  143. float_type d = this->desired_positions[i] - this->actual_positions[i];
  144. // offset to next position
  145. float_type dp = this->actual_positions[i + 1] - this->actual_positions[i];
  146. // offset to previous position
  147. float_type dm = this->actual_positions[i - 1] - this->actual_positions[i];
  148. // height ds
  149. float_type hp = (this->heights[i + 1] - this->heights[i]) / dp;
  150. float_type hm = (this->heights[i - 1] - this->heights[i]) / dm;
  151. if ( ( d >= 1. && dp > 1. ) || ( d <= -1. && dm < -1. ) )
  152. {
  153. short sign_d = static_cast<short>(d / std::abs(d));
  154. // try adjusting heights[i] using p-squared formula
  155. float_type h = this->heights[i] + sign_d / (dp - dm) * ( (sign_d - dm) * hp + (dp - sign_d) * hm );
  156. if ( this->heights[i - 1] < h && h < this->heights[i + 1] )
  157. {
  158. this->heights[i] = h;
  159. }
  160. else
  161. {
  162. // use linear formula
  163. if (d>0)
  164. {
  165. this->heights[i] += hp;
  166. }
  167. if (d<0)
  168. {
  169. this->heights[i] -= hm;
  170. }
  171. }
  172. this->actual_positions[i] += sign_d;
  173. }
  174. }
  175. }
  176. }
  177. result_type result(dont_care) const
  178. {
  179. return this->heights[2];
  180. }
  181. // make this accumulator serializeable
  182. // TODO split to save/load and check on parameters provided in ctor
  183. template<class Archive>
  184. void serialize(Archive & ar, const unsigned int file_version)
  185. {
  186. ar & p;
  187. ar & heights;
  188. ar & actual_positions;
  189. ar & desired_positions;
  190. }
  191. private:
  192. float_type p; // the quantile probability p
  193. array_type heights; // q_i
  194. array_type actual_positions; // n_i
  195. array_type desired_positions; // n'_i
  196. };
  197. } // namespace impl
  198. ///////////////////////////////////////////////////////////////////////////////
  199. // tag::weighted_p_square_quantile
  200. //
  201. namespace tag
  202. {
  203. struct weighted_p_square_quantile
  204. : depends_on<count, sum_of_weights>
  205. {
  206. typedef accumulators::impl::weighted_p_square_quantile_impl<mpl::_1, mpl::_2, regular> impl;
  207. };
  208. struct weighted_p_square_quantile_for_median
  209. : depends_on<count, sum_of_weights>
  210. {
  211. typedef accumulators::impl::weighted_p_square_quantile_impl<mpl::_1, mpl::_2, for_median> impl;
  212. };
  213. }
  214. ///////////////////////////////////////////////////////////////////////////////
  215. // extract::weighted_p_square_quantile
  216. // extract::weighted_p_square_quantile_for_median
  217. //
  218. namespace extract
  219. {
  220. extractor<tag::weighted_p_square_quantile> const weighted_p_square_quantile = {};
  221. extractor<tag::weighted_p_square_quantile_for_median> const weighted_p_square_quantile_for_median = {};
  222. BOOST_ACCUMULATORS_IGNORE_GLOBAL(weighted_p_square_quantile)
  223. BOOST_ACCUMULATORS_IGNORE_GLOBAL(weighted_p_square_quantile_for_median)
  224. }
  225. using extract::weighted_p_square_quantile;
  226. using extract::weighted_p_square_quantile_for_median;
  227. }} // namespace boost::accumulators
  228. #endif