handle_test_result.hpp 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515
  1. // (C) Copyright John Maddock 2006-7.
  2. // Use, modification and distribution are subject to the
  3. // Boost Software License, Version 1.0. (See accompanying file
  4. // LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
  5. #ifndef BOOST_MATH_HANDLE_TEST_RESULT
  6. #define BOOST_MATH_HANDLE_TEST_RESULT
  7. #include <boost/math/tools/stats.hpp>
  8. #include <boost/math/tools/precision.hpp>
  9. #include <boost/lexical_cast.hpp>
  10. #include <boost/regex.hpp>
  11. #include <boost/test/test_tools.hpp>
  12. #include <boost/filesystem.hpp>
  13. #include <boost/filesystem/fstream.hpp>
  14. #include <boost/interprocess/sync/named_mutex.hpp>
  15. #include <boost/interprocess/sync/scoped_lock.hpp>
  16. #include <boost/math/special_functions/fpclassify.hpp>
  17. #include <iostream>
  18. #include <iomanip>
  19. #include <vector>
  20. #include <set>
  21. #include <boost/math/tools/test.hpp>
  22. inline std::string sanitize_string(const std::string& s)
  23. {
  24. static const boost::regex e("[^a-zA-Z0-9]+");
  25. return boost::regex_replace(s, e, "_");
  26. }
  27. static std::string content;
  28. boost::filesystem::path path_to_content;
  29. struct content_loader
  30. {
  31. boost::interprocess::named_mutex mu;
  32. boost::interprocess::scoped_lock<boost::interprocess::named_mutex> lock;
  33. content_loader() : mu(boost::interprocess::open_or_create, "handle_test_result"), lock(mu)
  34. {
  35. boost::filesystem::path p(__FILE__);
  36. p = p.parent_path();
  37. p /= "doc";
  38. p /= "accuracy_tables.qbk";
  39. path_to_content = p;
  40. if(boost::filesystem::exists(p))
  41. {
  42. boost::filesystem::ifstream is(p);
  43. if(is.good())
  44. {
  45. do
  46. {
  47. char c = static_cast<char>(is.get());
  48. if(c != EOF)
  49. content.append(1, c);
  50. } while(is.good());
  51. }
  52. }
  53. }
  54. ~content_loader()
  55. {
  56. boost::filesystem::ofstream os(path_to_content);
  57. os << content;
  58. }
  59. void instantiate()const
  60. {
  61. }
  62. };
  63. static const content_loader loader;
  64. void load_table(std::vector<std::vector<std::string> >& table, std::string::const_iterator begin, std::string::const_iterator end)
  65. {
  66. static const boost::regex item_e(
  67. "\\["
  68. "([^\\[\\]]*(?0)?)*"
  69. "\\]"
  70. );
  71. boost::regex_token_iterator<std::string::const_iterator> i(begin, end, item_e), j;
  72. while(i != j)
  73. {
  74. // Add a row:
  75. table.push_back(std::vector<std::string>());
  76. boost::regex_token_iterator<std::string::const_iterator> k(i->first + 1, i->second - 1, item_e);
  77. while(k != j)
  78. {
  79. // Add a cell:
  80. table.back().push_back(std::string(k->first + 1, k->second - 1));
  81. ++k;
  82. }
  83. ++i;
  84. }
  85. }
  86. std::string save_table(std::vector<std::vector<std::string> >& table)
  87. {
  88. std::string result;
  89. for(std::vector<std::vector<std::string> >::const_iterator i = table.begin(), j = table.end(); i != j; ++i)
  90. {
  91. result += "[";
  92. for(std::vector<std::string>::const_iterator k = i->begin(), l = i->end(); k != l; ++k)
  93. {
  94. result += "[";
  95. result += *k;
  96. result += "]";
  97. }
  98. result += "]\n";
  99. }
  100. return result;
  101. }
  102. void add_to_all_sections(const std::string& id, std::string list_name = "all_sections")
  103. {
  104. std::string::size_type pos = content.find("[template " + list_name + "[]"), end_pos;
  105. if(pos == std::string::npos)
  106. {
  107. //
  108. // Just append to the end:
  109. //
  110. content.append("\n[template ").append(list_name).append("[]\n[").append(id).append("]\n]\n");
  111. }
  112. else
  113. {
  114. //
  115. // Read in the all list of sections, add our new one (in alphabetical order),
  116. // and then rewrite the whole thing:
  117. //
  118. static const boost::regex item_e(
  119. "\\["
  120. "([^\\[\\]]*(?0)?)*"
  121. "\\]|\\]"
  122. );
  123. boost::regex_token_iterator<std::string::const_iterator> i(content.begin() + pos + 12 + list_name.size(), content.end(), item_e), j;
  124. std::set<std::string> sections;
  125. while(i != j)
  126. {
  127. if(i->length() == 1)
  128. {
  129. end_pos = i->first - content.begin();
  130. break;
  131. }
  132. sections.insert(std::string(i->first + 1, i->second - 1));
  133. ++i;
  134. }
  135. sections.insert(id);
  136. std::string new_list = "\n";
  137. for(std::set<std::string>::const_iterator sec = sections.begin(); sec != sections.end(); ++sec)
  138. {
  139. new_list += "[" + *sec + "]\n";
  140. }
  141. content.replace(pos + 12 + list_name.size(), end_pos - pos - 12 - list_name.size(), new_list);
  142. }
  143. }
  144. void add_cell(const std::string& cell_name, const std::string& table_name, const std::string& row_name, const std::string& type_name)
  145. {
  146. //
  147. // Load the table, add our data, and re-write:
  148. //
  149. std::string table_id = "table_" + sanitize_string(table_name);
  150. std::string column_heading = BOOST_COMPILER;
  151. column_heading += "[br]";
  152. column_heading += BOOST_PLATFORM;
  153. column_heading += "[br]";
  154. column_heading += type_name;
  155. boost::regex table_e("\\[table:" + table_id
  156. + "\\s[^\\[]+"
  157. "((\\["
  158. "([^\\[\\]]*(?2)?)*"
  159. "\\]\\s*)*\\s*)"
  160. "\\]"
  161. );
  162. boost::smatch table_location;
  163. if(regex_search(content, table_location, table_e))
  164. {
  165. std::vector<std::vector<std::string> > table_data;
  166. load_table(table_data, table_location[1].first, table_location[1].second);
  167. //
  168. // Figure out which column we're on:
  169. //
  170. unsigned column_id = 1001u;
  171. for(unsigned i = 0; i < table_data[0].size(); ++i)
  172. {
  173. if(table_data[0][i] == column_heading)
  174. {
  175. column_id = i;
  176. break;
  177. }
  178. }
  179. if(column_id > 1000)
  180. {
  181. //
  182. // Need a new column, must be adding a new compiler to the table!
  183. //
  184. table_data[0].push_back(column_heading);
  185. for(unsigned i = 1; i < table_data.size(); ++i)
  186. table_data[i].push_back(std::string());
  187. column_id = table_data[0].size() - 1;
  188. }
  189. //
  190. // Figure out the row:
  191. //
  192. unsigned row_id = 1001;
  193. for(unsigned i = 1; i < table_data.size(); ++i)
  194. {
  195. if(table_data[i][0] == row_name)
  196. {
  197. row_id = i;
  198. break;
  199. }
  200. }
  201. if(row_id > 1000)
  202. {
  203. //
  204. // Need a new row, add it now:
  205. //
  206. table_data.push_back(std::vector<std::string>());
  207. table_data.back().push_back(row_name);
  208. for(unsigned i = 1; i < table_data[0].size(); ++i)
  209. table_data.back().push_back(std::string());
  210. row_id = table_data.size() - 1;
  211. }
  212. //
  213. // Update the entry:
  214. //
  215. std::string& s = table_data[row_id][column_id];
  216. if(s.empty())
  217. {
  218. std::cout << "Adding " << cell_name << " to empty cell.";
  219. s = "[" + cell_name + "]";
  220. }
  221. else
  222. {
  223. if(cell_name.find("_boost_") != std::string::npos)
  224. {
  225. std::cout << "Adding " << cell_name << " to start of cell.";
  226. s.insert(0, "[" + cell_name + "][br][br]");
  227. }
  228. else
  229. {
  230. std::cout << "Adding " << cell_name << " to end of cell.";
  231. if((s.find("_boost_") != std::string::npos) && (s.find("[br]") == std::string::npos))
  232. s += "[br]"; // extra break if we're adding directly after the boost results.
  233. s += "[br][" + cell_name + "]";
  234. }
  235. }
  236. //
  237. // Convert back to a string and insert into content:
  238. std::string c = save_table(table_data);
  239. content.replace(table_location.position(1), table_location.length(1), c);
  240. }
  241. else
  242. {
  243. //
  244. // Create a new table and try again:
  245. //
  246. std::string new_table = "\n[template " + table_id;
  247. new_table += "[]\n[table:" + table_id;
  248. new_table += " Error rates for ";
  249. new_table += table_name;
  250. new_table += "\n[[][";
  251. new_table += column_heading;
  252. new_table += "]]\n";
  253. new_table += "[[";
  254. new_table += row_name;
  255. new_table += "][[";
  256. new_table += cell_name;
  257. new_table += "]]]\n]\n]\n";
  258. std::string::size_type pos = content.find("[/tables:]");
  259. if(pos != std::string::npos)
  260. content.insert(pos + 10, new_table);
  261. else
  262. content += "\n\n[/tables:]\n" + new_table;
  263. //
  264. // Add a section for this table as well:
  265. //
  266. std::string section_id = "section_" + sanitize_string(table_name);
  267. if(content.find(section_id + "[]") == std::string::npos)
  268. {
  269. std::string new_section = "\n[template " + section_id + "[]\n[section:" + section_id + " " + table_name + "]\n[" + table_id + "]\n[endsect]\n]\n";
  270. pos = content.find("[/sections:]");
  271. if(pos != std::string::npos)
  272. content.insert(pos + 12, new_section);
  273. else
  274. content += "\n\n[/sections:]\n" + new_section;
  275. add_to_all_sections(section_id);
  276. }
  277. //
  278. // Add to list of all tables (not in sections):
  279. //
  280. add_to_all_sections(table_id, "all_tables");
  281. }
  282. }
  283. void set_result(const std::string& cell_name, const std::string& cell_content, const std::string& table_name, const std::string& row_name, const std::string& type_name)
  284. {
  285. loader.instantiate();
  286. const boost::regex e("\\[template\\s+" + cell_name +
  287. "\\[\\]([^\\n]*)\\]$");
  288. boost::smatch what;
  289. if(regex_search(content, what, e))
  290. {
  291. content.replace(what.position(1), what.length(1), cell_content);
  292. }
  293. else
  294. {
  295. // Need to add new content:
  296. std::string::size_type pos = content.find("[/Cell Content:]");
  297. std::string t = "\n[template " + cell_name + "[] " + cell_content + "]";
  298. if(pos != std::string::npos)
  299. content.insert(pos + 16, t);
  300. else
  301. {
  302. content.insert(0, t);
  303. content.insert(0, "[/Cell Content:]");
  304. }
  305. }
  306. //
  307. // Check to verify that our content is actually used somewhere,
  308. // if not we need to create a place for it:
  309. //
  310. if(content.find("[" + cell_name + "]") == std::string::npos)
  311. add_cell(cell_name, table_name, row_name, type_name);
  312. }
  313. void set_error_content(const std::string& id, const std::string& error_s)
  314. {
  315. boost::regex content_e("\\[template\\s+" + id +
  316. "\\[\\]\\s+"
  317. "("
  318. "[^\\]\\[]*"
  319. "(?:"
  320. "\\["
  321. "([^\\[\\]]*(?2)?)*"
  322. "\\]"
  323. "[^\\]\\[]*"
  324. ")*"
  325. ")"
  326. "\\]");
  327. boost::smatch what;
  328. if(regex_search(content, what, content_e))
  329. {
  330. // replace existing content:
  331. content.replace(what.position(1), what.length(1), error_s);
  332. }
  333. else
  334. {
  335. // add new content:
  336. std::string::size_type pos = content.find("[/error_content:]");
  337. if(pos != std::string::npos)
  338. {
  339. content.insert(pos + 17, "\n[template " + id + "[]\n" + error_s + "\n]\n");
  340. }
  341. else
  342. content.append("\n[/error_content:]\n[template " + id + "[]\n" + error_s + "\n]\n");
  343. }
  344. //
  345. // Add to all_errors if not already there:
  346. //
  347. if(content.find("[" + id + "]") == std::string::npos)
  348. {
  349. // Find all_errors template:
  350. std::string::size_type pos = content.find("[template all_errors[]\n");
  351. if(pos != std::string::npos)
  352. {
  353. content.insert(pos + 23, "[" + id + "]\n");
  354. }
  355. else
  356. {
  357. content.append("\n[template all_errors[]\n[").append(id).append("]\n]\n");
  358. }
  359. }
  360. }
  361. void remove_error_content(const std::string& error_id)
  362. {
  363. // remove use template first:
  364. std::string::size_type pos = content.find("[" + error_id + "]");
  365. if(pos != std::string::npos)
  366. {
  367. content.erase(pos, 2 + error_id.size());
  368. }
  369. // then the template define itself:
  370. boost::regex content_e("\\[template\\s+" + error_id +
  371. "\\[\\]\\s+"
  372. "("
  373. "[^\\]\\[]*"
  374. "(?:"
  375. "\\["
  376. "([^\\[\\]]*(?2)?)*"
  377. "\\]"
  378. "[^\\]\\[]*"
  379. ")*"
  380. ")"
  381. "\\]");
  382. boost::smatch what;
  383. if(regex_search(content, what, content_e))
  384. {
  385. content.erase(what.position(), what.length());
  386. }
  387. }
  388. template <class T, class Seq>
  389. void handle_test_result(const boost::math::tools::test_result<T>& result,
  390. const Seq& worst, int row,
  391. const char* type_name,
  392. const char* test_name,
  393. const char* group_name)
  394. {
  395. T eps = boost::math::tools::epsilon<T>();
  396. T max_error_found = (result.max)() / eps;
  397. T mean_error_found = result.rms() / eps;
  398. std::string cell_name = sanitize_string(BOOST_COMPILER) + "_" + sanitize_string(BOOST_PLATFORM) + "_" + sanitize_string(type_name)
  399. + "_" + sanitize_string(test_name) + "_" + sanitize_string(TEST_LIBRARY_NAME) + "_" + sanitize_string(group_name);
  400. std::stringstream ss;
  401. ss << std::setprecision(3);
  402. if(std::string(TEST_LIBRARY_NAME) != "boost")
  403. ss << "(['" << TEST_LIBRARY_NAME << ":] ";
  404. else
  405. ss << "[role blue ";
  406. if((result.max)() > std::sqrt(eps))
  407. ss << "[role red ";
  408. ss << "Max = ";
  409. if((boost::math::isfinite)(max_error_found))
  410. ss << max_error_found;
  411. else
  412. ss << "+INF";
  413. ss << "[epsilon] (Mean = ";
  414. if((boost::math::isfinite)(mean_error_found))
  415. ss << mean_error_found;
  416. else
  417. ss << "+INF";
  418. ss << "[epsilon])";
  419. //
  420. // Now check for error output from gross errors or unexpected exceptions:
  421. //
  422. std::stringbuf* pbuf = dynamic_cast<std::stringbuf*>(std::cerr.rdbuf());
  423. bool have_errors = false;
  424. std::string error_id = "errors_" + cell_name;
  425. if(pbuf)
  426. {
  427. std::string err_s = pbuf->str();
  428. if(err_s.size())
  429. {
  430. if(err_s.size() > 4096)
  431. {
  432. std::string::size_type pos = err_s.find("\n", 4096);
  433. if(pos != std::string::npos)
  434. {
  435. err_s.erase(pos);
  436. err_s += "\n*** FURTHER CONTENT HAS BEEN TRUNCATED FOR BREVITY ***\n";
  437. }
  438. }
  439. std::string::size_type pos = err_s.find("\n");
  440. while(pos != std::string::npos)
  441. {
  442. err_s.replace(pos, 1, "[br]");
  443. pos = err_s.find("\n");
  444. }
  445. err_s = "[h4 Error Output For " + std::string(test_name) + std::string(" with compiler ") + std::string(BOOST_COMPILER)
  446. + std::string(" and library ") + std::string(TEST_LIBRARY_NAME) + " and test data "
  447. + std::string(group_name) + "]\n\n[#" + error_id + "]\n" + err_s + std::string("\n\n\n");
  448. ss << " [link " << error_id << " And other failures.]";
  449. pbuf->str("");
  450. set_error_content(error_id, err_s);
  451. have_errors = true;
  452. }
  453. }
  454. if(!have_errors)
  455. remove_error_content(error_id);
  456. if(std::string(TEST_LIBRARY_NAME) != "boost")
  457. ss << ")";
  458. else
  459. ss << "]";
  460. if((result.max)() > std::sqrt(eps))
  461. ss << "]";
  462. std::string cell_content = ss.str();
  463. set_result(cell_name, cell_content, test_name, group_name, type_name);
  464. }
  465. struct error_stream_replacer
  466. {
  467. std::streambuf* old_buf;
  468. std::stringstream ss;
  469. error_stream_replacer()
  470. {
  471. old_buf = std::cerr.rdbuf();
  472. std::cerr.rdbuf(ss.rdbuf());
  473. }
  474. ~error_stream_replacer()
  475. {
  476. std::cerr.rdbuf(old_buf);
  477. }
  478. };
  479. #endif // BOOST_MATH_HANDLE_TEST_RESULT