code_converter_test.cpp 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387
  1. // (C) Copyright 2008 CodeRage, LLC (turkanis at coderage dot com)
  2. // (C) Copyright 2004-2007 Jonathan Turkanis
  3. // Distributed under the Boost Software License, Version 1.0. (See accompanying
  4. // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt.)
  5. // See http://www.boost.org/libs/iostreams for documentation.
  6. #include <boost/iostreams/detail/config/wide_streams.hpp>
  7. #ifdef BOOST_IOSTREAMS_NO_WIDE_STREAMS
  8. # error wide streams not supported on this platform
  9. #endif
  10. #include <algorithm> // equal.
  11. #include <locale>
  12. #include <string>
  13. #include <boost/config.hpp> // BOOST_DEDUCED_TYPENAME.
  14. #include <boost/iostreams/code_converter.hpp>
  15. #include <boost/iostreams/copy.hpp>
  16. #include <boost/iostreams/detail/add_facet.hpp>
  17. #include <boost/iostreams/device/back_inserter.hpp>
  18. #include <boost/iostreams/detail/config/windows_posix.hpp>
  19. #include <boost/iostreams/device/file.hpp>
  20. #if !defined(__COMO__) || !defined(BOOST_COMO_STRICT)
  21. # if defined(BOOST_IOSTREAMS_NO_LIB) || defined(BOOST_ALL_NO_LIB)
  22. # include "../src/file_descriptor.cpp"
  23. # else
  24. # include <boost/iostreams/device/file_descriptor.hpp>
  25. # endif
  26. #endif
  27. #include <boost/iostreams/stream.hpp>
  28. #include <boost/test/test_tools.hpp>
  29. #include <boost/test/unit_test.hpp>
  30. #include "detail/closable.hpp"
  31. #include "detail/operation_sequence.hpp"
  32. #include "detail/temp_file.hpp"
  33. // Include codevct facets
  34. #include "detail/null_padded_codecvt.hpp"
  35. #include "detail/utf8_codecvt_facet.hpp"
  36. #ifdef BOOST_IOSTREAMS_USE_DINKUM_COREX
  37. # include <codecvt/8859_1>
  38. # include <codecvt/8859_10>
  39. # include <codecvt/8859_13>
  40. # include <codecvt/8859_14>
  41. # include <codecvt/8859_15>
  42. # include <codecvt/8859_16>
  43. # include <codecvt/8859_2>
  44. # include <codecvt/8859_3>
  45. # include <codecvt/8859_4>
  46. # include <codecvt/8859_5>
  47. # include <codecvt/8859_6>
  48. # include <codecvt/8859_7>
  49. # include <codecvt/8859_8>
  50. # include <codecvt/8859_9>
  51. # include <codecvt/baltic>
  52. # include <codecvt/big5>
  53. # include <codecvt/cp037>
  54. # include <codecvt/cp1006>
  55. # include <codecvt/cp1026>
  56. # include <codecvt/cp1250>
  57. # include <codecvt/cp1251>
  58. # include <codecvt/cp1252>
  59. # include <codecvt/cp1253>
  60. # include <codecvt/cp1254>
  61. # include <codecvt/cp1255>
  62. # include <codecvt/cp1256>
  63. # include <codecvt/cp1257>
  64. # include <codecvt/cp1258>
  65. # include <codecvt/cp424>
  66. # include <codecvt/cp437>
  67. # include <codecvt/cp500>
  68. # include <codecvt/cp737>
  69. # include <codecvt/cp775>
  70. # include <codecvt/cp850>
  71. # include <codecvt/cp852>
  72. # include <codecvt/cp855>
  73. # include <codecvt/cp856>
  74. # include <codecvt/cp857>
  75. # include <codecvt/cp860>
  76. # include <codecvt/cp861>
  77. # include <codecvt/cp862>
  78. # include <codecvt/cp863>
  79. # include <codecvt/cp864>
  80. # include <codecvt/cp865>
  81. # include <codecvt/cp866>
  82. # include <codecvt/cp869>
  83. # include <codecvt/cp874>
  84. # include <codecvt/cp875>
  85. # include <codecvt/cp932>
  86. # include <codecvt/cp936>
  87. # include <codecvt/cp949>
  88. # include <codecvt/cp950>
  89. # include <codecvt/cyrillic>
  90. # include <codecvt/ebcdic>
  91. # include <codecvt/euc>
  92. # include <codecvt/euc_0208>
  93. # include <codecvt/gb12345>
  94. # include <codecvt/gb2312>
  95. # include <codecvt/greek>
  96. # include <codecvt/iceland>
  97. # include <codecvt/jis>
  98. # include <codecvt/jis_0208>
  99. # include <codecvt/jis0201>
  100. # include <codecvt/ksc5601>
  101. # include <codecvt/latin2>
  102. # include <codecvt/one_one>
  103. # include <codecvt/roman>
  104. # include <codecvt/sjis>
  105. # include <codecvt/sjis_0208>
  106. # include <codecvt/turkish>
  107. # include <codecvt/utf16>
  108. # include <codecvt/utf8>
  109. # include <codecvt/utf8_utf16>
  110. # include <codecvt/xjis>
  111. #endif // #ifdef BOOST_IOSTREAMS_USE_DINKUM_COREX]
  112. #include <iostream>
  113. using namespace std;
  114. using namespace boost::iostreams;
  115. using namespace boost::iostreams::detail;
  116. using namespace boost::iostreams::test;
  117. using boost::unit_test::test_suite;
  118. namespace io = boost::iostreams;
  119. const int max_length = 30;
  120. const unsigned int pattern_length = 100;
  121. const unsigned int pattern_reps = 100;
  122. template<typename Codecvt>
  123. bool valid_char(typename codecvt_intern<Codecvt>::type c)
  124. {
  125. typedef typename codecvt_state<Codecvt>::type state_type;
  126. typedef typename codecvt_intern<Codecvt>::type intern_type;
  127. Codecvt cvt;
  128. state_type state = state_type();
  129. const intern_type* nint;
  130. char* next;
  131. char buf[max_length];
  132. return cvt.out( state, &c, &c + 1, nint,
  133. buf, buf + max_length, next )
  134. ==
  135. codecvt_base::ok;
  136. }
  137. template<typename Codecvt>
  138. basic_string<
  139. BOOST_DEDUCED_TYPENAME
  140. codecvt_intern<Codecvt>::type
  141. >
  142. test_string()
  143. {
  144. typedef typename codecvt_intern<Codecvt>::type intern_type;
  145. std::basic_string<intern_type> pattern, result;
  146. for (intern_type c = 255; pattern.size() < pattern_length; --c)
  147. if (valid_char<Codecvt>(c))
  148. pattern += c;
  149. result.reserve(pattern.size() * pattern_reps);
  150. for (unsigned int w = 0; w < pattern_reps; ++w)
  151. result += pattern;
  152. return result;
  153. }
  154. // Como can't compile file_descriptor.cpp in strict mode; this failure
  155. // is detected by file_descriptor_test.cpp.
  156. #if !defined(__COMO__) || !defined(BOOST_COMO_STRICT)
  157. typedef io::file_descriptor_source classic_file_source;
  158. typedef io::file_descriptor_sink classic_file_sink;
  159. #else
  160. struct classic_file_source : io::source {
  161. classic_file_source(const std::string& path)
  162. : file_(new filebuf)
  163. {
  164. file_->pubimbue(locale::classic());
  165. file_->open(path.c_str(), BOOST_IOS::in | BOOST_IOS::binary);
  166. }
  167. streamsize read(char* s, streamsize n) { return file_->sgetn(s, n); }
  168. boost::shared_ptr<filebuf> file_;
  169. };
  170. struct classic_file_sink : io::sink {
  171. classic_file_sink(const std::string& path)
  172. : file_(new filebuf)
  173. {
  174. file_->pubimbue(locale::classic());
  175. file_->open(path.c_str(), BOOST_IOS::out | BOOST_IOS::binary);
  176. }
  177. streamsize write(const char* s, streamsize n) { return file_->sputn(s, n); }
  178. boost::shared_ptr<filebuf> file_;
  179. };
  180. #endif
  181. template<typename Codecvt>
  182. bool codecvt_test1()
  183. {
  184. typedef basic_string<
  185. BOOST_DEDUCED_TYPENAME
  186. codecvt_intern<Codecvt>::type
  187. > string_type;
  188. typedef code_converter<classic_file_source, Codecvt> wide_file_source;
  189. typedef code_converter<classic_file_sink, Codecvt> wide_file_sink;
  190. BOOST_CHECK(Codecvt().max_length() <= max_length);
  191. temp_file temp;
  192. string_type test = test_string<Codecvt>();
  193. stream<wide_file_sink> out(temp.name());
  194. out.write(test.data(), static_cast<streamsize>(test.size()));
  195. out.close();
  196. stream<wide_file_source> in(temp.name());
  197. string_type test2;
  198. io::copy(in, io::back_inserter(test2));
  199. return test == test2;
  200. }
  201. template<typename Codecvt>
  202. bool codecvt_test2()
  203. {
  204. typedef basic_string<
  205. BOOST_DEDUCED_TYPENAME
  206. codecvt_intern<Codecvt>::type
  207. > string_type;
  208. typedef code_converter<classic_file_source> wide_file_source;
  209. typedef code_converter<classic_file_sink> wide_file_sink;
  210. // Set global locale.
  211. locale loc = add_facet(locale(), new Codecvt);
  212. locale::global(loc);
  213. temp_file temp;
  214. string_type test = test_string<Codecvt>();
  215. stream<wide_file_sink> out(temp.name());
  216. out.write(test.data(), static_cast<streamsize>(test.size()));
  217. out.close();
  218. stream<wide_file_source> in(temp.name());
  219. string_type test2;
  220. io::copy(in, io::back_inserter(test2));
  221. return test == test2;
  222. }
  223. template<typename Codecvt>
  224. bool codecvt_test()
  225. {
  226. return codecvt_test1<Codecvt>() && codecvt_test2<Codecvt>();
  227. }
  228. void code_converter_test()
  229. {
  230. BOOST_CHECK((codecvt_test<utf8_codecvt_facet<wchar_t, char> >()));
  231. BOOST_CHECK(codecvt_test<null_padded_codecvt>());
  232. BOOST_CHECK(codecvt_test<stateless_null_padded_codecvt>());
  233. #ifdef BOOST_IOSTREAMS_USE_DINKUM_COREX
  234. using namespace Dinkum::conversions;
  235. BOOST_CHECK(codecvt_test< codecvt_8859_1<wchar_t> >());
  236. BOOST_CHECK(codecvt_test< codecvt_8859_10<wchar_t> >());
  237. BOOST_CHECK(codecvt_test< codecvt_8859_13<wchar_t> >());
  238. BOOST_CHECK(codecvt_test< codecvt_8859_14<wchar_t> >());
  239. BOOST_CHECK(codecvt_test< codecvt_8859_15<wchar_t> >());
  240. BOOST_CHECK(codecvt_test< codecvt_8859_16<wchar_t> >());
  241. BOOST_CHECK(codecvt_test< codecvt_8859_2<wchar_t> >());
  242. BOOST_CHECK(codecvt_test< codecvt_8859_3<wchar_t> >());
  243. BOOST_CHECK(codecvt_test< codecvt_8859_4<wchar_t> >());
  244. BOOST_CHECK(codecvt_test< codecvt_8859_5<wchar_t> >());
  245. BOOST_CHECK(codecvt_test< codecvt_8859_6<wchar_t> >());
  246. BOOST_CHECK(codecvt_test< codecvt_8859_7<wchar_t> >());
  247. BOOST_CHECK(codecvt_test< codecvt_8859_8<wchar_t> >());
  248. BOOST_CHECK(codecvt_test< codecvt_8859_9<wchar_t> >());
  249. BOOST_CHECK(codecvt_test< codecvt_baltic<wchar_t> >());
  250. BOOST_CHECK(codecvt_test< codecvt_big5<wchar_t> >());
  251. BOOST_CHECK(codecvt_test< codecvt_cp037<wchar_t> >());
  252. BOOST_CHECK(codecvt_test< codecvt_cp1006<wchar_t> >());
  253. BOOST_CHECK(codecvt_test< codecvt_cp1026<wchar_t> >());
  254. BOOST_CHECK(codecvt_test< codecvt_cp1250<wchar_t> >());
  255. BOOST_CHECK(codecvt_test< codecvt_cp1251<wchar_t> >());
  256. BOOST_CHECK(codecvt_test< codecvt_cp1252<wchar_t> >());
  257. BOOST_CHECK(codecvt_test< codecvt_cp1253<wchar_t> >());
  258. BOOST_CHECK(codecvt_test< codecvt_cp1254<wchar_t> >());
  259. BOOST_CHECK(codecvt_test< codecvt_cp1255<wchar_t> >());
  260. BOOST_CHECK(codecvt_test< codecvt_cp1256<wchar_t> >());
  261. BOOST_CHECK(codecvt_test< codecvt_cp1257<wchar_t> >());
  262. BOOST_CHECK(codecvt_test< codecvt_cp1258<wchar_t> >());
  263. BOOST_CHECK(codecvt_test< codecvt_cp424<wchar_t> >());
  264. BOOST_CHECK(codecvt_test< codecvt_cp437<wchar_t> >());
  265. BOOST_CHECK(codecvt_test< codecvt_cp500<wchar_t> >());
  266. BOOST_CHECK(codecvt_test< codecvt_cp737<wchar_t> >());
  267. BOOST_CHECK(codecvt_test< codecvt_cp775<wchar_t> >());
  268. BOOST_CHECK(codecvt_test< codecvt_cp850<wchar_t> >());
  269. BOOST_CHECK(codecvt_test< codecvt_cp852<wchar_t> >());
  270. BOOST_CHECK(codecvt_test< codecvt_cp855<wchar_t> >());
  271. BOOST_CHECK(codecvt_test< codecvt_cp856<wchar_t> >());
  272. BOOST_CHECK(codecvt_test< codecvt_cp857<wchar_t> >());
  273. BOOST_CHECK(codecvt_test< codecvt_cp860<wchar_t> >());
  274. BOOST_CHECK(codecvt_test< codecvt_cp861<wchar_t> >());
  275. BOOST_CHECK(codecvt_test< codecvt_cp862<wchar_t> >());
  276. BOOST_CHECK(codecvt_test< codecvt_cp863<wchar_t> >());
  277. BOOST_CHECK(codecvt_test< codecvt_cp864<wchar_t> >());
  278. BOOST_CHECK(codecvt_test< codecvt_cp865<wchar_t> >());
  279. BOOST_CHECK(codecvt_test< codecvt_cp866<wchar_t> >());
  280. BOOST_CHECK(codecvt_test< codecvt_cp869<wchar_t> >());
  281. BOOST_CHECK(codecvt_test< codecvt_cp874<wchar_t> >());
  282. BOOST_CHECK(codecvt_test< codecvt_cp875<wchar_t> >());
  283. BOOST_CHECK(codecvt_test< codecvt_cp932<wchar_t> >());
  284. BOOST_CHECK(codecvt_test< codecvt_cp936<wchar_t> >());
  285. BOOST_CHECK(codecvt_test< codecvt_cp949<wchar_t> >());
  286. BOOST_CHECK(codecvt_test< codecvt_cp950<wchar_t> >());
  287. BOOST_CHECK(codecvt_test< codecvt_cyrillic<wchar_t> >());
  288. BOOST_CHECK(codecvt_test< codecvt_ebcdic<wchar_t> >());
  289. BOOST_CHECK(codecvt_test< codecvt_euc<wchar_t> >());
  290. BOOST_CHECK(codecvt_test< codecvt_euc_0208<wchar_t> >());
  291. BOOST_CHECK(codecvt_test< codecvt_gb12345<wchar_t> >());
  292. BOOST_CHECK(codecvt_test< codecvt_gb2312<wchar_t> >());
  293. BOOST_CHECK(codecvt_test< codecvt_greek<wchar_t> >());
  294. BOOST_CHECK(codecvt_test< codecvt_iceland<wchar_t> >());
  295. BOOST_CHECK(codecvt_test< codecvt_jis<wchar_t> >());
  296. BOOST_CHECK(codecvt_test< codecvt_jis_0208<wchar_t> >());
  297. BOOST_CHECK(codecvt_test< codecvt_jis0201<wchar_t> >());
  298. BOOST_CHECK(codecvt_test< codecvt_ksc5601<wchar_t> >());
  299. BOOST_CHECK(codecvt_test< codecvt_latin2<wchar_t> >());
  300. BOOST_CHECK(codecvt_test< codecvt_one_one<wchar_t> >());
  301. BOOST_CHECK(codecvt_test< codecvt_roman<wchar_t> >());
  302. BOOST_CHECK(codecvt_test< codecvt_sjis<wchar_t> >());
  303. BOOST_CHECK(codecvt_test< codecvt_sjis_0208<wchar_t> >());
  304. BOOST_CHECK(codecvt_test< codecvt_turkish<wchar_t> >());
  305. BOOST_CHECK(codecvt_test< codecvt_utf16<wchar_t> >());
  306. BOOST_CHECK(codecvt_test< codecvt_utf8<wchar_t> >());
  307. BOOST_CHECK(codecvt_test< codecvt_utf8_utf16<wchar_t> >());
  308. #endif
  309. }
  310. /* Defer pending further testing
  311. void close_test()
  312. {
  313. typedef utf8_codecvt_facet<wchar_t, char> codecvt_type;
  314. // Test code converter based on a source
  315. {
  316. operation_sequence seq;
  317. io::wchain<input> ch;
  318. ch.push(
  319. code_converter<closable_device<input>, codecvt_type>(
  320. seq.new_operation(1)
  321. )
  322. );
  323. BOOST_CHECK_NO_THROW(ch.reset());
  324. BOOST_CHECK_OPERATION_SEQUENCE(seq);
  325. }
  326. // Test code converter based on a sink
  327. {
  328. operation_sequence seq;
  329. io::wchain<output> ch;
  330. ch.push(
  331. code_converter<closable_device<output>, codecvt_type>(
  332. seq.new_operation(1)
  333. )
  334. );
  335. BOOST_CHECK_NO_THROW(ch.reset());
  336. BOOST_CHECK_OPERATION_SEQUENCE(seq);
  337. }
  338. // Test code converter based on a bidirectional device
  339. {
  340. operation_sequence seq;
  341. io::wchain<bidirectional> ch;
  342. ch.push(
  343. code_converter<closable_device<bidirectional>, codecvt_type>(
  344. seq.new_operation(1),
  345. seq.new_operation(2)
  346. )
  347. );
  348. BOOST_CHECK_NO_THROW(ch.reset());
  349. BOOST_CHECK_OPERATION_SEQUENCE(seq);
  350. }
  351. }*/
  352. test_suite* init_unit_test_suite(int, char* [])
  353. {
  354. test_suite* test = BOOST_TEST_SUITE("code_converter test");
  355. test->add(BOOST_TEST_CASE(&code_converter_test));
  356. //test->add(BOOST_TEST_CASE(&close_test));
  357. return test;
  358. }