char_token_def.hpp 7.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242
  1. // Copyright (c) 2001-2011 Hartmut Kaiser
  2. //
  3. // Distributed under the Boost Software License, Version 1.0. (See accompanying
  4. // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
  5. #if !defined(BOOST_SPIRIT_LEX_CHAR_TOKEN_DEF_MAR_28_2007_0626PM)
  6. #define BOOST_SPIRIT_LEX_CHAR_TOKEN_DEF_MAR_28_2007_0626PM
  7. #if defined(_MSC_VER)
  8. #pragma once
  9. #endif
  10. #include <boost/spirit/home/support/common_terminals.hpp>
  11. #include <boost/spirit/home/support/string_traits.hpp>
  12. #include <boost/spirit/home/lex/domain.hpp>
  13. #include <boost/spirit/home/lex/lexer_type.hpp>
  14. #include <boost/spirit/home/lex/meta_compiler.hpp>
  15. namespace boost { namespace spirit
  16. {
  17. ///////////////////////////////////////////////////////////////////////////
  18. // Enablers
  19. ///////////////////////////////////////////////////////////////////////////
  20. // enables 'x'
  21. template <>
  22. struct use_terminal<lex::domain, char>
  23. : mpl::true_ {};
  24. // enables "x"
  25. template <>
  26. struct use_terminal<lex::domain, char[2]>
  27. : mpl::true_ {};
  28. // enables wchar_t
  29. template <>
  30. struct use_terminal<lex::domain, wchar_t>
  31. : mpl::true_ {};
  32. // enables L"x"
  33. template <>
  34. struct use_terminal<lex::domain, wchar_t[2]>
  35. : mpl::true_ {};
  36. // enables char_('x'), char_("x")
  37. template <typename CharEncoding, typename A0>
  38. struct use_terminal<lex::domain
  39. , terminal_ex<
  40. tag::char_code<tag::char_, CharEncoding>
  41. , fusion::vector1<A0> > >
  42. : mpl::true_ {};
  43. // enables char_('x', ID), char_("x", ID)
  44. template <typename CharEncoding, typename A0, typename A1>
  45. struct use_terminal<lex::domain
  46. , terminal_ex<
  47. tag::char_code<tag::char_, CharEncoding>
  48. , fusion::vector2<A0, A1> > >
  49. : mpl::true_ {};
  50. }}
  51. namespace boost { namespace spirit { namespace lex
  52. {
  53. // use char_ from standard character set by default
  54. #ifndef BOOST_SPIRIT_NO_PREDEFINED_TERMINALS
  55. using spirit::standard::char_;
  56. #endif
  57. using spirit::standard::char_type;
  58. ///////////////////////////////////////////////////////////////////////////
  59. //
  60. // char_token_def
  61. // represents a single character token definition
  62. //
  63. ///////////////////////////////////////////////////////////////////////////
  64. template <typename CharEncoding = char_encoding::standard
  65. , typename IdType = std::size_t>
  66. struct char_token_def
  67. : primitive_lexer<char_token_def<CharEncoding, IdType> >
  68. {
  69. typedef typename CharEncoding::char_type char_type;
  70. char_token_def(char_type ch, IdType const& id)
  71. : ch(ch), id_(id), unique_id_(std::size_t(~0))
  72. , token_state_(std::size_t(~0))
  73. {}
  74. template <typename LexerDef, typename String>
  75. void collect(LexerDef& lexdef, String const& state
  76. , String const& targetstate) const
  77. {
  78. std::size_t state_id = lexdef.add_state(state.c_str());
  79. // If the following assertion fires you are probably trying to use
  80. // a single char_token_def instance in more than one lexer state.
  81. // This is not possible. Please create a separate token_def instance
  82. // from the same regular expression for each lexer state it needs
  83. // to be associated with.
  84. BOOST_ASSERT(
  85. (std::size_t(~0) == token_state_ || state_id == token_state_) &&
  86. "Can't use single char_token_def with more than one lexer state");
  87. char_type const* target = targetstate.empty() ? 0 : targetstate.c_str();
  88. if (target)
  89. lexdef.add_state(target);
  90. token_state_ = state_id;
  91. unique_id_ = lexdef.add_token (state.c_str(), ch, id_, target);
  92. }
  93. template <typename LexerDef>
  94. void add_actions(LexerDef&) const {}
  95. IdType id() const { return id_; }
  96. std::size_t unique_id() const { return unique_id_; }
  97. std::size_t state() const { return token_state_; }
  98. char_type ch;
  99. mutable IdType id_;
  100. mutable std::size_t unique_id_;
  101. mutable std::size_t token_state_;
  102. };
  103. ///////////////////////////////////////////////////////////////////////////
  104. // Lexer generators: make_xxx function (objects)
  105. ///////////////////////////////////////////////////////////////////////////
  106. namespace detail
  107. {
  108. template <typename CharEncoding>
  109. struct basic_literal
  110. {
  111. typedef char_token_def<CharEncoding> result_type;
  112. template <typename Char>
  113. result_type operator()(Char ch, unused_type) const
  114. {
  115. return result_type(ch, ch);
  116. }
  117. template <typename Char>
  118. result_type operator()(Char const* str, unused_type) const
  119. {
  120. return result_type(str[0], str[0]);
  121. }
  122. };
  123. }
  124. // literals: 'x', "x"
  125. template <typename Modifiers>
  126. struct make_primitive<char, Modifiers>
  127. : detail::basic_literal<char_encoding::standard> {};
  128. template <typename Modifiers>
  129. struct make_primitive<char const(&)[2], Modifiers>
  130. : detail::basic_literal<char_encoding::standard> {};
  131. // literals: L'x', L"x"
  132. template <typename Modifiers>
  133. struct make_primitive<wchar_t, Modifiers>
  134. : detail::basic_literal<char_encoding::standard_wide> {};
  135. template <typename Modifiers>
  136. struct make_primitive<wchar_t const(&)[2], Modifiers>
  137. : detail::basic_literal<char_encoding::standard_wide> {};
  138. // handle char_('x')
  139. template <typename CharEncoding, typename Modifiers, typename A0>
  140. struct make_primitive<
  141. terminal_ex<
  142. tag::char_code<tag::char_, CharEncoding>
  143. , fusion::vector1<A0>
  144. >
  145. , Modifiers>
  146. {
  147. typedef char_token_def<CharEncoding> result_type;
  148. template <typename Terminal>
  149. result_type operator()(Terminal const& term, unused_type) const
  150. {
  151. return result_type(fusion::at_c<0>(term.args), fusion::at_c<0>(term.args));
  152. }
  153. };
  154. // handle char_("x")
  155. template <typename CharEncoding, typename Modifiers, typename Char>
  156. struct make_primitive<
  157. terminal_ex<
  158. tag::char_code<tag::char_, CharEncoding>
  159. , fusion::vector1<Char(&)[2]> // single char strings
  160. >
  161. , Modifiers>
  162. {
  163. typedef char_token_def<CharEncoding> result_type;
  164. template <typename Terminal>
  165. result_type operator()(Terminal const& term, unused_type) const
  166. {
  167. Char ch = fusion::at_c<0>(term.args)[0];
  168. return result_type(ch, ch);
  169. }
  170. };
  171. // handle char_('x', ID)
  172. template <typename CharEncoding, typename Modifiers, typename A0, typename A1>
  173. struct make_primitive<
  174. terminal_ex<
  175. tag::char_code<tag::char_, CharEncoding>
  176. , fusion::vector2<A0, A1>
  177. >
  178. , Modifiers>
  179. {
  180. typedef char_token_def<CharEncoding> result_type;
  181. template <typename Terminal>
  182. result_type operator()(Terminal const& term, unused_type) const
  183. {
  184. return result_type(
  185. fusion::at_c<0>(term.args), fusion::at_c<1>(term.args));
  186. }
  187. };
  188. // handle char_("x", ID)
  189. template <typename CharEncoding, typename Modifiers, typename Char, typename A1>
  190. struct make_primitive<
  191. terminal_ex<
  192. tag::char_code<tag::char_, CharEncoding>
  193. , fusion::vector2<Char(&)[2], A1> // single char strings
  194. >
  195. , Modifiers>
  196. {
  197. typedef char_token_def<CharEncoding> result_type;
  198. template <typename Terminal>
  199. result_type operator()(Terminal const& term, unused_type) const
  200. {
  201. return result_type(
  202. fusion::at_c<0>(term.args)[0], fusion::at_c<1>(term.args));
  203. }
  204. };
  205. }}} // namespace boost::spirit::lex
  206. #endif