123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023 |
- // Copyright (c) 2008-2009 Ben Hanson
- // Copyright (c) 2008-2011 Hartmut Kaiser
- //
- // Distributed under the Boost Software License, Version 1.0. (See accompanying
- // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
- #if !defined(BOOST_SPIRIT_LEX_LEXERTL_GENERATE_CPP_FEB_10_2008_0855PM)
- #define BOOST_SPIRIT_LEX_LEXERTL_GENERATE_CPP_FEB_10_2008_0855PM
- #if defined(_MSC_VER)
- #pragma once
- #endif
- #include <boost/spirit/home/support/detail/lexer/char_traits.hpp>
- #include <boost/spirit/home/support/detail/lexer/consts.hpp>
- #include <boost/spirit/home/support/detail/lexer/rules.hpp>
- #include <boost/spirit/home/support/detail/lexer/size_t.hpp>
- #include <boost/spirit/home/support/detail/lexer/state_machine.hpp>
- #include <boost/spirit/home/support/detail/lexer/debug.hpp>
- #include <boost/spirit/home/lex/lexer/lexertl/static_version.hpp>
- #include <boost/algorithm/string.hpp>
- #include <boost/scoped_array.hpp>
- ///////////////////////////////////////////////////////////////////////////////
- namespace boost { namespace spirit { namespace lex { namespace lexertl
- {
- namespace detail
- {
- ///////////////////////////////////////////////////////////////////////////
- template <typename CharT>
- struct string_lit;
- template <>
- struct string_lit<char>
- {
- static char get(char c) { return c; }
- static std::string get(char const* str = "") { return str; }
- };
- template <>
- struct string_lit<wchar_t>
- {
- static wchar_t get(char c)
- {
- typedef std::ctype<wchar_t> ctype_t;
- return std::use_facet<ctype_t>(std::locale()).widen(c);
- }
- static std::basic_string<wchar_t> get(char const* source = "")
- {
- using namespace std; // some systems have size_t in ns std
- size_t len = strlen(source);
- boost::scoped_array<wchar_t> result (new wchar_t[len+1]);
- result.get()[len] = '\0';
- // working with wide character streams is supported only if the
- // platform provides the std::ctype<wchar_t> facet
- BOOST_ASSERT(std::has_facet<std::ctype<wchar_t> >(std::locale()));
- std::use_facet<std::ctype<wchar_t> >(std::locale())
- .widen(source, source + len, result.get());
- return result.get();
- }
- };
- template <typename Char>
- inline Char L(char c)
- {
- return string_lit<Char>::get(c);
- }
- template <typename Char>
- inline std::basic_string<Char> L(char const* c = "")
- {
- return string_lit<Char>::get(c);
- }
- ///////////////////////////////////////////////////////////////////////////
- template <typename Char>
- inline bool
- generate_delimiter(std::basic_ostream<Char> &os_)
- {
- os_ << std::basic_string<Char>(80, '/') << "\n";
- return os_.good();
- }
- ///////////////////////////////////////////////////////////////////////////
- // Generate a table of the names of the used lexer states, which is a bit
- // tricky, because the table stored with the rules is sorted based on the
- // names, but we need it sorted using the state ids.
- template <typename Char>
- inline bool
- generate_cpp_state_info (boost::lexer::basic_rules<Char> const& rules_
- , std::basic_ostream<Char> &os_, Char const* name_suffix)
- {
- // we need to re-sort the state names in ascending order of the state
- // ids, filling possible gaps in between later
- typedef typename
- boost::lexer::basic_rules<Char>::string_size_t_map::const_iterator
- state_iterator;
- typedef std::map<std::size_t, Char const*> reverse_state_map_type;
- reverse_state_map_type reverse_state_map;
- state_iterator send = rules_.statemap().end();
- for (state_iterator sit = rules_.statemap().begin(); sit != send; ++sit)
- {
- typedef typename reverse_state_map_type::value_type value_type;
- reverse_state_map.insert(value_type((*sit).second, (*sit).first.c_str()));
- }
- generate_delimiter(os_);
- os_ << "// this table defines the names of the lexer states\n";
- os_ << boost::lexer::detail::strings<Char>::char_name()
- << " const* const lexer_state_names"
- << (name_suffix[0] ? "_" : "") << name_suffix
- << "[" << rules_.statemap().size() << "] = \n{\n";
- typedef typename reverse_state_map_type::iterator iterator;
- iterator rend = reverse_state_map.end();
- std::size_t last_id = 0;
- for (iterator rit = reverse_state_map.begin(); rit != rend; ++last_id)
- {
- for (/**/; last_id < (*rit).first; ++last_id)
- {
- os_ << " 0, // \"<undefined state>\"\n";
- }
- os_ << " "
- << boost::lexer::detail::strings<Char>::char_prefix()
- << "\"" << (*rit).second << "\"";
- if (++rit != rend)
- os_ << ",\n";
- else
- os_ << "\n"; // don't generate the final comma
- }
- os_ << "};\n\n";
- generate_delimiter(os_);
- os_ << "// this variable defines the number of lexer states\n";
- os_ << "std::size_t const lexer_state_count"
- << (name_suffix[0] ? "_" : "") << name_suffix
- << " = " << rules_.statemap().size() << ";\n\n";
- return os_.good();
- }
- template <typename Char>
- inline bool
- generate_cpp_state_table (std::basic_ostream<Char> &os_
- , Char const* name_suffix, bool bol, bool eol)
- {
- std::basic_string<Char> suffix(L<Char>(name_suffix[0] ? "_" : ""));
- suffix += name_suffix;
- generate_delimiter(os_);
- os_ << "// this defines a generic accessors for the information above\n";
- os_ << "struct lexer" << suffix << "\n{\n";
- os_ << " // version number and feature-set of compatible static lexer engine\n";
- os_ << " enum\n";
- os_ << " {\n static_version = " << SPIRIT_STATIC_LEXER_VERSION << ",\n";
- os_ << " supports_bol = " << std::boolalpha << bol << ",\n";
- os_ << " supports_eol = " << std::boolalpha << eol << "\n";
- os_ << " };\n\n";
- os_ << " // return the number of lexer states\n";
- os_ << " static std::size_t state_count()\n";
- os_ << " {\n return lexer_state_count" << suffix << "; \n }\n\n";
- os_ << " // return the name of the lexer state as given by 'idx'\n";
- os_ << " static " << boost::lexer::detail::strings<Char>::char_name()
- << " const* state_name(std::size_t idx)\n";
- os_ << " {\n return lexer_state_names" << suffix << "[idx]; \n }\n\n";
- os_ << " // return the next matched token\n";
- os_ << " template<typename Iterator>\n";
- os_ << " static std::size_t next(std::size_t &start_state_, bool& bol_\n";
- os_ << " , Iterator &start_token_, Iterator const& end_, std::size_t& unique_id_)\n";
- os_ << " {\n return next_token" << suffix
- << "(start_state_, bol_, start_token_, end_, unique_id_);\n }\n";
- os_ << "};\n\n";
- return os_.good();
- }
- ///////////////////////////////////////////////////////////////////////////
- // generate function body based on traversing the DFA tables
- template <typename Char>
- bool generate_function_body_dfa(std::basic_ostream<Char>& os_
- , boost::lexer::basic_state_machine<Char> const &sm_)
- {
- std::size_t const dfas_ = sm_.data()._dfa->size();
- std::size_t const lookups_ = sm_.data()._lookup->front()->size();
- os_ << " enum {end_state_index, id_index, unique_id_index, "
- "state_index, bol_index,\n";
- os_ << " eol_index, dead_state_index, dfa_offset};\n\n";
- os_ << " static std::size_t const npos = "
- "static_cast<std::size_t>(~0);\n";
- if (dfas_ > 1)
- {
- for (std::size_t state_ = 0; state_ < dfas_; ++state_)
- {
- std::size_t i_ = 0;
- std::size_t j_ = 1;
- std::size_t count_ = lookups_ / 8;
- std::size_t const* lookup_ = &sm_.data()._lookup[state_]->front();
- std::size_t const* dfa_ = &sm_.data()._dfa[state_]->front();
- os_ << " static std::size_t const lookup" << state_
- << "_[" << lookups_ << "] = {\n ";
- for (/**/; i_ < count_; ++i_)
- {
- std::size_t const index_ = i_ * 8;
- os_ << lookup_[index_];
- for (/**/; j_ < 8; ++j_)
- {
- os_ << ", " << lookup_[index_ + j_];
- }
- if (i_ < count_ - 1)
- {
- os_ << ",\n ";
- }
- j_ = 1;
- }
- os_ << " };\n";
- count_ = sm_.data()._dfa[state_]->size ();
- os_ << " static const std::size_t dfa" << state_ << "_["
- << count_ << "] = {\n ";
- count_ /= 8;
- for (i_ = 0; i_ < count_; ++i_)
- {
- std::size_t const index_ = i_ * 8;
- os_ << dfa_[index_];
- for (j_ = 1; j_ < 8; ++j_)
- {
- os_ << ", " << dfa_[index_ + j_];
- }
- if (i_ < count_ - 1)
- {
- os_ << ",\n ";
- }
- }
- std::size_t const mod_ = sm_.data()._dfa[state_]->size () % 8;
- if (mod_)
- {
- std::size_t const index_ = count_ * 8;
- if (count_)
- {
- os_ << ",\n ";
- }
- os_ << dfa_[index_];
- for (j_ = 1; j_ < mod_; ++j_)
- {
- os_ << ", " << dfa_[index_ + j_];
- }
- }
- os_ << " };\n";
- }
- std::size_t count_ = sm_.data()._dfa_alphabet.size();
- std::size_t i_ = 1;
- os_ << " static std::size_t const* lookup_arr_[" << count_
- << "] = { lookup0_";
- for (i_ = 1; i_ < count_; ++i_)
- {
- os_ << ", " << "lookup" << i_ << "_";
- }
- os_ << " };\n";
- os_ << " static std::size_t const dfa_alphabet_arr_["
- << count_ << "] = { ";
- os_ << sm_.data()._dfa_alphabet.front ();
- for (i_ = 1; i_ < count_; ++i_)
- {
- os_ << ", " << sm_.data()._dfa_alphabet[i_];
- }
- os_ << " };\n";
- os_ << " static std::size_t const* dfa_arr_[" << count_
- << "] = { ";
- os_ << "dfa0_";
- for (i_ = 1; i_ < count_; ++i_)
- {
- os_ << ", " << "dfa" << i_ << "_";
- }
- os_ << " };\n";
- }
- else
- {
- std::size_t const* lookup_ = &sm_.data()._lookup[0]->front();
- std::size_t const* dfa_ = &sm_.data()._dfa[0]->front();
- std::size_t i_ = 0;
- std::size_t j_ = 1;
- std::size_t count_ = lookups_ / 8;
- os_ << " static std::size_t const lookup_[";
- os_ << sm_.data()._lookup[0]->size() << "] = {\n ";
- for (/**/; i_ < count_; ++i_)
- {
- const std::size_t index_ = i_ * 8;
- os_ << lookup_[index_];
- for (/**/; j_ < 8; ++j_)
- {
- os_ << ", " << lookup_[index_ + j_];
- }
- if (i_ < count_ - 1)
- {
- os_ << ",\n ";
- }
- j_ = 1;
- }
- os_ << " };\n";
- os_ << " static std::size_t const dfa_alphabet_ = "
- << sm_.data()._dfa_alphabet.front () << ";\n";
- os_ << " static std::size_t const dfa_["
- << sm_.data()._dfa[0]->size () << "] = {\n ";
- count_ = sm_.data()._dfa[0]->size () / 8;
- for (i_ = 0; i_ < count_; ++i_)
- {
- const std::size_t index_ = i_ * 8;
- os_ << dfa_[index_];
- for (j_ = 1; j_ < 8; ++j_)
- {
- os_ << ", " << dfa_[index_ + j_];
- }
- if (i_ < count_ - 1)
- {
- os_ << ",\n ";
- }
- }
- const std::size_t mod_ = sm_.data()._dfa[0]->size () % 8;
- if (mod_)
- {
- const std::size_t index_ = count_ * 8;
- if (count_)
- {
- os_ << ",\n ";
- }
- os_ << dfa_[index_];
- for (j_ = 1; j_ < mod_; ++j_)
- {
- os_ << ", " << dfa_[index_ + j_];
- }
- }
- os_ << " };\n";
- }
- os_ << "\n if (start_token_ == end_)\n";
- os_ << " {\n";
- os_ << " unique_id_ = npos;\n";
- os_ << " return 0;\n";
- os_ << " }\n\n";
- if (sm_.data()._seen_BOL_assertion)
- {
- os_ << " bool bol = bol_;\n\n";
- }
- if (dfas_ > 1)
- {
- os_ << "again:\n";
- os_ << " std::size_t const* lookup_ = lookup_arr_[start_state_];\n";
- os_ << " std::size_t dfa_alphabet_ = dfa_alphabet_arr_[start_state_];\n";
- os_ << " std::size_t const*dfa_ = dfa_arr_[start_state_];\n";
- }
- os_ << " std::size_t const* ptr_ = dfa_ + dfa_alphabet_;\n";
- os_ << " Iterator curr_ = start_token_;\n";
- os_ << " bool end_state_ = *ptr_ != 0;\n";
- os_ << " std::size_t id_ = *(ptr_ + id_index);\n";
- os_ << " std::size_t uid_ = *(ptr_ + unique_id_index);\n";
- if (dfas_ > 1)
- {
- os_ << " std::size_t end_start_state_ = start_state_;\n";
- }
- if (sm_.data()._seen_BOL_assertion)
- {
- os_ << " bool end_bol_ = bol_;\n";
- }
- os_ << " Iterator end_token_ = start_token_;\n\n";
- os_ << " while (curr_ != end_)\n";
- os_ << " {\n";
- if (sm_.data()._seen_BOL_assertion)
- {
- os_ << " std::size_t const BOL_state_ = ptr_[bol_index];\n\n";
- }
- if (sm_.data()._seen_EOL_assertion)
- {
- os_ << " std::size_t const EOL_state_ = ptr_[eol_index];\n\n";
- }
- if (sm_.data()._seen_BOL_assertion && sm_.data()._seen_EOL_assertion)
- {
- os_ << " if (BOL_state_ && bol)\n";
- os_ << " {\n";
- os_ << " ptr_ = &dfa_[BOL_state_ * dfa_alphabet_];\n";
- os_ << " }\n";
- os_ << " else if (EOL_state_ && *curr_ == '\\n')\n";
- os_ << " {\n";
- os_ << " ptr_ = &dfa_[EOL_state_ * dfa_alphabet_];\n";
- os_ << " }\n";
- os_ << " else\n";
- os_ << " {\n";
- if (lookups_ == 256)
- {
- os_ << " unsigned char index = \n";
- os_ << " static_cast<unsigned char>(*curr_++);\n";
- }
- else
- {
- os_ << " std::size_t index = *curr_++\n";
- }
- os_ << " bol = (index == '\\n') ? true : false;\n";
- os_ << " std::size_t const state_ = ptr_[\n";
- os_ << " lookup_[static_cast<std::size_t>(index)]];\n";
- os_ << '\n';
- os_ << " if (state_ == 0) break;\n";
- os_ << '\n';
- os_ << " ptr_ = &dfa_[state_ * dfa_alphabet_];\n";
- os_ << " }\n\n";
- }
- else if (sm_.data()._seen_BOL_assertion)
- {
- os_ << " if (BOL_state_ && bol)\n";
- os_ << " {\n";
- os_ << " ptr_ = &dfa_[BOL_state_ * dfa_alphabet_];\n";
- os_ << " }\n";
- os_ << " else\n";
- os_ << " {\n";
- if (lookups_ == 256)
- {
- os_ << " unsigned char index = \n";
- os_ << " static_cast<unsigned char>(*curr_++);\n";
- }
- else
- {
- os_ << " std::size_t index = *curr_++\n";
- }
- os_ << " bol = (index == '\\n') ? true : false;\n";
- os_ << " std::size_t const state_ = ptr_[\n";
- os_ << " lookup_[static_cast<std::size_t>(index)]];\n";
- os_ << '\n';
- os_ << " if (state_ == 0) break;\n";
- os_ << '\n';
- os_ << " ptr_ = &dfa_[state_ * dfa_alphabet_];\n";
- os_ << " }\n\n";
- }
- else if (sm_.data()._seen_EOL_assertion)
- {
- os_ << " if (EOL_state_ && *curr_ == '\\n')\n";
- os_ << " {\n";
- os_ << " ptr_ = &dfa_[EOL_state_ * dfa_alphabet_];\n";
- os_ << " }\n";
- os_ << " else\n";
- os_ << " {\n";
- if (lookups_ == 256)
- {
- os_ << " unsigned char index = \n";
- os_ << " static_cast<unsigned char>(*curr_++);\n";
- }
- else
- {
- os_ << " std::size_t index = *curr_++\n";
- }
- os_ << " bol = (index == '\\n') ? true : false;\n";
- os_ << " std::size_t const state_ = ptr_[\n";
- os_ << " lookup_[static_cast<std::size_t>(index)]];\n";
- os_ << '\n';
- os_ << " if (state_ == 0) break;\n";
- os_ << '\n';
- os_ << " ptr_ = &dfa_[state_ * dfa_alphabet_];\n";
- os_ << " }\n\n";
- }
- else
- {
- os_ << " std::size_t const state_ =\n";
- if (lookups_ == 256)
- {
- os_ << " ptr_[lookup_["
- "static_cast<unsigned char>(*curr_++)]];\n";
- }
- else
- {
- os_ << " ptr_[lookup_[*curr_++]];\n";
- }
- os_ << '\n';
- os_ << " if (state_ == 0) break;\n";
- os_ << '\n';
- os_ << " ptr_ = &dfa_[state_ * dfa_alphabet_];\n\n";
- }
- os_ << " if (*ptr_)\n";
- os_ << " {\n";
- os_ << " end_state_ = true;\n";
- os_ << " id_ = *(ptr_ + id_index);\n";
- os_ << " uid_ = *(ptr_ + unique_id_index);\n";
- if (dfas_ > 1)
- {
- os_ << " end_start_state_ = *(ptr_ + state_index);\n";
- }
- if (sm_.data()._seen_BOL_assertion)
- {
- os_ << " end_bol_ = bol;\n";
- }
- os_ << " end_token_ = curr_;\n";
- os_ << " }\n";
- os_ << " }\n\n";
- if (sm_.data()._seen_EOL_assertion)
- {
- os_ << " std::size_t const EOL_state_ = ptr_[eol_index];\n\n";
- os_ << " if (EOL_state_ && curr_ == end_)\n";
- os_ << " {\n";
- os_ << " ptr_ = &dfa_[EOL_state_ * dfa_alphabet_];\n\n";
- os_ << " if (*ptr_)\n";
- os_ << " {\n";
- os_ << " end_state_ = true;\n";
- os_ << " id_ = *(ptr_ + id_index);\n";
- os_ << " uid_ = *(ptr_ + unique_id_index);\n";
- if (dfas_ > 1)
- {
- os_ << " end_start_state_ = *(ptr_ + state_index);\n";
- }
- if (sm_.data()._seen_BOL_assertion)
- {
- os_ << " end_bol_ = bol;\n";
- }
- os_ << " end_token_ = curr_;\n";
- os_ << " }\n";
- os_ << " }\n\n";
- }
- os_ << " if (end_state_)\n";
- os_ << " {\n";
- os_ << " // return longest match\n";
- os_ << " start_token_ = end_token_;\n";
- if (dfas_ > 1)
- {
- os_ << " start_state_ = end_start_state_;\n";
- os_ << " if (id_ == 0)\n";
- os_ << " {\n";
- if (sm_.data()._seen_BOL_assertion)
- {
- os_ << " bol = end_bol_;\n";
- }
- os_ << " goto again;\n";
- os_ << " }\n";
- if (sm_.data()._seen_BOL_assertion)
- {
- os_ << " else\n";
- os_ << " {\n";
- os_ << " bol_ = end_bol_;\n";
- os_ << " }\n";
- }
- }
- else if (sm_.data()._seen_BOL_assertion)
- {
- os_ << " bol_ = end_bol_;\n";
- }
- os_ << " }\n";
- os_ << " else\n";
- os_ << " {\n";
- if (sm_.data()._seen_BOL_assertion)
- {
- os_ << " bol_ = (*start_token_ == '\\n') ? true : false;\n";
- }
- os_ << " id_ = npos;\n";
- os_ << " uid_ = npos;\n";
- os_ << " }\n\n";
- os_ << " unique_id_ = uid_;\n";
- os_ << " return id_;\n";
- return os_.good();
- }
- ///////////////////////////////////////////////////////////////////////////
- template <typename Char>
- inline std::basic_string<Char> get_charlit(Char ch)
- {
- std::basic_string<Char> result;
- boost::lexer::basic_string_token<Char>::escape_char(ch, result);
- return result;
- }
- // check whether state0_0 is referenced from any of the other states
- template <typename Char>
- bool need_label0_0(boost::lexer::basic_state_machine<Char> const &sm_)
- {
- typedef typename boost::lexer::basic_state_machine<Char>::iterator
- iterator_type;
- iterator_type iter_ = sm_.begin();
- std::size_t const states_ = iter_->states;
- for (std::size_t state_ = 0; state_ < states_; ++state_)
- {
- if (0 == iter_->bol_index || 0 == iter_->eol_index)
- {
- return true;
- }
- std::size_t const transitions_ = iter_->transitions;
- for (std::size_t t_ = 0; t_ < transitions_; ++t_)
- {
- if (0 == iter_->goto_state)
- {
- return true;
- }
- ++iter_;
- }
- if (transitions_ == 0) ++iter_;
- }
- return false;
- }
- ///////////////////////////////////////////////////////////////////////////
- template <typename Char>
- bool generate_function_body_switch(std::basic_ostream<Char> & os_
- , boost::lexer::basic_state_machine<Char> const &sm_)
- {
- typedef typename boost::lexer::basic_state_machine<Char>::iterator
- iterator_type;
- std::size_t const lookups_ = sm_.data()._lookup->front ()->size ();
- iterator_type iter_ = sm_.begin();
- iterator_type labeliter_ = iter_;
- iterator_type end_ = sm_.end();
- std::size_t const dfas_ = sm_.data()._dfa->size ();
- os_ << " static std::size_t const npos = "
- "static_cast<std::size_t>(~0);\n";
- os_ << "\n if (start_token_ == end_)\n";
- os_ << " {\n";
- os_ << " unique_id_ = npos;\n";
- os_ << " return 0;\n";
- os_ << " }\n\n";
- if (sm_.data()._seen_BOL_assertion)
- {
- os_ << " bool bol = bol_;\n";
- }
- if (dfas_ > 1)
- {
- os_ << "again:\n";
- }
- os_ << " Iterator curr_ = start_token_;\n";
- os_ << " bool end_state_ = false;\n";
- os_ << " std::size_t id_ = npos;\n";
- os_ << " std::size_t uid_ = npos;\n";
- if (dfas_ > 1)
- {
- os_ << " std::size_t end_start_state_ = start_state_;\n";
- }
- if (sm_.data()._seen_BOL_assertion)
- {
- os_ << " bool end_bol_ = bol_;\n";
- }
- os_ << " Iterator end_token_ = start_token_;\n";
- os_ << '\n';
- os_ << " " << ((lookups_ == 256) ? "char" : "wchar_t")
- << " ch_ = 0;\n\n";
- if (dfas_ > 1)
- {
- os_ << " switch (start_state_)\n";
- os_ << " {\n";
- for (std::size_t i_ = 0; i_ < dfas_; ++i_)
- {
- os_ << " case " << i_ << ":\n";
- os_ << " goto state" << i_ << "_0;\n";
- os_ << " break;\n";
- }
- os_ << " default:\n";
- os_ << " goto end;\n";
- os_ << " break;\n";
- os_ << " }\n";
- }
- bool need_state0_0_label = need_label0_0(sm_);
- for (std::size_t dfa_ = 0; dfa_ < dfas_; ++dfa_)
- {
- std::size_t const states_ = iter_->states;
- for (std::size_t state_ = 0; state_ < states_; ++state_)
- {
- std::size_t const transitions_ = iter_->transitions;
- std::size_t t_ = 0;
- if (dfas_ > 1 || dfa_ != 0 || state_ != 0 || need_state0_0_label)
- {
- os_ << "\nstate" << dfa_ << '_' << state_ << ":\n";
- }
- if (iter_->end_state)
- {
- os_ << " end_state_ = true;\n";
- os_ << " id_ = " << iter_->id << ";\n";
- os_ << " uid_ = " << iter_->unique_id << ";\n";
- os_ << " end_token_ = curr_;\n";
- if (dfas_ > 1)
- {
- os_ << " end_start_state_ = " << iter_->goto_dfa <<
- ";\n";
- }
- if (sm_.data()._seen_BOL_assertion)
- {
- os_ << " end_bol_ = bol;\n";
- }
- if (transitions_) os_ << '\n';
- }
- if (t_ < transitions_ ||
- iter_->bol_index != boost::lexer::npos ||
- iter_->eol_index != boost::lexer::npos)
- {
- os_ << " if (curr_ == end_) goto end;\n";
- os_ << " ch_ = *curr_;\n";
- if (iter_->bol_index != boost::lexer::npos)
- {
- os_ << "\n if (bol) goto state" << dfa_ << '_'
- << iter_->bol_index << ";\n";
- }
- if (iter_->eol_index != boost::lexer::npos)
- {
- os_ << "\n if (ch_ == '\\n') goto state" << dfa_
- << '_' << iter_->eol_index << ";\n";
- }
- os_ << " ++curr_;\n";
- }
- for (/**/; t_ < transitions_; ++t_)
- {
- Char const *ptr_ = iter_->token._charset.c_str();
- Char const *end_ = ptr_ + iter_->token._charset.size();
- Char start_char_ = 0;
- Char curr_char_ = 0;
- bool range_ = false;
- bool first_char_ = true;
- os_ << "\n if (";
- while (ptr_ != end_)
- {
- curr_char_ = *ptr_++;
- if (*ptr_ == curr_char_ + 1)
- {
- if (!range_)
- {
- start_char_ = curr_char_;
- }
- range_ = true;
- }
- else
- {
- if (!first_char_)
- {
- os_ << ((iter_->token._negated) ? " && " : " || ");
- }
- else
- {
- first_char_ = false;
- }
- if (range_)
- {
- if (iter_->token._negated)
- {
- os_ << "!";
- }
- os_ << "(ch_ >= '" << get_charlit(start_char_)
- << "' && ch_ <= '"
- << get_charlit(curr_char_) << "')";
- range_ = false;
- }
- else
- {
- os_ << "ch_ "
- << ((iter_->token._negated) ? "!=" : "==")
- << " '" << get_charlit(curr_char_) << "'";
- }
- }
- }
- os_ << ") goto state" << dfa_ << '_' << iter_->goto_state
- << ";\n";
- ++iter_;
- }
- if (!(dfa_ == dfas_ - 1 && state_ == states_ - 1))
- {
- os_ << " goto end;\n";
- }
- if (transitions_ == 0) ++iter_;
- }
- }
- os_ << "\nend:\n";
- os_ << " if (end_state_)\n";
- os_ << " {\n";
- os_ << " // return longest match\n";
- os_ << " start_token_ = end_token_;\n";
- if (dfas_ > 1)
- {
- os_ << " start_state_ = end_start_state_;\n";
- os_ << "\n if (id_ == 0)\n";
- os_ << " {\n";
- if (sm_.data()._seen_BOL_assertion)
- {
- os_ << " bol = end_bol_;\n";
- }
- os_ << " goto again;\n";
- os_ << " }\n";
- if (sm_.data()._seen_BOL_assertion)
- {
- os_ << " else\n";
- os_ << " {\n";
- os_ << " bol_ = end_bol_;\n";
- os_ << " }\n";
- }
- }
- else if (sm_.data()._seen_BOL_assertion)
- {
- os_ << " bol_ = end_bol_;\n";
- }
- os_ << " }\n";
- os_ << " else\n";
- os_ << " {\n";
- if (sm_.data()._seen_BOL_assertion)
- {
- os_ << " bol_ = (*start_token_ == '\\n') ? true : false;\n";
- }
- os_ << " id_ = npos;\n";
- os_ << " uid_ = npos;\n";
- os_ << " }\n\n";
- os_ << " unique_id_ = uid_;\n";
- os_ << " return id_;\n";
- return os_.good();
- }
- ///////////////////////////////////////////////////////////////////////////
- // Generate a tokenizer for the given state machine.
- template <typename Char, typename F>
- inline bool
- generate_cpp (boost::lexer::basic_state_machine<Char> const& sm_
- , boost::lexer::basic_rules<Char> const& rules_
- , std::basic_ostream<Char> &os_, Char const* name_suffix
- , F generate_function_body)
- {
- if (sm_.data()._lookup->empty())
- return false;
- std::size_t const dfas_ = sm_.data()._dfa->size();
- // std::size_t const lookups_ = sm_.data()._lookup->front()->size();
- os_ << "// Copyright (c) 2008-2009 Ben Hanson\n";
- os_ << "// Copyright (c) 2008-2011 Hartmut Kaiser\n";
- os_ << "//\n";
- os_ << "// Distributed under the Boost Software License, "
- "Version 1.0. (See accompanying\n";
- os_ << "// file licence_1_0.txt or copy at "
- "http://www.boost.org/LICENSE_1_0.txt)\n\n";
- os_ << "// Auto-generated by boost::lexer, do not edit\n\n";
- std::basic_string<Char> guard(name_suffix);
- guard += L<Char>(name_suffix[0] ? "_" : "");
- guard += L<Char>(__DATE__ "_" __TIME__);
- typename std::basic_string<Char>::size_type p =
- guard.find_first_of(L<Char>(": "));
- while (std::string::npos != p)
- {
- guard.replace(p, 1, L<Char>("_"));
- p = guard.find_first_of(L<Char>(": "), p);
- }
- boost::to_upper(guard);
- os_ << "#if !defined(BOOST_SPIRIT_LEXER_NEXT_TOKEN_" << guard << ")\n";
- os_ << "#define BOOST_SPIRIT_LEXER_NEXT_TOKEN_" << guard << "\n\n";
- os_ << "#include <boost/spirit/home/support/detail/lexer/char_traits.hpp>\n\n";
- generate_delimiter(os_);
- os_ << "// the generated table of state names and the tokenizer have to be\n"
- "// defined in the boost::spirit::lex::lexertl::static_ namespace\n";
- os_ << "namespace boost { namespace spirit { namespace lex { "
- "namespace lexertl { namespace static_ {\n\n";
- // generate the lexer state information variables
- if (!generate_cpp_state_info(rules_, os_, name_suffix))
- return false;
- generate_delimiter(os_);
- os_ << "// this function returns the next matched token\n";
- os_ << "template<typename Iterator>\n";
- os_ << "std::size_t next_token" << (name_suffix[0] ? "_" : "")
- << name_suffix << " (";
- if (dfas_ > 1)
- {
- os_ << "std::size_t& start_state_, ";
- }
- else
- {
- os_ << "std::size_t& /*start_state_*/, ";
- }
- if (sm_.data()._seen_BOL_assertion)
- {
- os_ << "bool& bol_, ";
- }
- else
- {
- os_ << "bool& /*bol_*/, ";
- }
- os_ << "\n ";
- os_ << "Iterator &start_token_, Iterator const& end_, ";
- os_ << "std::size_t& unique_id_)\n";
- os_ << "{\n";
- if (!generate_function_body(os_, sm_))
- return false;
- os_ << "}\n\n";
- if (!generate_cpp_state_table<Char>(os_, name_suffix
- , sm_.data()._seen_BOL_assertion, sm_.data()._seen_EOL_assertion))
- {
- return false;
- }
- os_ << "}}}}} // namespace boost::spirit::lex::lexertl::static_\n\n";
- os_ << "#endif\n";
- return os_.good();
- }
- } // namespace detail
- ///////////////////////////////////////////////////////////////////////////
- template <typename Lexer, typename F>
- inline bool
- generate_static(Lexer const& lexer
- , std::basic_ostream<typename Lexer::char_type>& os
- , typename Lexer::char_type const* name_suffix, F f)
- {
- if (!lexer.init_dfa(true)) // always minimize DFA for static lexers
- return false;
- return detail::generate_cpp(lexer.state_machine_, lexer.rules_, os
- , name_suffix, f);
- }
- ///////////////////////////////////////////////////////////////////////////
- // deprecated function, will be removed in the future (this has been
- // replaced by the function generate_static_dfa - see below).
- template <typename Lexer>
- inline bool
- generate_static(Lexer const& lexer
- , std::basic_ostream<typename Lexer::char_type>& os
- , typename Lexer::char_type const* name_suffix =
- detail::L<typename Lexer::char_type>())
- {
- return generate_static(lexer, os, name_suffix
- , &detail::generate_function_body_dfa<typename Lexer::char_type>);
- }
- ///////////////////////////////////////////////////////////////////////////
- template <typename Lexer>
- inline bool
- generate_static_dfa(Lexer const& lexer
- , std::basic_ostream<typename Lexer::char_type>& os
- , typename Lexer::char_type const* name_suffix =
- detail::L<typename Lexer::char_type>())
- {
- return generate_static(lexer, os, name_suffix
- , &detail::generate_function_body_dfa<typename Lexer::char_type>);
- }
- ///////////////////////////////////////////////////////////////////////////
- template <typename Lexer>
- inline bool
- generate_static_switch(Lexer const& lexer
- , std::basic_ostream<typename Lexer::char_type>& os
- , typename Lexer::char_type const* name_suffix =
- detail::L<typename Lexer::char_type>())
- {
- return generate_static(lexer, os, name_suffix
- , &detail::generate_function_body_switch<typename Lexer::char_type>);
- }
- ///////////////////////////////////////////////////////////////////////////////
- }}}}
- #endif
|