123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120 |
- <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
- <html>
- <head>
- <meta http-equiv="Content-Language" content="en-us">
- <meta http-equiv="Content-Type" content="text/html; charset=us-ascii">
- <meta name="GENERATOR" content="Microsoft FrontPage 6.0">
- <meta name="ProgId" content="FrontPage.Editor.Document">
- <title>Introduction</title>
- </head>
- <body bgcolor="#FFFFFF">
- <p><img src="../../../boost.png" alt="C++ Boost" width="277" height=
- "86"/><br></p>
- <h1 align="center">Introduction</h1>
- <p align="left">The Boost Tokenizer package provides a flexible and
- easy-to-use way to break a string or other character sequence into a series
- of tokens. Below is a simple example that will break up a phrase into
- words.</p>
- <div align="left">
- <pre>
- // simple_example_1.cpp
- #include<iostream>
- #include<boost/tokenizer.hpp>
- #include<string>
- int main(){
- using namespace std;
- using namespace boost;
- string s = "This is, a test";
- tokenizer<> tok(s);
- for(tokenizer<>::iterator beg=tok.begin(); beg!=tok.end();++beg){
- cout << *beg << "\n";
- }
- }
- </pre>
- </div>
- <p align="left">You can choose how the string gets parsed by using the
- TokenizerFunction. If you do not specify anything, the default
- TokenizerFunction is <em>char_delimiters_separator<char></em> which
- defaults to breaking up a string based on space and punctuation. Here is an
- example using another TokenizerFunction called
- <em>escaped_list_separator</em>. This TokenizerFunction parses a superset
- of comma-separated value (CSV) lines. The format looks like this:</p>
- <p align="left">Field 1,"putting quotes around fields, allows commas",Field
- 3</p>
- <p align="left">Below is an example that will break the previous line into
- its three fields.</p>
- <div align="left">
- <pre>
- // simple_example_2.cpp
- #include<iostream>
- #include<boost/tokenizer.hpp>
- #include<string>
- int main(){
- using namespace std;
- using namespace boost;
- string s = "Field 1,\"putting quotes around fields, allows commas\",Field 3";
- tokenizer<escaped_list_separator<char> > tok(s);
- for(tokenizer<escaped_list_separator<char> >::iterator beg=tok.begin(); beg!=tok.end();++beg){
- cout << *beg << "\n";
- }
- }
- </pre>
- </div>
- <p align="left">Finally, for some TokenizerFunctions you have to pass
- something into the constructor in order to do anything interesting. An
- example is the offset_separator. This class breaks a string into tokens based
- on offsets. For example, when <em>12252001</em> is parsed using offsets of
- 2,2,4 it becomes <em>12 25 2001</em>. Below is the code used.</p>
- <div align="left">
- <pre>
- // simple_example_3.cpp
- #include<iostream>
- #include<boost/tokenizer.hpp>
- #include<string>
- int main(){
- using namespace std;
- using namespace boost;
- string s = "12252001";
- int offsets[] = {2,2,4};
- offset_separator f(offsets, offsets+3);
- tokenizer<offset_separator> tok(s,f);
- for(tokenizer<offset_separator>::iterator beg=tok.begin(); beg!=tok.end();++beg){
- cout << *beg << "\n";
- }
- }
- </pre>
- </div>
- <p align="left"> </p>
- <hr>
- <p><a href="http://validator.w3.org/check?uri=referer"><img border="0" src=
- "../../doc/images/valid-html401.png" alt="Valid HTML 4.01 Transitional"
- height="31" width="88"></a></p>
- <p>Revised
- <!--webbot bot="Timestamp" s-type="EDITED" s-format="%d %B %Y" startspan -->9 June 2010<!--webbot bot="Timestamp" endspan i-checksum="38518" --></p>
- <p><i>Copyright © 2001 John R. Bandela</i></p>
- <p><i>Distributed under the Boost Software License, Version 1.0. (See
- accompanying file <a href="../../LICENSE_1_0.txt">LICENSE_1_0.txt</a> or
- copy at <a href=
- "http://www.boost.org/LICENSE_1_0.txt">http://www.boost.org/LICENSE_1_0.txt</a>)</i></p>
- </body>
- </html>
|