123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284 |
- <html>
- <head>
- <meta http-equiv="Content-Type" content="text/html; charset=US-ASCII">
- <title>Data-driven test cases</title>
- <link rel="stylesheet" href="../../../boostbook.css" type="text/css">
- <meta name="generator" content="DocBook XSL Stylesheets V1.79.1">
- <link rel="home" href="../../../index.html" title="Boost.Test">
- <link rel="up" href="../test_cases.html" title="Test cases">
- <link rel="prev" href="test_organization_nullary.html" title="Test cases without parameters">
- <link rel="next" href="test_case_generation/datasets.html" title="Datasets">
- </head>
- <body bgcolor="white" text="black" link="#0000FF" vlink="#840084" alink="#0000FF">
- <table cellpadding="2" width="100%"><tr>
- <td valign="top"><img alt="Boost C++ Libraries" width="277" height="86" src="../../../../../../../boost.png"></td>
- <td align="center"><a href="../../../../../../../index.html">Home</a></td>
- <td align="center"><a href="../../../../../../../libs/libraries.htm">Libraries</a></td>
- <td align="center"><a href="http://www.boost.org/users/people.html">People</a></td>
- <td align="center"><a href="http://www.boost.org/users/faq.html">FAQ</a></td>
- <td align="center"><a href="../../../../../../../more/index.htm">More</a></td>
- </tr></table>
- <hr>
- <div class="spirit-nav">
- <a accesskey="p" href="test_organization_nullary.html"><img src="../../../../../../../doc/src/images/prev.png" alt="Prev"></a><a accesskey="u" href="../test_cases.html"><img src="../../../../../../../doc/src/images/up.png" alt="Up"></a><a accesskey="h" href="../../../index.html"><img src="../../../../../../../doc/src/images/home.png" alt="Home"></a><a accesskey="n" href="test_case_generation/datasets.html"><img src="../../../../../../../doc/src/images/next.png" alt="Next"></a>
- </div>
- <div class="section">
- <div class="titlepage"><div><div><h4 class="title">
- <a name="boost_test.tests_organization.test_cases.test_case_generation"></a><a class="link" href="test_case_generation.html" title="Data-driven test cases">Data-driven
- test cases</a>
- </h4></div></div></div>
- <div class="toc"><dl class="toc">
- <dt><span class="section"><a href="test_case_generation/datasets.html">Datasets</a></span></dt>
- <dd><dl>
- <dt><span class="section"><a href="test_case_generation/datasets.html#boost_test.tests_organization.test_cases.test_case_generation.datasets.dataset_interface">Dataset
- interface</a></span></dt>
- <dt><span class="section"><a href="test_case_generation/datasets.html#boost_test.tests_organization.test_cases.test_case_generation.datasets.dataset_creation_and_delayed_cre">Dataset
- creation and delayed creation</a></span></dt>
- </dl></dd>
- <dt><span class="section"><a href="test_case_generation/datasets_auto_registration.html">Declaring
- and registering test cases with datasets</a></span></dt>
- <dt><span class="section"><a href="test_case_generation/operations.html">Operations
- on dataset</a></span></dt>
- <dd><dl>
- <dt><span class="section"><a href="test_case_generation/operations.html#boost_test.tests_organization.test_cases.test_case_generation.operations.joins">Joins</a></span></dt>
- <dt><span class="section"><a href="test_case_generation/operations.html#boost_test.tests_organization.test_cases.test_case_generation.operations.zips">Zips</a></span></dt>
- <dt><span class="section"><a href="test_case_generation/operations.html#boost_test.tests_organization.test_cases.test_case_generation.operations.grid_cartesian_products">Grid
- (Cartesian products)</a></span></dt>
- </dl></dd>
- <dt><span class="section"><a href="test_case_generation/generators.html">Datasets
- generators</a></span></dt>
- </dl></div>
- <h5>
- <a name="boost_test.tests_organization.test_cases.test_case_generation.h0"></a>
- <span class="phrase"><a name="boost_test.tests_organization.test_cases.test_case_generation.why_data_driven_test_cases"></a></span><a class="link" href="test_case_generation.html#boost_test.tests_organization.test_cases.test_case_generation.why_data_driven_test_cases">Why
- data-driven test cases?</a>
- </h5>
- <p>
- Some tests are required to be repeated for a series of different input
- parameters. One way to achieve this is manually register a test case for
- each parameter. You can also invoke a test function with all parameters
- manually from within your test case, like this:
- </p>
- <pre class="programlisting"><span class="keyword">void</span> <span class="identifier">single_test</span><span class="special">(</span> <span class="keyword">int</span> <span class="identifier">i</span> <span class="special">)</span>
- <span class="special">{</span>
- <a class="link" href="../../utf_reference/testing_tool_ref/assertion_boost_test_universal_macro.html" title="BOOST_TEST"><code class="computeroutput"><span class="identifier">BOOST_TEST</span></code></a><span class="special">(</span> <span class="comment">/* test assertion */</span> <span class="special">);</span>
- <span class="special">}</span>
- <span class="keyword">void</span> <span class="identifier">combined_test</span><span class="special">()</span>
- <span class="special">{</span>
- <span class="keyword">int</span> <span class="identifier">params</span><span class="special">[]</span> <span class="special">=</span> <span class="special">{</span> <span class="number">1</span><span class="special">,</span> <span class="number">2</span><span class="special">,</span> <span class="number">3</span><span class="special">,</span> <span class="number">4</span><span class="special">,</span> <span class="number">5</span> <span class="special">};</span>
- <span class="identifier">std</span><span class="special">::</span><span class="identifier">for_each</span><span class="special">(</span> <span class="identifier">params</span><span class="special">,</span> <span class="identifier">params</span><span class="special">+</span><span class="number">5</span><span class="special">,</span> <span class="special">&</span><span class="identifier">single_test</span> <span class="special">);</span>
- <span class="special">}</span>
- </pre>
- <p>
- The approach above has several drawbacks:
- </p>
- <div class="itemizedlist"><ul class="itemizedlist" style="list-style-type: disc; ">
- <li class="listitem">
- the logic for running the tests is inside a test itself: <code class="computeroutput"><span class="identifier">single_test</span></code> in the above example
- is run from the test case <code class="computeroutput"><span class="identifier">combined_test</span></code>
- while its execution would be better handled by the <span class="emphasis"><em>Unit Test
- Framework</em></span>
- </li>
- <li class="listitem">
- in case of fatal failure for one of the values in <code class="computeroutput"><span class="identifier">param</span></code>
- array above (say a failure in <a class="link" href="../../utf_reference/testing_tool_ref/assertion_boost_test_universal_macro.html" title="BOOST_TEST"><code class="computeroutput"><span class="identifier">BOOST_TEST_REQUIRE</span></code></a>), the test
- <code class="computeroutput"><span class="identifier">combined_test</span></code> is aborted
- and the next test-case in the test tree is executed.
- </li>
- <li class="listitem">
- in case of failure, the reporting is not accurate enough: the test
- should certainly be reran during debugging sessions by a human or additional
- logic for reporting should be implemented in the test itself.
- </li>
- </ul></div>
- <h5>
- <a name="boost_test.tests_organization.test_cases.test_case_generation.h1"></a>
- <span class="phrase"><a name="boost_test.tests_organization.test_cases.test_case_generation.parameter_generation_scalability"></a></span><a class="link" href="test_case_generation.html#boost_test.tests_organization.test_cases.test_case_generation.parameter_generation_scalability">Parameter
- generation, scalability and composition</a>
- </h5>
- <p>
- In some circumstance, one would like to run a parametrized test over an
- <span class="emphasis"><em>arbitrary large</em></span> set of values. Enumerating the parameters
- by hand is not a solution that scales well, especially when these parameters
- can be described in another function that generates these values. However,
- this solution has also limitations
- </p>
- <div class="itemizedlist"><ul class="itemizedlist" style="list-style-type: disc; ">
- <li class="listitem">
- <span class="bold"><strong>Generating functions</strong></span>: suppose we have
- a function <code class="computeroutput"><span class="identifier">func</span><span class="special">(</span><span class="keyword">float</span> <span class="identifier">f</span><span class="special">)</span></code>, where <code class="computeroutput"><span class="identifier">f</span></code>
- is any number in [0, 1]. We are not interested that much in the exact
- value, but we would like to test <code class="computeroutput"><span class="identifier">func</span></code>.
- What about, instead of writing the <code class="computeroutput"><span class="identifier">f</span></code>
- for which <code class="computeroutput"><span class="identifier">func</span></code> will
- be tested against, we choose randomly <code class="computeroutput"><span class="identifier">f</span></code>
- in [0, 1]? And also what about instead of having only one value for
- <code class="computeroutput"><span class="identifier">f</span></code>, we run the test
- on arbitrarily many numbers? We easily understand from this small example
- that tests requiring parameters are more powerful when, instead of
- writing down constant values in the test, a generating function is
- provided.
- </li>
- <li class="listitem">
- <span class="bold"><strong>Scalability</strong></span>: suppose we have a test
- case on <code class="computeroutput"><span class="identifier">func1</span></code>, on which
- we test <code class="computeroutput"><span class="identifier">N</span></code> values written
- as constant in the test file. What does the test ensure? We have the
- guaranty that <code class="computeroutput"><span class="identifier">func1</span></code>
- is working on these <code class="computeroutput"><span class="identifier">N</span></code>
- values. Yet in this setting <code class="computeroutput"><span class="identifier">N</span></code>
- is necessarily finite and usually small. How would we extend or scale
- <code class="computeroutput"><span class="identifier">N</span></code> easily? One solution
- is to be able to generate new values, and to be able to define a test
- on the <span class="bold"><strong>class</strong></span> of possible inputs for
- <code class="computeroutput"><span class="identifier">func1</span></code> on which the
- function should have a defined behavior. To some extent, <code class="computeroutput"><span class="identifier">N</span></code> constant written down in the test
- are just an excerpt of the possible inputs of <code class="computeroutput"><span class="identifier">func1</span></code>,
- and working on the class of inputs gives more flexibility and power
- to the test.
- </li>
- <li class="listitem">
- <p class="simpara">
- <span class="bold"><strong>Composition</strong></span>: suppose we already have
- test cases for two functions <code class="computeroutput"><span class="identifier">func1</span></code>
- and <code class="computeroutput"><span class="identifier">func2</span></code>, taking as
- argument the types <code class="computeroutput"><span class="identifier">T1</span></code>
- and <code class="computeroutput"><span class="identifier">T2</span></code> respectively.
- Now we would like to test a new functions <code class="computeroutput"><span class="identifier">func3</span></code>
- that takes as argument a type <code class="computeroutput"><span class="identifier">T3</span></code>
- containing <code class="computeroutput"><span class="identifier">T1</span></code> and
- <code class="computeroutput"><span class="identifier">T2</span></code>, and calling <code class="computeroutput"><span class="identifier">func1</span></code> and <code class="computeroutput"><span class="identifier">func2</span></code>
- through a known algorithm. An example of such a setting would be
- </p>
- <pre class="programlisting"><span class="comment">// Returns the log of x</span>
- <span class="comment">// Precondition: x strictly positive.</span>
- <span class="keyword">double</span> <span class="identifier">fast_log</span><span class="special">(</span><span class="keyword">double</span> <span class="identifier">x</span><span class="special">);</span>
- <span class="comment">// Returns 1/(x-1)</span>
- <span class="comment">// Precondition: x != 1</span>
- <span class="keyword">double</span> <span class="identifier">fast_inv</span><span class="special">(</span><span class="keyword">double</span> <span class="identifier">x</span><span class="special">);</span>
- <span class="keyword">struct</span> <span class="identifier">dummy</span> <span class="special">{</span>
- <span class="keyword">unsigned</span> <span class="keyword">int</span> <span class="identifier">field1</span><span class="special">;</span>
- <span class="keyword">unsigned</span> <span class="keyword">int</span> <span class="identifier">field2</span><span class="special">;</span>
- <span class="special">};</span>
- <span class="keyword">double</span> <span class="identifier">func3</span><span class="special">(</span><span class="identifier">dummy</span> <span class="identifier">value</span><span class="special">)</span>
- <span class="special">{</span>
- <span class="keyword">return</span> <span class="number">0.5</span> <span class="special">*</span> <span class="special">(</span><span class="identifier">exp</span><span class="special">(</span><span class="identifier">fast_log</span><span class="special">(</span><span class="identifier">value</span><span class="special">.</span><span class="identifier">field1</span><span class="special">))/</span><span class="identifier">value</span><span class="special">.</span><span class="identifier">field1</span> <span class="special">+</span> <span class="identifier">value</span><span class="special">.</span><span class="identifier">field2</span><span class="special">/</span><span class="identifier">fast_inv</span><span class="special">(</span><span class="identifier">value</span><span class="special">.</span><span class="identifier">field2</span><span class="special">));</span>
- <span class="special">}</span>
- </pre>
- <p class="simpara">
- In this example,
- </p>
- <p class="simpara">
- <div class="itemizedlist"><ul class="itemizedlist" style="list-style-type: circle; ">
- <li class="listitem">
- <code class="computeroutput"><span class="identifier">func3</span></code> inherits
- from the preconditions of <code class="computeroutput"><span class="identifier">fast_log</span></code>
- and <code class="computeroutput"><span class="identifier">fast_inv</span></code>:
- it is defined in <code class="computeroutput"><span class="special">(</span><span class="number">0</span><span class="special">,</span> <span class="special">+</span><span class="identifier">infinity</span><span class="special">)</span></code> and in <code class="computeroutput"><span class="special">[-</span><span class="identifier">C</span><span class="special">,</span>
- <span class="special">+</span><span class="identifier">C</span><span class="special">]</span> <span class="special">-</span>
- <span class="special">{</span><span class="number">1</span><span class="special">}</span></code> for <code class="computeroutput"><span class="identifier">field1</span></code>
- and <code class="computeroutput"><span class="identifier">field2</span></code> respectively
- (<code class="computeroutput"><span class="identifier">C</span></code> being a constant
- arbitrarily big).
- </li>
- <li class="listitem">
- as defined above, <code class="computeroutput"><span class="identifier">func3</span></code>
- should be close to 1 everywhere on its definition domain.
- </li>
- <li class="listitem">
- we would like to reuse the properties of <code class="computeroutput"><span class="identifier">fast_log</span></code>
- and <code class="computeroutput"><span class="identifier">fast_inv</span></code>
- in the compound function <code class="computeroutput"><span class="identifier">func3</span></code>
- and assert that <code class="computeroutput"><span class="identifier">func3</span></code>
- is well defined over an arbitrary large definition domain.
- </li>
- </ul></div>
- </p>
- <p class="simpara">
- Having parametrized tests on <code class="computeroutput"><span class="identifier">func3</span></code>
- hardly tells us about the possible numerical properties or instabilities
- close to the point <code class="computeroutput"><span class="special">{</span><span class="identifier">field1</span>
- <span class="special">=</span> <span class="number">0</span><span class="special">,</span> <span class="identifier">field2</span>
- <span class="special">=</span> <span class="number">1</span><span class="special">}</span></code>. Indeed, the parametrized test may
- test for some points around (0,1), but will fail to provide an <span class="bold"><strong>asymptotic behavior</strong></span> of the function close to
- this point.
- </p>
- </li>
- </ul></div>
- <h5>
- <a name="boost_test.tests_organization.test_cases.test_case_generation.h2"></a>
- <span class="phrase"><a name="boost_test.tests_organization.test_cases.test_case_generation.data_driven_tests_in_the_boost_t"></a></span><a class="link" href="test_case_generation.html#boost_test.tests_organization.test_cases.test_case_generation.data_driven_tests_in_the_boost_t">Data
- driven tests in the Boost.Test framework</a>
- </h5>
- <p>
- The facilities provided by the <span class="emphasis"><em>Unit Test Framework</em></span>
- addressed the issues described above:
- </p>
- <div class="itemizedlist"><ul class="itemizedlist" style="list-style-type: disc; ">
- <li class="listitem">
- the notion of <span class="bold"><strong>datasets</strong></span> eases the description
- of the class of inputs for test cases. The datasets also implement
- several operations that enable their combinations to create new, more
- complex datasets,
- </li>
- <li class="listitem">
- two macros, <a class="link" href="../../utf_reference/test_org_reference/test_org_boost_test_dataset.html" title="BOOST_DATA_TEST_CASE"><code class="computeroutput"><span class="identifier">BOOST_DATA_TEST_CASE</span></code></a> and
- <a class="link" href="../../utf_reference/test_org_reference/test_org_boost_test_dataset_fixture.html" title="BOOST_DATA_TEST_CASE_F"><code class="computeroutput"><span class="identifier">BOOST_DATA_TEST_CASE_F</span></code></a>, respectively
- without and with fixture support, are used for the declaration and
- registration of a test case over a collection of values (samples),
- </li>
- <li class="listitem">
- each test case, associated to a unique value, is executed independently
- from others. These tests are guarded in the same way regular test cases
- are, which makes the execution of the tests over each sample of a dataset
- isolated, robust, repeatable and ease the debugging,
- </li>
- <li class="listitem">
- several datasets generating functions are provided by the <span class="emphasis"><em>Unit
- Test Framework</em></span>
- </li>
- </ul></div>
- <p>
- The remainder of this section covers the notions and feature provided by
- the <span class="emphasis"><em>Unit Test Framework</em></span> about the data-driven test
- cases, in particular:
- </p>
- <div class="orderedlist"><ol class="orderedlist" type="1">
- <li class="listitem">
- the notion of <a class="link" href="test_case_generation/datasets.html" title="Datasets"><span class="bold"><strong>dataset</strong></span> and <span class="bold"><strong>sample</strong></span></a>
- is introduced
- </li>
- <li class="listitem">
- <a class="link" href="test_case_generation/datasets_auto_registration.html" title="Declaring and registering test cases with datasets">the
- declaration and registration</a> of the data-driven test cases are
- explained,
- </li>
- <li class="listitem">
- the <a class="link" href="test_case_generation/operations.html" title="Operations on dataset"><span class="emphasis"><em>operations</em></span></a>
- on datasets are detailed
- </li>
- <li class="listitem">
- and finally the built-in <a class="link" href="test_case_generation/generators.html" title="Datasets generators">dataset
- generators</a> are introduced.
- </li>
- </ol></div>
- </div>
- <table xmlns:rev="http://www.cs.rpi.edu/~gregod/boost/tools/doc/revision" width="100%"><tr>
- <td align="left"></td>
- <td align="right"><div class="copyright-footer">Copyright © 2001-2019 Boost.Test
- contributors<p>
- Distributed under the Boost Software License, Version 1.0. (See accompanying
- file LICENSE_1_0.txt or copy at <a href="http://www.boost.org/LICENSE_1_0.txt" target="_top">http://www.boost.org/LICENSE_1_0.txt</a>)
- </p>
- </div></td>
- </tr></table>
- <hr>
- <div class="spirit-nav">
- <a accesskey="p" href="test_organization_nullary.html"><img src="../../../../../../../doc/src/images/prev.png" alt="Prev"></a><a accesskey="u" href="../test_cases.html"><img src="../../../../../../../doc/src/images/up.png" alt="Up"></a><a accesskey="h" href="../../../index.html"><img src="../../../../../../../doc/src/images/home.png" alt="Home"></a><a accesskey="n" href="test_case_generation/datasets.html"><img src="../../../../../../../doc/src/images/next.png" alt="Next"></a>
- </div>
- </body>
- </html>
|