Added a simple example demonstrating how to use cosmic-ray to improve a test suite.

.gitignore

@@ -11,8 +11,6 @@ test_project/*.json
 .cache
 .hypothesis
 .csearchindex
-
 .idea
 .tox
-
-deploy_key
+deploy_key

docs/conf.py

@@ -71,7 +71,7 @@
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
 # This patterns also effect to html_static_path and html_extra_path
-exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', '.idea']
 
 # The name of the Pygments (syntax highlighting) style to use.
 pygments_style = 'sphinx'

docs/examples.rst

@@ -0,0 +1,4 @@
+Examples
+========
+
+.. include:: ../examples/simple_math/simple_math.rst

docs/index.rst

@@ -40,8 +40,7 @@ And, of course, patches and ideas are welcome.
    implementation
    tests
    continuous_integration
-
-
+   examples
 
 Indices and tables
 ==================

docs/quickstart.rst

@@ -96,3 +96,6 @@ Ray, you can run these tests, too, like this:
 In this case we're passing the ``--verbose`` flag to the ``exec``
 command so that you can see what Cosmic Ray is doing. If everything goes
 as expected, the ``cr-report`` command will report a 0% survival rate.
+
+See :ref:`examples-simple_math` for a step-by-step guide for
+dealing with tests that have a non-zero mutation survival rate.

examples/bowling_game_score_calculator/config.yml

@@ -0,0 +1,12 @@
+module: score_calculator
+
+baseline: 10
+
+exclude-modules:
+
+test-runner:
+  name: unittest
+  args: test_score_calculator
+
+execution-engine:
+  name: local

examples/bowling_game_score_calculator/score_calculator.py

@@ -0,0 +1,48 @@
+"""
+This is a simple class to demonstrate the cosmic-ray library.
+The BowlingGame class keeps and calculates the score of a ten-pin bowling
+game for one player.
+The traditional bowling scoring is used:
+https://en.wikipedia.org/wiki/Ten-pin_bowling#Traditional_scoring
+"""
+
+
+ALL_PINS = 10
+
+class BowlingGame():
+    def __init__(self):
+        self.score_count = 0
+        self.spare = False
+        self.strike = False
+
+    def score(self):
+        return self.score_count
+
+    def roll(self, first_roll, second_roll):
+        frame_result = first_roll + second_roll
+        self._handle_spare_and_strikes(first_roll, frame_result)
+        self.score_count += frame_result
+
+    def _handle_spare_and_strikes(self, first_roll, frame_result):
+        self._award_previous_spare_count(first_roll)
+        self._award_previous_strike_count(frame_result)
+        self._check_for_strike(first_roll)
+        self._check_for_spare(frame_result)
+
+    def _award_previous_spare_count(self, first_roll):
+        if self.spare == True:
+            self.score_count += first_roll
+            self.spare = False
+
+    def _award_previous_strike_count(self, frame_result):
+        if self.strike == True:
+            self.score_count += frame_result
+            self.strike = False
+
+    def _check_for_strike(self, first_roll):
+        if first_roll == ALL_PINS:
+            self.strike = True
+
+    def _check_for_spare(self, frame_result):
+        if frame_result == ALL_PINS and not self.strike:
+            self.spare = True

examples/bowling_game_score_calculator/test_score_calculator.py

@@ -0,0 +1,62 @@
+import unittest
+
+from score_calculator import BowlingGame
+
+
+class ScoreCalculatorTest(unittest.TestCase):
+    def setUp(self):
+        self.game = BowlingGame()
+
+    def test_create_bowling_game(self):
+        self.assertIsInstance(self.game, BowlingGame)
+
+    def test_the_score_of_a_new_game_is_zero(self):
+        self.assertEqual(self.game.score(), 0)
+
+    def test_the_count_of_the_first_frame_is_added_to_the_score(self):
+        self.game.roll(2, 3)
+        self.assertEqual(self.game.score(), 5)
+
+    def test_multiple_frame_results_are_kept_in_the_score(self):
+        self.game.roll(2, 4)
+        self.game.roll(6, 2)
+        self.assertEqual(self.game.score(), 14)
+
+    def test_spares_are_detected_for_the_next_frame(self):
+        self.game.roll(6, 4)
+        self.assertTrue(self.game.spare)
+
+    def test_previous_spare_results_in_that_next_roll_points_are_doubled(self):
+        self.game.roll(6, 4)
+        self.game.roll(5, 3)
+        self.assertEqual(self.game.score(), 23)
+
+    def test_double_spares_are_counted_correctly(self):
+        self.game.roll(6, 4)
+        self.game.roll(5, 5)
+        self.game.roll(8, 0)
+        self.assertEqual(self.game.score(), 41)
+
+    def test_the_spare_flag_is_removed_in_the_next_frame(self):
+        self.game.roll(6, 4)
+        self.game.roll(1, 1)
+        self.game.roll(2, 0)
+        self.assertEqual(self.game.score(), 15)
+
+    def test_a_strike_is_detected_and_no_spare_flag_is_set(self):
+        self.game.roll(10, 0)
+        self.assertTrue(self.game.strike)
+        self.assertFalse(self.game.spare)
+
+    def test_previous_strike_doubles_the_next_frame_pin_count(self):
+        self.game.roll(10, 0)
+        self.game.roll(5, 4)
+        self.assertEqual(self.game.score(), 28)
+
+    def test_the_strike_flag_is_removed_in_the_next_frame(self):
+        self.game.roll(10, 0)
+        self.game.roll(1, 1)
+        self.game.roll(2, 5)
+        self.assertEqual(self.game.score(), 21)
+
+        # case with strike after spare or the way around

examples/simple_math/.gitignore

@@ -0,0 +1 @@
+*.json

examples/simple_math/cosmic-ray-bad_tests.conf

@@ -0,0 +1,13 @@
+# Run the simple_math tests with pytest
+module: simple_math
+
+baseline: 10
+
+exclude-modules:
+
+test-runner:
+  name: pytest
+  args: -x test_simple_math_bad.py
+
+execution-engine:
+  name: local

examples/simple_math/cosmic-ray-good_tests.conf

@@ -0,0 +1,13 @@
+# Run the simple_math tests with pytest
+module: simple_math
+
+baseline: 10
+
+exclude-modules:
+
+test-runner:
+  name: pytest
+  args: -x test_simple_math_good.py
+
+execution-engine:
+  name: local

examples/simple_math/simple_math.py

@@ -0,0 +1,26 @@
+"""
+-----------
+Simple Math
+-----------
+
+A set of simple math functions.
+This is paired up with a test suite and intended to be run with cosmic-ray.
+The idea is that cosmic-ray should kill every mutant when that suite is run;
+if it doesn't, then we've got a problem.
+"""
+
+
+def mult_by_2(x):
+    return x + x
+
+
+def square(x):
+    return x*x
+
+
+def cube(x):
+    return x*x*x
+
+
+def is_positive(x):
+    return x > 0

examples/simple_math/simple_math.rst

@@ -0,0 +1,73 @@
+.. _examples-simple_math:
+
+Improving the tests for a simple module
+---------------------------------------
+
+This example demonstrates how to use cosmic-ray to improve the testing
+suite for a module called ``simple_math``. The code is located in the
+``examples/simple_math`` directory.
+
+::
+
+    # examples/simple_math/simple_math.py
+
+    def mult_by_2(x):
+        return x + x
+
+    def square(x):
+        return x*x
+
+    def cube(x):
+        return x*x*x
+
+    def is_positive(x):
+        return x > 0
+
+
+We would like to measure the performance of a testing suite,
+``test_simple_math_bad.py``, with intention to improve it.
+First run cosmic-ray on the so-called 'bad' testing suite.
+
+::
+
+    cosmic-ray init cosmic-ray-bad_tests.conf bad_session
+    cosmic-ray --verbose exec bad_session
+    cosmic-ray dump bad_session | cr-report
+
+You should end up with at least one mutant that survives. This is because the test
+``test_mult_by_2`` from ``test_simple_math_bad.py`` still passes when we replace
+``x + x`` with ``x * x`` or ``x ** x``, as they all return the same answer, ``4``,
+when ``x = 2``.
+
+Here is the bad test that lets the mutant(s) survive:
+
+.. code-block:: python
+
+    # examples/simple_math/test_simple_math_bad.py
+
+    def test_mult_by_2():
+        assert mult_by_2(2) == 4
+
+To fix this bad test, we decorate it so that a range
+of values of `x` are tested:
+
+.. code-block:: python
+
+    # examples/simple_math/test_simple_math_good.py
+
+    @pytest.mark.parametrize('x', range(-5, 5))
+    def test_mult_by_2(x):
+        assert mult_by_2(x) == x * 2
+
+Now this test should fail for all the mutations to the underlying
+function ``mult_by_2``, which is what we want it to do.
+Run cosmic-ray again on the new testing suite, ``test_simple_math_good.py``
+
+::
+
+    cosmic-ray init cosmic-ray-good_tests.conf good_session
+    cosmic-ray --verbose exec good_session
+    cosmic-ray dump good_session | cr-report
+
+You should now get 0% survival rate for the mutants. This means your
+testing suite is now more robust.

examples/simple_math/test_simple_math_bad.py

@@ -0,0 +1,25 @@
+from simple_math import square, cube, mult_by_2, is_positive
+
+
+def test_square():
+    assert square(3) == 9
+
+
+def test_cube():
+    assert cube(2) == 8
+
+
+def test_mult_by_2():
+    assert mult_by_2(2) == 4
+
+
+def test_is_positive_for_positive_numbers():
+    assert is_positive(1)
+    assert is_positive(2)
+    assert is_positive(3)
+
+
+def test_is_positive_for_non_positive_numbers():
+    assert not is_positive(0)
+    assert not is_positive(-1)
+    assert not is_positive(-2)

examples/simple_math/test_simple_math_good.py

@@ -0,0 +1,28 @@
+from simple_math import square, cube, mult_by_2, is_positive
+
+import pytest
+
+def test_square():
+    assert square(3) == 9
+
+
+def test_cube():
+    assert cube(2) == 8
+
+
+@pytest.mark.parametrize('x', range(-5, 5))
+def test_mult_by_2(x):
+    assert mult_by_2(x) == x * 2
+
+
+def test_is_positive_for_positive_numbers():
+    assert is_positive(1)
+    assert is_positive(2)
+    assert is_positive(3)
+
+
+def test_is_positive_for_non_positive_numbers():
+    assert not is_positive(0)
+    assert not is_positive(-1)
+    assert not is_positive(-2)
+