tests.validation_test.make_tst_function() - Code Metrics - nekedome/project-euler - Measure and Improve Code Quality continuously with Scrutinizer

tests.validation_test.make_tst_function() A
last analyzed 2018-07-11 15:35 UTC

↳ Parent: tests.validation_test

Complexity

Conditions

Size

Total Lines	25
Code Lines	10

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
cc	1
eloc	10
nop	4
dl	0
loc	25
rs	9.9
c	0
b	0
f	0

"""
:mod:`tests.validation_test` -- Validation of Implemented Solutions
===================================================================

.. module:: tests.validation_test
   :synopsis: Validation of all implemented solutions.

.. moduleauthor:: Bill Maroney <[email protected]>
"""

import csv
import importlib
import os
import pkgutil
import re
import unittest
from time import time
from typing import Callable, Union

SOLUTION_MODULE_ROOT = "solutions"
SOLUTION_MODULE_PATH = "{}.problem".format(SOLUTION_MODULE_ROOT) + "{}"
DOC_ROOT = "docs"


results = {}  # global, shared, variable to hold the results for all validation tests



def teardown_module(module):

    """ The teardown (module) function is called when all tests in this module are complete

    This function will collect and report on the results of all validation tests. That is, their correctness and their

    run-times. These reports are collated into ordered CSV files, each capturing up to 100 contiguous tests:


    * tests 1-100
    * tests 101-200
    * etc.

    These CSVs will be picked up by other code to ultimately build the documented project. These CSVs will contribute

    to the tables of solutions and their respective run-times.

    :param module: the module under teardown (i.e. :mod:`tests.validation_test`)
    :return: None
    """

    def build_csv(start_problem: int, end_problem: int) -> None:
        """ Helper function to build a CSV for 100 problems in a 10x10 grid """

        # Check that start_problem and end_problem define a block of 100 problems
        assert isinstance(start_problem, int), "start_problem must be an integer"
        assert isinstance(end_problem, int), "end_problem must be an integer"
        assert start_problem % 100 == 1, "start_problem must be 1 modulo 100"
        assert end_problem % 100 == 0, "end_problem must be 0 modulo 100"
        assert start_problem < end_problem, "start_problem must be less than end_problem"
        assert end_problem - start_problem + 1 == 100, "start_problem and end_problem must be 100 apart"


        global results  # map the local results variable to the globally shared variable


        header = [""] + ["***{}**".format(i % 10) for i in range(1, 11)]

        path = os.path.join(DOC_ROOT, "{}-{}.csv".format(start_problem, end_problem))
        with open(path, "w", newline="") as fp:

            # Open a CSV file and create a header row
            cw = csv.writer(fp)

            cw.writerow(header)

            row = []  # temporary list to hold each row as it is populated

            for i in range(start_problem, end_problem + 1):
                # Add the row range to the start of a CSV row
                if i % 10 == 1:
                    row.append("**{} - {}**".format(i, i + 9))

                # Add the results for problem number i to the next cell in the CSV
                try:
                    if results[i]["correct"]:
                        row.append("|tick| :doc:`{:.2f} <solutions/{}>`".format(results[i]["time"], i))

                    else:
                        row.append("|warning| :doc:`{:.2f} <solutions/{}>`".format(results[i]["time"], i))

                except KeyError:
                    # KeyError caused by results[i], i.e. there isn't a solution for problem number i

                    row.append("|cross|")

                # Flush the 10-long row to the CSV file
                if i % 10 == 0:
                    cw.writerow(row)
                    row = []  # blank row for the next one

    # Construct CSVs in blocks of 100 problems
    build_csv(1, 100)
    build_csv(101, 200)
    build_csv(201, 300)
    build_csv(301, 400)
    build_csv(401, 500)
    build_csv(501, 600)
    build_csv(601, 700)


class TestSolutions(unittest.TestCase):
    """ Will contain all registered solutions once :func:`tests.validation_test.register_solutions` is invoked """

    pass


def make_failure(message: str):
    """ Build a test that unconditionally fails, used to flag errors in dynamic test generation

    :param message: the description of the error
    :return: a unit test function
    """

    def test(self):
class SomeClass:
    def some_method(self):
        """Do x and return foo."""
        self.fail(message)  # unconditionally fail with the provided error message

    return test


def make_tst_function(description: str, problem_number: int, solver: Callable[[None], Union[int, str]],

                      expected_answer: Union[int, str]):
    """ Build a test that computes the answer to the given problem and checks its correctness

    :param description: a label to attach to this test case
    :param problem_number: the Project Euler problem number
    :param solver: the function that computes the answer
    :param expected_answer: the expected answer to this problem
    :return: a unit test function
    """

    def test(self):
class SomeClass:
    def some_method(self):
        """Do x and return foo."""
        # Compute the answer using the given solver, note the time taken
        t0 = time()

        computed_answer = solver()
        t1 = time()


        # Record the results (correctness and run-time) in the global, shared, variable
        global results

        results[problem_number] = {"correct": computed_answer == expected_answer, "time": t1 - t0}

        # Check that the computed answer matches the expected one
        self.assertEqual(computed_answer, expected_answer, description)

    return test


def register_solutions():
    """ Dynamically load and run each solution present, test the computed answers match expected answers """


    # Dynamically identify all solutions and for each solved problem, check the answer for correctness

    for importer, modname, is_package in pkgutil.iter_modules([SOLUTION_MODULE_ROOT]):

        rem = re.match("^problem(?P<problem_number>\d+)", modname)

        problem_number = int(rem.group("problem_number"))
        test_name = "test_problem_{:03d}".format(problem_number)

        # Load this solution
        try:
            mod = importlib.import_module("{}.{}".format(SOLUTION_MODULE_ROOT, modname))
        except ModuleNotFoundError:

            err_msg = "solution for problem {} doesn't currently exist.".format(problem_number)
            test_func = make_failure(err_msg)
            setattr(TestSolutions, test_name, test_func)
            continue  # cannot run the test

        # Retrieve the expected answer
        try:
            expected_answer = mod.expected_answer
        except AttributeError:
            err_msg = "expected answer for problem {} doesn't currently exist.".format(problem_number)

            test_func = make_failure(err_msg)
            setattr(TestSolutions, test_name, test_func)
            continue  # cannot validate answer without an expected one

        # Check that the expected answer has a value
        if expected_answer is None:
            err_msg = "expected answer for problem {} hasn't been set (i.e. is still None).".format(problem_number)

            test_func = make_failure(err_msg)
            setattr(TestSolutions, test_name, test_func)
            continue  # cannot validate answer without an expected one

        # Register a test function for this solution
        test_func = make_tst_function(test_name, problem_number, mod.solve, mod.expected_answer)
        setattr(TestSolutions, test_name, test_func)


1			"""
2			:mod:`tests.validation_test` -- Validation of Implemented Solutions
3			===================================================================
4
5			.. module:: tests.validation_test
6			:synopsis: Validation of all implemented solutions.
7
8			.. moduleauthor:: Bill Maroney <[email protected]>
9			"""
10
11			import csv
12			import importlib
13			import os
14			import pkgutil
15			import re
16			import unittest
17			from time import time
18			from typing import Callable, Union
19
20			SOLUTION_MODULE_ROOT = "solutions"
21			SOLUTION_MODULE_PATH = "{}.problem".format(SOLUTION_MODULE_ROOT) + "{}"
22			DOC_ROOT = "docs"
23
24
25			results = {} # global, shared, variable to hold the results for all validation tests
			0 ignored issues – show Coding Style Naming introduced 2018-05-28 04:15 UTC by Report Bug Copy Issue Report The name `results` does not conform to the constant naming conventions (`(([A-Z_][A-Z0-9_])\|(__.__))$`). This check looks for invalid names for a range of different identifiers. You can set regular expressions to which the identifiers must conform if the defaults do not match your requirements. If your project includes a Pylint configuration file, the settings contained in that file take precedence. To find out more about Pylint, please refer to their site. Loading history...
26
27
28			def teardown_module(module):
			0 ignored issues – show Unused Code introduced 2018-05-28 04:15 UTC by Report Bug Copy Issue Report The argument `module` seems to be unused. Loading history...
29			""" The teardown (module) function is called when all tests in this module are complete
30
31			This function will collect and report on the results of all validation tests. That is, their correctness and their
			0 ignored issues – show Coding Style introduced 2018-05-28 04:15 UTC by Report Bug Copy Issue Report This line is too long as per the coding-style (118/100). This check looks for lines that are too long. You can specify the maximum line length. Loading history...
32			run-times. These reports are collated into ordered CSV files, each capturing up to 100 contiguous tests:
			0 ignored issues – show Coding Style introduced 2018-05-28 04:15 UTC by Report Bug Copy Issue Report This line is too long as per the coding-style (108/100). This check looks for lines that are too long. You can specify the maximum line length. Loading history...
33
34			* tests 1-100
35			* tests 101-200
36			* etc.
37
38			These CSVs will be picked up by other code to ultimately build the documented project. These CSVs will contribute
			0 ignored issues – show Coding Style introduced 2018-05-28 04:15 UTC by Report Bug Copy Issue Report This line is too long as per the coding-style (117/100). This check looks for lines that are too long. You can specify the maximum line length. Loading history...
39			to the tables of solutions and their respective run-times.
40
41			:param module: the module under teardown (i.e. :mod:`tests.validation_test`)
42			:return: None
43			"""
44
45			def build_csv(start_problem: int, end_problem: int) -> None:
46			""" Helper function to build a CSV for 100 problems in a 10x10 grid """
47
48			# Check that start_problem and end_problem define a block of 100 problems
49			assert isinstance(start_problem, int), "start_problem must be an integer"
50			assert isinstance(end_problem, int), "end_problem must be an integer"
51			assert start_problem % 100 == 1, "start_problem must be 1 modulo 100"
52			assert end_problem % 100 == 0, "end_problem must be 0 modulo 100"
53			assert start_problem < end_problem, "start_problem must be less than end_problem"
54			assert end_problem - start_problem + 1 == 100, "start_problem and end_problem must be 100 apart"
			0 ignored issues – show Coding Style introduced 2018-05-28 04:15 UTC by Report Bug Copy Issue Report This line is too long as per the coding-style (104/100). This check looks for lines that are too long. You can specify the maximum line length. Loading history...
55
56			global results # map the local results variable to the globally shared variable
			0 ignored issues – show Coding Style Naming introduced 2018-05-28 04:15 UTC by Report Bug Copy Issue Report The name `results` does not conform to the constant naming conventions (`(([A-Z_][A-Z0-9_])\|(__.__))$`). This check looks for invalid names for a range of different identifiers. You can set regular expressions to which the identifiers must conform if the defaults do not match your requirements. If your project includes a Pylint configuration file, the settings contained in that file take precedence. To find out more about Pylint, please refer to their site. Loading history... Coding Style introduced 2018-05-28 04:15 UTC by Report Bug Copy Issue Report Usage of the `global` statement should be avoided. Usage of `global` can make code hard to read and test, its usage is generally not recommended unless you are dealing with legacy code. Loading history...
57
58			header = [""] + ["*{}".format(i % 10) for i in range(1, 11)]
59
60			path = os.path.join(DOC_ROOT, "{}-{}.csv".format(start_problem, end_problem))
61			with open(path, "w", newline="") as fp:
			0 ignored issues – show Coding Style Naming introduced 2018-05-28 04:15 UTC by Report Bug Copy Issue Report The name `fp` does not conform to the variable naming conventions (`(([a-z][a-z0-9_]{2,30})\|(_[a-z0-9_]*))$`). This check looks for invalid names for a range of different identifiers. You can set regular expressions to which the identifiers must conform if the defaults do not match your requirements. If your project includes a Pylint configuration file, the settings contained in that file take precedence. To find out more about Pylint, please refer to their site. Loading history...
62			# Open a CSV file and create a header row
63			cw = csv.writer(fp)
			0 ignored issues – show Coding Style Naming introduced 2018-05-28 04:15 UTC by Report Bug Copy Issue Report The name `cw` does not conform to the variable naming conventions (`(([a-z][a-z0-9_]{2,30})\|(_[a-z0-9_]*))$`). This check looks for invalid names for a range of different identifiers. You can set regular expressions to which the identifiers must conform if the defaults do not match your requirements. If your project includes a Pylint configuration file, the settings contained in that file take precedence. To find out more about Pylint, please refer to their site. Loading history...
64			cw.writerow(header)
65
66			row = [] # temporary list to hold each row as it is populated
67
68			for i in range(start_problem, end_problem + 1):
69			# Add the row range to the start of a CSV row
70			if i % 10 == 1:
71			row.append("{} - {}".format(i, i + 9))
72
73			# Add the results for problem number i to the next cell in the CSV
74			try:
75			if results[i]["correct"]:
76			row.append("\|tick\| :doc:`{:.2f} <solutions/{}>`".format(results[i]["time"], i))
			0 ignored issues – show Coding Style introduced 2018-05-28 04:15 UTC by Report Bug Copy Issue Report This line is too long as per the coding-style (103/100). This check looks for lines that are too long. You can specify the maximum line length. Loading history...
77			else:
78			row.append("\|warning\| :doc:`{:.2f} <solutions/{}>`".format(results[i]["time"], i))
			0 ignored issues – show Coding Style introduced 2018-05-28 04:15 UTC by Report Bug Copy Issue Report This line is too long as per the coding-style (106/100). This check looks for lines that are too long. You can specify the maximum line length. Loading history...
79			except KeyError:
80			# KeyError caused by results[i], i.e. there isn't a solution for problem number i
			0 ignored issues – show Coding Style introduced 2018-05-28 04:15 UTC by Report Bug Copy Issue Report This line is too long as per the coding-style (101/100). This check looks for lines that are too long. You can specify the maximum line length. Loading history...
81			row.append("\|cross\|")
82
83			# Flush the 10-long row to the CSV file
84			if i % 10 == 0:
85			cw.writerow(row)
86			row = [] # blank row for the next one
87
88			# Construct CSVs in blocks of 100 problems
89			build_csv(1, 100)
90			build_csv(101, 200)
91			build_csv(201, 300)
92			build_csv(301, 400)
93			build_csv(401, 500)
94			build_csv(501, 600)
95			build_csv(601, 700)
96
97
98			class TestSolutions(unittest.TestCase):
99			""" Will contain all registered solutions once :func:`tests.validation_test.register_solutions` is invoked """
			0 ignored issues – show Coding Style introduced 2018-05-28 04:15 UTC by Report Bug Copy Issue Report This line is too long as per the coding-style (114/100). This check looks for lines that are too long. You can specify the maximum line length. Loading history...
100			pass
101
102
103			def make_failure(message: str):
104			""" Build a test that unconditionally fails, used to flag errors in dynamic test generation
105
106			:param message: the description of the error
107			:return: a unit test function
108			"""
109
110			def test(self):
			0 ignored issues – show Coding Style introduced 2018-05-28 04:15 UTC by Report Bug Copy Issue Report This function should have a docstring. The coding style of this project requires that you add a docstring to this code element. Below, you find an example for methods: class SomeClass: def some_method(self): """Do x and return foo.""" If you would like to know more about docstrings, we recommend to read PEP-257: Docstring Conventions. Loading history...
111			self.fail(message) # unconditionally fail with the provided error message
112
113			return test
114
115
116			def make_tst_function(description: str, problem_number: int, solver: Callable[[None], Union[int, str]],
			0 ignored issues – show Coding Style introduced 2018-05-28 04:15 UTC by Report Bug Copy Issue Report This line is too long as per the coding-style (103/100). This check looks for lines that are too long. You can specify the maximum line length. Loading history...
117			expected_answer: Union[int, str]):
118			""" Build a test that computes the answer to the given problem and checks its correctness
119
120			:param description: a label to attach to this test case
121			:param problem_number: the Project Euler problem number
122			:param solver: the function that computes the answer
123			:param expected_answer: the expected answer to this problem
124			:return: a unit test function
125			"""
126
127			def test(self):
			0 ignored issues – show Coding Style introduced 2018-05-28 04:15 UTC by Report Bug Copy Issue Report This function should have a docstring. The coding style of this project requires that you add a docstring to this code element. Below, you find an example for methods: class SomeClass: def some_method(self): """Do x and return foo.""" If you would like to know more about docstrings, we recommend to read PEP-257: Docstring Conventions. Loading history...
128			# Compute the answer using the given solver, note the time taken
129			t0 = time()
			0 ignored issues – show Coding Style Naming introduced 2018-05-28 04:15 UTC by Report Bug Copy Issue Report The name `t0` does not conform to the variable naming conventions (`(([a-z][a-z0-9_]{2,30})\|(_[a-z0-9_]*))$`). This check looks for invalid names for a range of different identifiers. You can set regular expressions to which the identifiers must conform if the defaults do not match your requirements. If your project includes a Pylint configuration file, the settings contained in that file take precedence. To find out more about Pylint, please refer to their site. Loading history...
130			computed_answer = solver()
131			t1 = time()
			0 ignored issues – show Coding Style Naming introduced 2018-05-28 04:15 UTC by Report Bug Copy Issue Report The name `t1` does not conform to the variable naming conventions (`(([a-z][a-z0-9_]{2,30})\|(_[a-z0-9_]*))$`). This check looks for invalid names for a range of different identifiers. You can set regular expressions to which the identifiers must conform if the defaults do not match your requirements. If your project includes a Pylint configuration file, the settings contained in that file take precedence. To find out more about Pylint, please refer to their site. Loading history...
132
133			# Record the results (correctness and run-time) in the global, shared, variable
134			global results
			0 ignored issues – show Coding Style Naming introduced 2018-05-28 04:15 UTC by Report Bug Copy Issue Report The name `results` does not conform to the constant naming conventions (`(([A-Z_][A-Z0-9_])\|(__.__))$`). This check looks for invalid names for a range of different identifiers. You can set regular expressions to which the identifiers must conform if the defaults do not match your requirements. If your project includes a Pylint configuration file, the settings contained in that file take precedence. To find out more about Pylint, please refer to their site. Loading history... Coding Style introduced 2018-05-28 04:15 UTC by Report Bug Copy Issue Report Usage of the `global` statement should be avoided. Usage of `global` can make code hard to read and test, its usage is generally not recommended unless you are dealing with legacy code. Loading history...
135			results[problem_number] = {"correct": computed_answer == expected_answer, "time": t1 - t0}
136
137			# Check that the computed answer matches the expected one
138			self.assertEqual(computed_answer, expected_answer, description)
139
140			return test
141
142
143			def register_solutions():
144			""" Dynamically load and run each solution present, test the computed answers match expected answers """
			0 ignored issues – show Coding Style introduced 2018-05-28 04:15 UTC by Report Bug Copy Issue Report This line is too long as per the coding-style (108/100). This check looks for lines that are too long. You can specify the maximum line length. Loading history...
145
146			# Dynamically identify all solutions and for each solved problem, check the answer for correctness
			0 ignored issues – show Coding Style introduced 2018-05-28 04:15 UTC by Report Bug Copy Issue Report This line is too long as per the coding-style (102/100). This check looks for lines that are too long. You can specify the maximum line length. Loading history...
147			for importer, modname, is_package in pkgutil.iter_modules([SOLUTION_MODULE_ROOT]):
			0 ignored issues – show Unused Code introduced 2018-05-28 11:31 UTC by Report Bug Copy Issue Report The variable `importer` seems to be unused. Loading history... Unused Code introduced 2018-05-28 11:31 UTC by Report Bug Copy Issue Report The variable `is_package` seems to be unused. Loading history...
148			rem = re.match("^problem(?P<problem_number>\d+)", modname)
			0 ignored issues – show Bug introduced 2018-05-28 04:15 UTC by Report Bug Copy Issue Report A suspicious escape sequence `\d` was found. Did you maybe forget to add an `r` prefix? Escape sequences in Python are generally interpreted according to rules similar to standard C. Only if strings are prefixed with `r` or `R` are they interpreted as regular expressions. The escape sequence that was used indicates that you might have intended to write a regular expression. Learn more about the available escape sequences. in the Python documentation. Loading history...
149			problem_number = int(rem.group("problem_number"))
150			test_name = "test_problem_{:03d}".format(problem_number)
151
152			# Load this solution
153			try:
154			mod = importlib.import_module("{}.{}".format(SOLUTION_MODULE_ROOT, modname))
155			except ModuleNotFoundError:
			0 ignored issues – show Comprehensibility Best Practice introduced 2018-05-28 04:15 UTC by Report Bug Copy Issue Report The variable `ModuleNotFoundError` does not seem to be defined. Loading history... Comprehensibility Best Practice introduced 2018-05-28 04:15 UTC by Report Bug Copy Issue Report Undefined variable 'ModuleNotFoundError' Loading history...
156			err_msg = "solution for problem {} doesn't currently exist.".format(problem_number)
157			test_func = make_failure(err_msg)
158			setattr(TestSolutions, test_name, test_func)
159			continue # cannot run the test
160
161			# Retrieve the expected answer
162			try:
163			expected_answer = mod.expected_answer
164			except AttributeError:
165			err_msg = "expected answer for problem {} doesn't currently exist.".format(problem_number)
			0 ignored issues – show Coding Style introduced 2018-05-28 04:15 UTC by Report Bug Copy Issue Report This line is too long as per the coding-style (102/100). This check looks for lines that are too long. You can specify the maximum line length. Loading history...
166			test_func = make_failure(err_msg)
167			setattr(TestSolutions, test_name, test_func)
168			continue # cannot validate answer without an expected one
169
170			# Check that the expected answer has a value
171			if expected_answer is None:
172			err_msg = "expected answer for problem {} hasn't been set (i.e. is still None).".format(problem_number)
			0 ignored issues – show Coding Style introduced 2018-05-28 04:15 UTC by Report Bug Copy Issue Report This line is too long as per the coding-style (115/100). This check looks for lines that are too long. You can specify the maximum line length. Loading history...
173			test_func = make_failure(err_msg)
174			setattr(TestSolutions, test_name, test_func)
175			continue # cannot validate answer without an expected one
176
177			# Register a test function for this solution
178			test_func = make_tst_function(test_name, problem_number, mod.solve, mod.expected_answer)
179			setattr(TestSolutions, test_name, test_func)
180

nekedome / project-euler

tests.validation_test.make_tst_function() A last analyzed 2018-07-11 15:35 UTC

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like

tests.validation_test.make_tst_function() A
last analyzed 2018-07-11 15:35 UTC