1import unittest
 2
 3import solution
 4
 5
 6class TestSanity(unittest.TestCase):
 7    """Check if all data is present."""
 8
 9    def test_requirements(self):
10        pass
11
12
13if __name__ == "__main__":
14    unittest.main()

  1import contextlib
  2import importlib
  3import io
  4import sys
  5import textwrap
  6import unittest
  7
  8import bangaranga
  9
 10
 11STUDENT_MODULE_NAME = "solution"
 12CORRECT_IMPLEMENTATION = "correct"
 13
 14
 15def format_student_output(output):
 16    return textwrap.indent(f"Output:\n{output}", "      | ", lambda line: True)
 17
 18
 19def iter_tests(suite):
 20    for item in suite:
 21        if isinstance(item, unittest.TestSuite):
 22            yield from iter_tests(item)
 23        else:
 24            yield item
 25
 26
 27class RecordingTextTestResult(unittest.TextTestResult):
 28    def startTestRun(self):
 29        super().startTestRun()
 30        self.successful_test_ids = set()
 31        self.failed_test_ids = set()
 32
 33    def addSuccess(self, test):
 34        super().addSuccess(test)
 35        self.successful_test_ids.add(test.id())
 36
 37    def addFailure(self, test, err):
 38        super().addFailure(test, err)
 39        self.failed_test_ids.add(test.id())
 40
 41    def addError(self, test, err):
 42        super().addError(test, err)
 43        self.failed_test_ids.add(test.id())
 44
 45
 46class RecordingTextTestRunner(unittest.TextTestRunner):
 47    resultclass = RecordingTextTestResult
 48
 49
 50def run_student_tests_against(implementation_name, only_test_ids=None):
 51    bangaranga._select_implementation(implementation_name)
 52    sys.modules.pop(STUDENT_MODULE_NAME, None)
 53    importlib.invalidate_caches()
 54    output = io.StringIO()
 55    try:
 56        with contextlib.redirect_stdout(output), contextlib.redirect_stderr(output):
 57            student_module = importlib.import_module(STUDENT_MODULE_NAME)
 58            suite = unittest.defaultTestLoader.loadTestsFromModule(student_module)
 59            if only_test_ids is not None:
 60                suite = unittest.TestSuite(test for test in iter_tests(suite) if test.id() in only_test_ids)
 61            result = RecordingTextTestRunner(stream=output, verbosity=0).run(suite)
 62    except BaseException as error:
 63        return {
 64            "was_successful": False,
 65            "successful_test_ids": set(),
 66            "failed_test_ids": {"<import-or-discovery-error>"},
 67            "output": f"{output.getvalue()}\n{type(error).__name__}: {error}",
 68        }
 69    finally:
 70        bangaranga._select_implementation(CORRECT_IMPLEMENTATION)
 71    return {
 72        "was_successful": result.wasSuccessful(),
 73        "successful_test_ids": result.successful_test_ids,
 74        "failed_test_ids": result.failed_test_ids,
 75        "output": output.getvalue(),
 76    }
 77
 78
 79class StudentTestSuiteEvaluationTests(unittest.TestCase):
 80    def test_student_tests_pass_against_correct_implementation(self):
 81        result = run_student_tests_against(CORRECT_IMPLEMENTATION)
 82        student_output = format_student_output(result["output"])
 83        self.assertTrue(
 84            result["was_successful"], f"Student tests should pass against the correct implementation.\n{student_output}"
 85        )
 86
 87
 88def make_bad_implementation_test(implementation_name):
 89    def test(self):
 90        correct_result = run_student_tests_against(CORRECT_IMPLEMENTATION)
 91        valid_test_ids = correct_result["successful_test_ids"]
 92        self.assertTrue(
 93            valid_test_ids,
 94            f"No student tests passed against the correct implementation.\n{format_student_output(correct_result['output'])}",
 95        )
 96
 97        bad_result = run_student_tests_against(implementation_name, only_test_ids=valid_test_ids)
 98        self.assertTrue(
 99            bad_result["failed_test_ids"],
100            f"Student tests passed against incorrect implementation: {implementation_name!r}.\n{format_student_output(bad_result['output'])}",
101        )
102
103    test.__name__ = f"test_student_tests_catch_{implementation_name}"
104    return test
105
106
107for implementation_name in bangaranga._IMPLEMENTATIONS:
108    if implementation_name == CORRECT_IMPLEMENTATION:
109        continue
110    test_name = f"test_student_tests_catch_{implementation_name}"
111    setattr(StudentTestSuiteEvaluationTests, test_name, make_bad_implementation_test(implementation_name))
112
113
114if __name__ == "__main__":
115    unittest.main()