diff --git a/tests/subtests/summaries/test_default.json b/tests/subtests/summaries/test_default.json index 72b94ffe..cbea11ff 100644 --- a/tests/subtests/summaries/test_default.json +++ b/tests/subtests/summaries/test_default.json @@ -1,4 +1,121 @@ { + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": { + "status": "failed", + "image_status": "diff", + "hash_status": null, + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/result-failed-diff.png", + "rms": 24.536840442259244, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": { + "status": "failed", + "image_status": "diff", + "hash_status": null, + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/result-failed-diff.png", + "rms": 24.537034749650488, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": { + "status": "failed", + "image_status": "diff", + "hash_status": null, + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/result-failed-diff.png", + "rms": 24.563779228775037, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, "subtests.subtest.test_functions.test_hmatch_imatch": { "status": "passed", "image_status": "match", @@ -219,5 +336,83 @@ "result_image": null, "baseline_hash": null, "result_hash": null + }, + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": { + "status": "failed", + "image_status": "diff", + "hash_status": null, + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/result-failed-diff.png", + "rms": 24.536840442259244, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": { + "status": "failed", + "image_status": "diff", + "hash_status": null, + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/result-failed-diff.png", + "rms": 24.537034749650488, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": { + "status": "failed", + "image_status": "diff", + "hash_status": null, + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/result-failed-diff.png", + "rms": 24.520257314106637, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": null, + "result_hash": null } -} \ No newline at end of file +} diff --git a/tests/subtests/summaries/test_generate.json b/tests/subtests/summaries/test_generate.json index 2f4e25af..f5f2cf7f 100644 --- a/tests/subtests/summaries/test_generate.json +++ b/tests/subtests/summaries/test_generate.json @@ -1,4 +1,121 @@ { + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, "subtests.subtest.test_functions.test_hmatch_imatch": { "status": "skipped", "image_status": "generated", @@ -219,5 +336,83 @@ "result_image": null, "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null + }, + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null } -} \ No newline at end of file +} diff --git a/tests/subtests/summaries/test_generate_hashes_only.json b/tests/subtests/summaries/test_generate_hashes_only.json index 80344e20..aaba44b9 100644 --- a/tests/subtests/summaries/test_generate_hashes_only.json +++ b/tests/subtests/summaries/test_generate_hashes_only.json @@ -1,4 +1,121 @@ { + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/result-failed-diff.png", + "rms": 24.536840442259244, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/result-failed-diff.png", + "rms": 24.537034749650488, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/result-failed-diff.png", + "rms": 24.563779228775037, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, "subtests.subtest.test_functions.test_hmatch_imatch": { "status": "passed", "image_status": "match", @@ -219,5 +336,83 @@ "result_image": null, "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null + }, + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/result-failed-diff.png", + "rms": 24.536840442259244, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/result-failed-diff.png", + "rms": 24.537034749650488, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/result-failed-diff.png", + "rms": 24.520257314106637, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null } -} \ No newline at end of file +} diff --git a/tests/subtests/summaries/test_generate_images_only.json b/tests/subtests/summaries/test_generate_images_only.json index c216958a..9e701649 100644 --- a/tests/subtests/summaries/test_generate_images_only.json +++ b/tests/subtests/summaries/test_generate_images_only.json @@ -1,4 +1,121 @@ { + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, "subtests.subtest.test_functions.test_hmatch_imatch": { "status": "skipped", "image_status": "generated", @@ -219,5 +336,83 @@ "result_image": null, "baseline_hash": null, "result_hash": null + }, + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null } -} \ No newline at end of file +} diff --git a/tests/subtests/summaries/test_hash.json b/tests/subtests/summaries/test_hash.json index 2231d618..f463019d 100644 --- a/tests/subtests/summaries/test_hash.json +++ b/tests/subtests/summaries/test_hash.json @@ -1,4 +1,121 @@ { + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": { + "status": "failed", + "image_status": null, + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_classes\\.TestClass\\.test_hdiff_idiff_testclass\\.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": { + "status": "failed", + "image_status": null, + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_classes\\.TestClassWithSetupMethod\\.test_hdiff_idiff_testclasswithsetupmethod\\.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": { + "status": "failed", + "image_status": null, + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_classes\\.TestClassWithSetupClass\\.test_hdiff_idiff_testclasswithsetupclass\\.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, "subtests.subtest.test_functions.test_hmatch_imatch": { "status": "passed", "image_status": null, @@ -102,5 +219,83 @@ "result_image": "subtests.subtest.test_special.test_hdiff_imatch_removetext/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": { + "status": "failed", + "image_status": null, + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_unittest\\.TestCase\\.test_hdiff_idiff_testclass\\.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": { + "status": "failed", + "image_status": null, + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_unittest\\.TestCaseWithSetUp\\.test_hdiff_idiff_testcasewithsetup\\.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": { + "status": "failed", + "image_status": null, + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_unittest\\.TestCaseWithSetUpClass\\.test_hdiff_idiff_testcasewithsetupclass\\.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" } -} \ No newline at end of file +} diff --git a/tests/subtests/summaries/test_html_generate.json b/tests/subtests/summaries/test_html_generate.json index 28190c7a..5d25715f 100644 --- a/tests/subtests/summaries/test_html_generate.json +++ b/tests/subtests/summaries/test_html_generate.json @@ -1,4 +1,121 @@ { + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, "subtests.subtest.test_functions.test_hmatch_imatch": { "status": "skipped", "image_status": "generated", @@ -219,5 +336,83 @@ "result_image": null, "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null + }, + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null } -} \ No newline at end of file +} diff --git a/tests/subtests/summaries/test_html_generate_hashes_only.json b/tests/subtests/summaries/test_html_generate_hashes_only.json index 918335e7..3036ca89 100644 --- a/tests/subtests/summaries/test_html_generate_hashes_only.json +++ b/tests/subtests/summaries/test_html_generate_hashes_only.json @@ -1,4 +1,121 @@ { + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/result-failed-diff.png", + "rms": 24.536840442259244, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/result-failed-diff.png", + "rms": 24.537034749650488, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/result-failed-diff.png", + "rms": 24.563779228775037, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, "subtests.subtest.test_functions.test_hmatch_imatch": { "status": "passed", "image_status": "match", @@ -219,5 +336,83 @@ "result_image": "subtests.subtest.test_special.test_hdiff_imatch_removetext/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null + }, + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/result-failed-diff.png", + "rms": 24.536840442259244, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/result-failed-diff.png", + "rms": 24.537034749650488, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/result-failed-diff.png", + "rms": 24.520257314106637, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null } -} \ No newline at end of file +} diff --git a/tests/subtests/summaries/test_html_generate_images_only.json b/tests/subtests/summaries/test_html_generate_images_only.json index 44d03ea2..0b1c3eb8 100644 --- a/tests/subtests/summaries/test_html_generate_images_only.json +++ b/tests/subtests/summaries/test_html_generate_images_only.json @@ -1,4 +1,121 @@ { + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, "subtests.subtest.test_functions.test_hmatch_imatch": { "status": "skipped", "image_status": "generated", @@ -219,5 +336,83 @@ "result_image": null, "baseline_hash": null, "result_hash": null + }, + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null } -} \ No newline at end of file +} diff --git a/tests/subtests/summaries/test_html_hashes_only.json b/tests/subtests/summaries/test_html_hashes_only.json index 257c7a4e..c613126c 100644 --- a/tests/subtests/summaries/test_html_hashes_only.json +++ b/tests/subtests/summaries/test_html_hashes_only.json @@ -1,4 +1,121 @@ { + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": { + "status": "failed", + "image_status": null, + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_classes\\.TestClass\\.test_hdiff_idiff_testclass\\.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": { + "status": "failed", + "image_status": null, + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_classes\\.TestClassWithSetupMethod\\.test_hdiff_idiff_testclasswithsetupmethod\\.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": { + "status": "failed", + "image_status": null, + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_classes\\.TestClassWithSetupClass\\.test_hdiff_idiff_testclasswithsetupclass\\.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, "subtests.subtest.test_functions.test_hmatch_imatch": { "status": "passed", "image_status": null, @@ -102,5 +219,83 @@ "result_image": "subtests.subtest.test_special.test_hdiff_imatch_removetext/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": { + "status": "failed", + "image_status": null, + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_unittest\\.TestCase\\.test_hdiff_idiff_testclass\\.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": { + "status": "failed", + "image_status": null, + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_unittest\\.TestCaseWithSetUp\\.test_hdiff_idiff_testcasewithsetup\\.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": { + "status": "failed", + "image_status": null, + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_unittest\\.TestCaseWithSetUpClass\\.test_hdiff_idiff_testcasewithsetupclass\\.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" } -} \ No newline at end of file +} diff --git a/tests/subtests/summaries/test_html_images_only.json b/tests/subtests/summaries/test_html_images_only.json index e636c7b2..04534f58 100644 --- a/tests/subtests/summaries/test_html_images_only.json +++ b/tests/subtests/summaries/test_html_images_only.json @@ -1,4 +1,121 @@ { + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": { + "status": "failed", + "image_status": "diff", + "hash_status": null, + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/result-failed-diff.png", + "rms": 24.536840442259244, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": { + "status": "failed", + "image_status": "diff", + "hash_status": null, + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/result-failed-diff.png", + "rms": 24.537034749650488, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": { + "status": "failed", + "image_status": "diff", + "hash_status": null, + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/result-failed-diff.png", + "rms": 24.563779228775037, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second/result.png", + "baseline_hash": null, + "result_hash": null + }, "subtests.subtest.test_functions.test_hmatch_imatch": { "status": "passed", "image_status": "match", @@ -219,5 +336,83 @@ "result_image": "subtests.subtest.test_special.test_hdiff_imatch_removetext/result.png", "baseline_hash": null, "result_hash": null + }, + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": { + "status": "failed", + "image_status": "diff", + "hash_status": null, + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/result-failed-diff.png", + "rms": 24.536840442259244, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": { + "status": "failed", + "image_status": "diff", + "hash_status": null, + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/result-failed-diff.png", + "rms": 24.537034749650488, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": { + "status": "failed", + "image_status": "diff", + "hash_status": null, + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/result-failed-diff.png", + "rms": 24.520257314106637, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass/result.png", + "baseline_hash": null, + "result_hash": null } -} \ No newline at end of file +} diff --git a/tests/subtests/summaries/test_hybrid.json b/tests/subtests/summaries/test_hybrid.json index 0c4e7f15..62aa6269 100644 --- a/tests/subtests/summaries/test_hybrid.json +++ b/tests/subtests/summaries/test_hybrid.json @@ -1,4 +1,121 @@ { + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": { + "status": "failed", + "image_status": "diff", + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_classes\\.TestClass\\.test_hdiff_idiff_testclass\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/result-failed-diff.png", + "rms": 24.536840442259244, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": { + "status": "failed", + "image_status": "diff", + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_classes\\.TestClassWithSetupMethod\\.test_hdiff_idiff_testclasswithsetupmethod\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/result-failed-diff.png", + "rms": 24.537034749650488, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": { + "status": "failed", + "image_status": "diff", + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_classes\\.TestClassWithSetupClass\\.test_hdiff_idiff_testclasswithsetupclass\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/result-failed-diff.png", + "rms": 24.563779228775037, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, "subtests.subtest.test_functions.test_hmatch_imatch": { "status": "passed", "image_status": null, @@ -219,5 +336,83 @@ "result_image": "subtests.subtest.test_special.test_hdiff_imatch_removetext/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": { + "status": "failed", + "image_status": "diff", + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_unittest\\.TestCase\\.test_hdiff_idiff_testclass\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/result-failed-diff.png", + "rms": 24.536840442259244, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": { + "status": "failed", + "image_status": "diff", + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_unittest\\.TestCaseWithSetUp\\.test_hdiff_idiff_testcasewithsetup\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/result-failed-diff.png", + "rms": 24.537034749650488, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": { + "status": "failed", + "image_status": "diff", + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_unittest\\.TestCaseWithSetUpClass\\.test_hdiff_idiff_testcasewithsetupclass\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/result-failed-diff.png", + "rms": 24.520257314106637, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" } -} \ No newline at end of file +} diff --git a/tests/subtests/summaries/test_results_always.json b/tests/subtests/summaries/test_results_always.json index 6d51b863..e59d399d 100644 --- a/tests/subtests/summaries/test_results_always.json +++ b/tests/subtests/summaries/test_results_always.json @@ -1,4 +1,121 @@ { + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": { + "status": "passed", + "image_status": "match", + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded.", + "baseline_image": "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": { + "status": "failed", + "image_status": "diff", + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_classes\\.TestClass\\.test_hdiff_idiff_testclass\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/result-failed-diff.png", + "rms": 24.536840442259244, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": { + "status": "passed", + "image_status": "match", + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded.", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": { + "status": "failed", + "image_status": "diff", + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_classes\\.TestClassWithSetupMethod\\.test_hdiff_idiff_testclasswithsetupmethod\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/result-failed-diff.png", + "rms": 24.537034749650488, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": { + "status": "passed", + "image_status": "match", + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded.", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": { + "status": "failed", + "image_status": "diff", + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_classes\\.TestClassWithSetupClass\\.test_hdiff_idiff_testclasswithsetupclass\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/result-failed-diff.png", + "rms": 24.563779228775037, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": { + "status": "passed", + "image_status": "match", + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded.", + "baseline_image": "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": { + "status": "passed", + "image_status": "match", + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded.", + "baseline_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": { + "status": "passed", + "image_status": "match", + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded.", + "baseline_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, "subtests.subtest.test_functions.test_hmatch_imatch": { "status": "passed", "image_status": "match", @@ -219,5 +336,83 @@ "result_image": "subtests.subtest.test_special.test_hdiff_imatch_removetext/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": { + "status": "failed", + "image_status": "diff", + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_unittest\\.TestCase\\.test_hdiff_idiff_testclass\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/result-failed-diff.png", + "rms": 24.536840442259244, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": { + "status": "passed", + "image_status": "match", + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded.", + "baseline_image": "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": { + "status": "failed", + "image_status": "diff", + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_unittest\\.TestCaseWithSetUp\\.test_hdiff_idiff_testcasewithsetup\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/result-failed-diff.png", + "rms": 24.537034749650488, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": { + "status": "passed", + "image_status": "match", + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded.", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": { + "status": "failed", + "image_status": "diff", + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_unittest\\.TestCaseWithSetUpClass\\.test_hdiff_idiff_testcasewithsetupclass\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/result-failed-diff.png", + "rms": 24.520257314106637, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": { + "status": "passed", + "image_status": "match", + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded.", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" } -} \ No newline at end of file +}